diff options
Diffstat (limited to 'tools')
141 files changed, 2396 insertions, 890 deletions
diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 0921e31878..7fc0c7fe03 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -95,6 +95,19 @@ Binman uses the following terms: - binary - an input binary that goes into the image +Installation +------------ + +You can install binman using:: + + pip install binary-manager + +The name is chosen since binman conflicts with an existing package. + +If you are using binman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/binman/binman`. + + Relationship to FIT ------------------- @@ -838,6 +851,14 @@ offset-from-elf: is the symbol to lookup (relative to elf-base-sym) and <offset> is an offset to add to that value. +preserve: + Indicates that this entry should be preserved by any firmware updates. This + flag should be checked by the updater when it is deciding which entries to + update. This flag is normally attached to sections but can be attached to + a single entry in a section if the updater supports it. Not that binman + itself has no control over the updater's behaviour, so this is just a + signal. It is not enforced by binman. + Examples of the above options can be found in the tests. See the tools/binman/test directory. @@ -1122,8 +1143,7 @@ It is sometimes inconvenient to add a 'binman' node to the .dts file for each board. This can be done by using #include to bring in a common file. Another approach supported by the U-Boot build system is to automatically include a common header. You can then put the binman node (and anything else that is -specific to U-Boot, such as u-boot,dm-pre-reloc properies) in that header -file. +specific to U-Boot, such as bootph-all properies) in that header file. Binman will search for the following files in arch/<arch>/dts:: @@ -1327,6 +1347,22 @@ You can also replace just a selection of entries:: $ binman replace -i image.bin "*u-boot*" -I indir +It is possible to replace whole sections as well, but in that case any +information about entries within the section may become outdated. This is +because Binman cannot know whether things have moved around or resized within +the section, once you have updated its data. + +Technical note: With 'allow-repack', Binman writes information about the +original offset and size properties of each entry, if any were specified, in +the 'orig-offset' and 'orig-size' properties. This allows Binman to distinguish +between an entry which ended up being packed at an offset (or assigned a size) +and an entry which had a particular offset / size requested in the Binman +configuration. Where are particular offset / size was requested, this is treated +as set in stone, so Binman will ensure it doesn't change. Without this feature, +repacking an entry might cause it to disobey the original constraints provided +when it was created. + + Repacking an image involves .. _`BinmanLogging`: @@ -1408,6 +1444,16 @@ You can also use `--fetch all` to fetch all tools or `--fetch <tool>` to fetch a particular tool. Some tools are built from source code, in which case you will need to have at least the `build-essential` and `git` packages installed. +Tools are fetched into the `~/.binman-tools` directory. This directory is +automatically added to the toolpath so there is no need to use `--toolpath` to +specify it. If you want to use these tools outside binman, you may want to +add this directory to your `PATH`. For example, if you use bash, add this to +the end of `.bashrc`:: + + PATH="$HOME/.binman-tools:$PATH" + +To select a custom directory, use the `--tooldir` option. + Bintool Documentation ===================== @@ -1426,8 +1472,9 @@ Binman commands and arguments Usage:: - binman [-h] [-B BUILD_DIR] [-D] [-H] [--toolpath TOOLPATH] [-T THREADS] - [--test-section-timeout] [-v VERBOSITY] [-V] + binman [-h] [-B BUILD_DIR] [-D] [--tooldir TOOLDIR] [-H] + [--toolpath TOOLPATH] [-T THREADS] [--test-section-timeout] + [-v VERBOSITY] [-V] {build,bintool-docs,entry-docs,ls,extract,replace,test,tool} ... Binman provides the following commands: @@ -1452,11 +1499,13 @@ Options: -D, --debug Enabling debugging (provides a full traceback on error) +--tooldir TOOLDIR Set the directory to store tools + -H, --full-help Display the README file --toolpath TOOLPATH - Add a path to the directories containing tools + Add a path to the list of directories containing tools -T THREADS, --threads THREADS Number of threads to use (0=single-thread). Note that -T0 is useful for @@ -1664,6 +1713,12 @@ Options: -m, --map Output a map file for the updated image +-O OUTDIR, --outdir OUTDIR + Path to directory to use for intermediate and output files + +-p, --preserve + Preserve temporary output directory even if option -O is not given + This replaces one or more entries in an existing image. See `Replacing files in an image`_. diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 8fda13ff01..81629683df 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -18,10 +18,10 @@ import shutil import tempfile import urllib.error -from patman import command -from patman import terminal -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib import tout BINMAN_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -43,8 +43,6 @@ FETCH_NAMES = { # Status of tool fetching FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4) -DOWNLOAD_DESTDIR = os.path.join(os.getenv('HOME'), 'bin') - class Bintool: """Tool which operates on binaries to help produce entry contents @@ -53,6 +51,10 @@ class Bintool: # List of bintools to regard as missing missing_list = [] + # Directory to store tools. Note that this set up by set_tool_dir() which + # must be called before this class is used. + tooldir = '' + def __init__(self, name, desc, version_regex=None, version_args='-V'): self.name = name self.desc = desc @@ -112,6 +114,11 @@ class Bintool: obj = cls(name) return obj + @classmethod + def set_tool_dir(cls, pathname): + """Set the path to use to store and find tools""" + cls.tooldir = pathname + def show(self): """Show a line of information about a bintool""" if self.is_present(): @@ -208,7 +215,8 @@ class Bintool: return FAIL if result is not True: fname, tmpdir = result - dest = os.path.join(DOWNLOAD_DESTDIR, self.name) + dest = os.path.join(self.tooldir, self.name) + os.makedirs(self.tooldir, exist_ok=True) print(f"- writing to '{dest}'") shutil.move(fname, dest) if tmpdir: @@ -389,7 +397,7 @@ class Bintool: @classmethod def apt_install(cls, package): - """Install a bintool using the 'aot' tool + """Install a bintool using the 'apt' tool This requires use of servo so may request a password diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index 7efb8391db..f9b16d4c73 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -16,10 +16,10 @@ import urllib.error from binman import bintool from binman.bintool import Bintool -from patman import command -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools # pylint: disable=R0904 class TestBintool(unittest.TestCase): @@ -134,12 +134,14 @@ class TestBintool(unittest.TestCase): dirname = os.path.join(self._indir, 'download_dir') os.mkdir(dirname) fname = os.path.join(dirname, 'downloaded') + + # Rely on bintool to create this directory destdir = os.path.join(self._indir, 'dest_dir') - os.mkdir(destdir) + dest_fname = os.path.join(destdir, '_testing') self.seq = 0 - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', destdir): + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): with unittest.mock.patch.object(tools, 'download', side_effect=handle_download): with test_util.capture_sys_output() as (stdout, _): @@ -250,7 +252,7 @@ class TestBintool(unittest.TestCase): btest = Bintool.create('_testing') col = terminal.Color() self.fname = None - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', self._indir): with unittest.mock.patch.object(tools, 'run', side_effect=fake_run): with test_util.capture_sys_output() as (stdout, _): @@ -344,8 +346,11 @@ class TestBintool(unittest.TestCase): def test_failed_command(self): """Check that running a command that does not exist returns None""" - btool = Bintool.create('_testing') - result = btool.run_cmd_result('fred') + destdir = os.path.join(self._indir, 'dest_dir') + os.mkdir(destdir) + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): + btool = Bintool.create('_testing') + result = btool.run_cmd_result('fred') self.assertIsNone(result) diff --git a/tools/binman/bintools.rst b/tools/binman/bintools.rst index edb373ab59..c30e7eb9ff 100644 --- a/tools/binman/bintools.rst +++ b/tools/binman/bintools.rst @@ -10,6 +10,20 @@ binaries. It is fairly easy to create new bintools. Just add a new file to the +Bintool: bzip2: Compression/decompression using the bzip2 algorithm +------------------------------------------------------------------- + +This bintool supports running `bzip2` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man bzip2 + + + Bintool: cbfstool: Coreboot filesystem (CBFS) tool -------------------------------------------------- @@ -58,6 +72,20 @@ See `Chromium OS vboot documentation`_ for more information. +Bintool: gzip: Compression/decompression using the gzip algorithm +----------------------------------------------------------------- + +This bintool supports running `gzip` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man gzip + + + Bintool: ifwitool: Handles the 'ifwitool' tool ---------------------------------------------- @@ -101,6 +129,20 @@ Documentation is available via:: +Bintool: lzop: Compression/decompression using the lzop algorithm +----------------------------------------------------------------- + +This bintool supports running `lzop` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man lzop + + + Bintool: mkimage: Image generation for U-Boot --------------------------------------------- @@ -113,3 +155,31 @@ Support is provided for fetching this on Debian-like systems, using apt. +Bintool: xz: Compression/decompression using the xz algorithm +------------------------------------------------------------- + +This bintool supports running `xz` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man xz + + + +Bintool: zstd: Compression/decompression using the zstd algorithm +----------------------------------------------------------------- + +This bintool supports running `zstd` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man zstd + + + diff --git a/tools/binman/btool/lz4.py b/tools/binman/btool/lz4.py index dc9e37921a..fd520d13a5 100644 --- a/tools/binman/btool/lz4.py +++ b/tools/binman/btool/lz4.py @@ -60,7 +60,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollz4(bintool.Bintool): diff --git a/tools/binman/btool/lzma_alone.py b/tools/binman/btool/lzma_alone.py index 52a960fd2f..1fda2f68c7 100644 --- a/tools/binman/btool/lzma_alone.py +++ b/tools/binman/btool/lzma_alone.py @@ -37,7 +37,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollzma_alone(bintool.Bintool): diff --git a/tools/binman/btool/openssl.py b/tools/binman/btool/openssl.py new file mode 100644 index 0000000000..3a4dbdd6d7 --- /dev/null +++ b/tools/binman/btool/openssl.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2022 Google LLC +# +"""Bintool implementation for openssl + +openssl provides a number of features useful for signing images + +Documentation is at https://www.coreboot.org/CBFS + +Source code is at https://www.openssl.org/ +""" + +import hashlib + +from binman import bintool +from u_boot_pylib import tools + +class Bintoolopenssl(bintool.Bintool): + """openssl tool + + This bintool supports creating new openssl certificates. + + It also supports fetching a binary openssl + + Documentation about openssl is at https://www.openssl.org/ + """ + def __init__(self, name): + super().__init__( + name, 'openssl cryptography toolkit', + version_regex=r'OpenSSL (.*) \(', version_args='version') + + def x509_cert(self, cert_fname, input_fname, key_fname, cn, revision, + config_fname): + """Create a certificate + + Args: + cert_fname (str): Filename of certificate to create + input_fname (str): Filename containing data to sign + key_fname (str): Filename of .pem file + cn (str): Common name + revision (int): Revision number + config_fname (str): Filename to write fconfig into + + Returns: + str: Tool output + """ + indata = tools.read_file(input_fname) + hashval = hashlib.sha512(indata).hexdigest() + with open(config_fname, 'w', encoding='utf-8') as outf: + print(f'''[ req ] +distinguished_name = req_distinguished_name +x509_extensions = v3_ca +prompt = no +dirstring_type = nobmp + +[ req_distinguished_name ] +CN = {cert_fname} + +[ v3_ca ] +basicConstraints = CA:true +1.3.6.1.4.1.294.1.3 = ASN1:SEQUENCE:swrv +1.3.6.1.4.1.294.1.34 = ASN1:SEQUENCE:sysfw_image_integrity + +[ swrv ] +swrv = INTEGER:{revision} + +[ sysfw_image_integrity ] +shaType = OID:2.16.840.1.101.3.4.2.3 +shaValue = FORMAT:HEX,OCT:{hashval} +imageSize = INTEGER:{len(indata)} +''', file=outf) + args = ['req', '-new', '-x509', '-key', key_fname, '-nodes', + '-outform', 'DER', '-out', cert_fname, '-config', config_fname, + '-sha512'] + return self.run_cmd(*args) + + def fetch(self, method): + """Fetch handler for openssl + + This installs the openssl package using the apt utility. + + Args: + method (FETCH_...): Method to use + + Returns: + True if the file was fetched and now installed, None if a method + other than FETCH_BIN was requested + + Raises: + Valuerror: Fetching could not be completed + """ + if method != bintool.FETCH_BIN: + return None + return self.apt_install('openssl') diff --git a/tools/binman/cbfs_util.py b/tools/binman/cbfs_util.py index 7bd3d89798..fc56b40b75 100644 --- a/tools/binman/cbfs_util.py +++ b/tools/binman/cbfs_util.py @@ -22,8 +22,8 @@ import sys from binman import bintool from binman import elf -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # Set to True to enable printing output while working DEBUG = False diff --git a/tools/binman/cbfs_util_test.py b/tools/binman/cbfs_util_test.py index e0f792fd34..ee951d10cf 100755 --- a/tools/binman/cbfs_util_test.py +++ b/tools/binman/cbfs_util_test.py @@ -20,8 +20,8 @@ from binman import bintool from binman import cbfs_util from binman.cbfs_util import CbfsWriter from binman import elf -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools U_BOOT_DATA = b'1234' U_BOOT_DTB_DATA = b'udtb' diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 986d6f1a31..1b7bbe80cd 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -7,7 +7,13 @@ import argparse from argparse import ArgumentParser +import os from binman import state +import os +import pathlib + +BINMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = (BINMAN_DIR / "ftest.py").exists() def make_extract_parser(subparsers): """make_extract_parser: Make a subparser for the 'extract' command @@ -67,6 +73,14 @@ def ParseArgs(argv): options provides access to the options (e.g. option.debug) args is a list of string arguments """ + def _AddPreserve(pars): + pars.add_argument('-O', '--outdir', type=str, + action='store', help='Path to directory to use for intermediate ' + 'and output files') + pars.add_argument('-p', '--preserve', action='store_true',\ + help='Preserve temporary output directory even if option -O is not ' + 'given') + if '-H' in argv: argv.append('build') @@ -80,8 +94,11 @@ controlled by a description in the board device tree.''' help='Enabling debugging (provides a full traceback on error)') parser.add_argument('-H', '--full-help', action='store_true', default=False, help='Display the README file') + parser.add_argument('--tooldir', type=str, + default=os.path.join(os.getenv('HOME'), '.binman-tools'), + help='Set the directory to store tools') parser.add_argument('--toolpath', type=str, action='append', - help='Add a path to the directories containing tools') + help='Add a path to the list of directories containing tools') parser.add_argument('-T', '--threads', type=int, default=None, help='Number of threads to use (0=single-thread)') parser.add_argument('--test-section-timeout', action='store_true', @@ -118,12 +135,7 @@ controlled by a description in the board device tree.''' build_parser.add_argument('-n', '--no-expanded', action='store_true', help="Don't use 'expanded' versions of entries where available; " "normally 'u-boot' becomes 'u-boot-expanded', for example") - build_parser.add_argument('-O', '--outdir', type=str, - action='store', help='Path to directory to use for intermediate and ' - 'output files') - build_parser.add_argument('-p', '--preserve', action='store_true',\ - help='Preserve temporary output directory even if option -O is not ' - 'given') + _AddPreserve(build_parser) build_parser.add_argument('-u', '--update-fdt', action='store_true', default=False, help='Update the binman node with offset/size info') build_parser.add_argument('--update-fdt-in-elf', type=str, @@ -160,26 +172,30 @@ controlled by a description in the board device tree.''' help='Path to directory to use for input files') replace_parser.add_argument('-m', '--map', action='store_true', default=False, help='Output a map file for the updated image') + _AddPreserve(replace_parser) replace_parser.add_argument('paths', type=str, nargs='*', help='Paths within file to replace (wildcard)') - test_parser = subparsers.add_parser('test', help='Run tests') - test_parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') - test_parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') - test_parser.add_argument('-X', '--test-preserve-dirs', action='store_true', - help='Preserve and display test-created input directories; also ' - 'preserve the output directory if a single test is run (pass test ' - 'name at the end of the command line') - test_parser.add_argument('tests', nargs='*', - help='Test names to run (omit for all)') + if HAS_TESTS: + test_parser = subparsers.add_parser('test', help='Run tests') + test_parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + test_parser.add_argument('-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + test_parser.add_argument( + '-X', '--test-preserve-dirs', action='store_true', + help='Preserve and display test-created input directories; also ' + 'preserve the output directory if a single test is run (pass ' + 'test name at the end of the command line') + test_parser.add_argument('tests', nargs='*', + help='Test names to run (omit for all)') tool_parser = subparsers.add_parser('tool', help='Check bintools') tool_parser.add_argument('-l', '--list', action='store_true', help='List all known bintools') - tool_parser.add_argument('-f', '--fetch', action='store_true', - help='fetch a bintool from a known location (or: all/missing)') + tool_parser.add_argument( + '-f', '--fetch', action='store_true', + help='fetch a bintool from a known location (or: all/missing)') tool_parser.add_argument('bintools', type=str, nargs='*') return parser.parse_args(argv) diff --git a/tools/binman/control.py b/tools/binman/control.py index e64740094f..2f2b4893b7 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -7,19 +7,20 @@ from collections import OrderedDict import glob +import importlib.resources import os import pkg_resources import re import sys -from patman import tools from binman import bintool from binman import cbfs_util -from patman import command from binman import elf from binman import entry -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout # These are imported if needed since they import libfdt state = None @@ -402,6 +403,8 @@ def ReplaceEntries(image_fname, input_fname, indir, entry_paths, image_fname = os.path.abspath(image_fname) image = Image.FromFile(image_fname) + image.mark_build_done() + # Replace an entry from a single file, as a special case if input_fname: if not entry_paths: @@ -641,19 +644,29 @@ def Binman(args): global state if args.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README.rst') - ) + with importlib.resources.path('binman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 # Put these here so that we can import this module without libfdt from binman.image import Image from binman import state + tool_paths = [] + if args.toolpath: + tool_paths += args.toolpath + if args.tooldir: + tool_paths.append(args.tooldir) + tools.set_tool_paths(tool_paths or None) + bintool.Bintool.set_tool_dir(args.tooldir) + if args.cmd in ['ls', 'extract', 'replace', 'tool']: try: tout.init(args.verbosity) - tools.prepare_output_dir(None) + if args.cmd == 'replace': + tools.prepare_output_dir(args.outdir, args.preserve) + else: + tools.prepare_output_dir(None) if args.cmd == 'ls': ListEntries(args.image, args.paths) @@ -667,7 +680,6 @@ def Binman(args): allow_resize=not args.fix_size, write_map=args.map) if args.cmd == 'tool': - tools.set_tool_paths(args.toolpath) if args.list: bintool.Bintool.list_all() elif args.fetch: @@ -719,7 +731,6 @@ def Binman(args): try: tools.set_input_dirs(args.indir) tools.prepare_output_dir(args.outdir, args.preserve) - tools.set_tool_paths(args.toolpath) state.SetEntryArgs(args.entry_arg) state.SetThreads(args.threads) diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 3cc8a38449..5816284c32 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -13,9 +13,9 @@ import shutil import struct import tempfile -from patman import command -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout ELF_TOOLS = True try: diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 8cb55ebb81..c98083961b 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -12,10 +12,10 @@ import tempfile import unittest from binman import elf -from patman import command -from patman import test_util -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import test_util +from u_boot_pylib import tools +from u_boot_pylib import tout binman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index e177860a6a..b71af801fd 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -887,6 +887,11 @@ before its contents, so that it is possible to reconstruct the hierarchy from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. +To mark an area as preserved, use the normal 'preserved' flag in the entry. +This will result in the corresponding FMAP area having the +FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to +child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. @@ -2285,6 +2290,24 @@ and kernel are genuine. +.. _etype_x509_cert: + +Entry: x509-cert: An entry which contains an X509 certificate +------------------------------------------------------------- + +Properties / Entry arguments: + - content: List of phandles to entries to sign + +Output files: + - input.<unique_name> - input file passed to openssl + - cert.<unique_name> - output file generated by openssl (which is + used as the entry contents) + +openssl signs the provided data, writing the signature in this entry. This +allows verification that the data is genuine + + + .. _etype_x86_reset16: Entry: x86-reset16: x86 16-bit reset code for U-Boot diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 5eacc5fa6c..b10a43333e 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -14,9 +14,9 @@ import time from binman import bintool from binman import elf from dtoc import fdt_util -from patman import tools -from patman.tools import to_hex, to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex, to_hex_size +from u_boot_pylib import tout modules = {} @@ -100,6 +100,14 @@ class Entry(object): appear in the map optional (bool): True if this entry contains an optional external blob overlap (bool): True if this entry overlaps with others + preserve (bool): True if this entry should be preserved when updating + firmware. This means that it will not be changed by the update. + This is just a signal: enforcement of this is up to the updater. + This flag does not automatically propagate down to child entries. + build_done (bool): Indicates that the entry data has been built and does + not need to be done again. This is only used with 'binman replace', + to stop sections from being rebuilt if their entries have not been + replaced """ fake_dir = None @@ -148,6 +156,8 @@ class Entry(object): self.overlap = False self.elf_base_sym = None self.offset_from_elf = None + self.preserve = False + self.build_done = False @staticmethod def FindEntryClass(etype, expanded): @@ -310,6 +320,8 @@ class Entry(object): self.offset_from_elf = fdt_util.GetPhandleNameOffset(self._node, 'offset-from-elf') + self.preserve = fdt_util.GetBool(self._node, 'preserve') + def GetDefaultFilename(self): return None @@ -1006,6 +1018,7 @@ features to produce new behaviours. else: self.contents_size = self.pre_reset_size ok = self.ProcessContentsUpdate(data) + self.build_done = False self.Detail('WriteData: size=%x, ok=%s' % (len(data), ok)) section_ok = self.section.WriteChildData(self) return ok and section_ok @@ -1027,6 +1040,14 @@ features to produce new behaviours. True if the section could be updated successfully, False if the data is such that the section could not update """ + self.build_done = False + entry = self.section + + # Now we must rebuild all sections above this one + while entry and entry != entry.section: + self.build_done = False + entry = entry.section + return True def GetSiblingOrder(self): @@ -1104,7 +1125,7 @@ features to produce new behaviours. If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ # This is meaningless for anything other than blobs pass @@ -1349,3 +1370,11 @@ features to produce new behaviours. val = elf.GetSymbolOffset(entry.elf_fname, sym_name, entry.elf_base_sym) return val + offset + + def mark_build_done(self): + """Mark an entry as already built""" + self.build_done = True + entries = self.GetEntries() + if entries: + for entry in entries.values(): + entry.mark_build_done() diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index a6fbf62731..ac6582cf86 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -14,7 +14,7 @@ from binman import entry from binman.etype.blob import Entry_blob from dtoc import fdt from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class TestEntry(unittest.TestCase): def setUp(self): diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 1c1efb21a4..e092d98ce1 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry__testing(Entry): diff --git a/tools/binman/etype/atf_fip.py b/tools/binman/etype/atf_fip.py index 6ecd95b71f..73a3f85b9f 100644 --- a/tools/binman/etype/atf_fip.py +++ b/tools/binman/etype/atf_fip.py @@ -11,7 +11,7 @@ from binman.entry import Entry from binman.etype.section import Entry_section from binman.fip_util import FIP_TYPES, FipReader, FipWriter, UUID_LEN from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_atf_fip(Entry_section): """ARM Trusted Firmware's Firmware Image Package (FIP) @@ -270,4 +270,4 @@ class Entry_atf_fip(Entry_section): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index c7ddcedffb..064fae5036 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -8,8 +8,8 @@ from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob(Entry): """Arbitrary binary blob @@ -102,7 +102,7 @@ class Entry_blob(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ if self.faked: faked_blobs_list.append(self) diff --git a/tools/binman/etype/blob_ext.py b/tools/binman/etype/blob_ext.py index fba6271de2..ca26530738 100644 --- a/tools/binman/etype/blob_ext.py +++ b/tools/binman/etype/blob_ext.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext(Entry_blob): """Externally built binary blob @@ -26,11 +26,3 @@ class Entry_blob_ext(Entry_blob): def __init__(self, section, etype, node): Entry_blob.__init__(self, section, etype, node) self.external = True - - def SetAllowFakeBlob(self, allow_fake): - """Set whether the entry allows to create a fake blob - - Args: - allow_fake_blob: True if allowed, False if not allowed - """ - self.allow_fake = allow_fake diff --git a/tools/binman/etype/blob_ext_list.py b/tools/binman/etype/blob_ext_list.py index f00202e9eb..1bfcf6733a 100644 --- a/tools/binman/etype/blob_ext_list.py +++ b/tools/binman/etype/blob_ext_list.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext_list(Entry_blob): """List of externally built binary blobs diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index 832f8d038f..575aa624f6 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -295,7 +295,7 @@ class Entry_cbfs(Entry): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) def AddBintools(self, btools): super().AddBintools(btools) diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index 33c9d039a9..f1f6217940 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -9,8 +9,8 @@ image. """ from binman.entry import Entry -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout FDTMAP_MAGIC = b'_FDTMAP_' FDTMAP_HDR_LEN = 16 diff --git a/tools/binman/etype/files.py b/tools/binman/etype/files.py index 2081bc727b..c8757eafab 100644 --- a/tools/binman/etype/files.py +++ b/tools/binman/etype/files.py @@ -11,7 +11,7 @@ import os from binman.etype.section import Entry_section from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index c91d0152a8..7c93d4e268 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index cd2943533c..03fe88e7a6 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -12,7 +12,7 @@ from binman.etype.section import Entry_section from binman import elf from dtoc import fdt_util from dtoc.fdt import Fdt -from patman import tools +from u_boot_pylib import tools # Supported operations, with the fit,operation property OP_GEN_FDT_NODES, OP_SPLIT_ELF = range(2) @@ -453,6 +453,8 @@ class Entry_fit(Entry_section): args.update({'align': fdt_util.fdt32_to_cpu(align.value)}) if self.mkimage.run(reset_timestamp=True, output_fname=output_fname, **args) is None: + if not self.GetAllowMissing(): + self.Raise("Missing tool: 'mkimage'") # Bintool is missing; just use empty data as the output self.record_missing_bintool(self.mkimage) return tools.get_bytes(0, 1024) @@ -775,6 +777,8 @@ class Entry_fit(Entry_section): Args: image_pos (int): Position of this entry in the image """ + if self.build_done: + return super().SetImagePos(image_pos) # If mkimage is missing we'll have empty data, @@ -823,8 +827,11 @@ class Entry_fit(Entry_section): self.mkimage = self.AddBintool(btools, 'mkimage') def CheckMissing(self, missing_list): - # We must use our private entry list for this since generator notes + # We must use our private entry list for this since generator nodes # which are removed from self._entries will otherwise not show up as # missing for entry in self._priv_entries.values(): entry.CheckMissing(missing_list) + + def CheckEntries(self): + pass diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index 0c576202a4..3669d91a0b 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,9 +7,9 @@ from binman.entry import Entry from binman import fmap_util -from patman import tools -from patman.tools import to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex_size +from u_boot_pylib import tout class Entry_fmap(Entry): @@ -33,6 +33,11 @@ class Entry_fmap(Entry): from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. + To mark an area as preserved, use the normal 'preserved' flag in the entry. + This will result in the corresponding FMAP area having the + FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to + child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. """ def __init__(self, section, etype, node): @@ -48,6 +53,12 @@ class Entry_fmap(Entry): entries = entry.GetEntries() tout.debug("fmap: Add entry '%s' type '%s' (%s subentries)" % (entry.GetPath(), entry.etype, to_hex_size(entries))) + + # Collect any flag (separate lines to ensure code coverage) + flags = 0 + if entry.preserve: + flags = fmap_util.FMAP_AREA_PRESERVE + if entries and entry.etype != 'cbfs': # Create an area for the section, which encompasses all entries # within it @@ -59,7 +70,7 @@ class Entry_fmap(Entry): # Drop @ symbols in name name = entry.name.replace('@', '') areas.append( - fmap_util.FmapArea(pos, entry.size or 0, name, 0)) + fmap_util.FmapArea(pos, entry.size or 0, name, flags)) for subentry in entries.values(): _AddEntries(areas, subentry) else: @@ -67,7 +78,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + entry.name, flags)) entries = self.GetImage().GetEntries() areas = [] diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index ba2a362bb5..cca18af6e2 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -8,11 +8,11 @@ from collections import OrderedDict -from patman import command +from u_boot_pylib import command from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # Build GBB flags. # (src/platform/vboot_reference/firmware/include/gbb_header.h) diff --git a/tools/binman/etype/intel_ifwi.py b/tools/binman/etype/intel_ifwi.py index 04fad401ee..6513b97c3e 100644 --- a/tools/binman/etype/intel_ifwi.py +++ b/tools/binman/etype/intel_ifwi.py @@ -10,7 +10,7 @@ from collections import OrderedDict from binman.entry import Entry from binman.etype.blob_ext import Entry_blob_ext from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_intel_ifwi(Entry_blob_ext): """Intel Integrated Firmware Image (IFWI) file diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index 8a13d5ea8d..e028c44070 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_mkimage(Entry): """Binary produced by mkimage diff --git a/tools/binman/etype/null.py b/tools/binman/etype/null.py index c10d482447..263fb5244d 100644 --- a/tools/binman/etype/null.py +++ b/tools/binman/etype/null.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_null(Entry): """An entry which has no contents of its own diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py index b622281159..bd3545bffc 100644 --- a/tools/binman/etype/pre_load.py +++ b/tools/binman/etype/pre_load.py @@ -8,7 +8,7 @@ import os import struct from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools from binman.entry import Entry from binman.etype.collection import Entry_collection diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 57b91ff726..c36edd1350 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -16,9 +16,9 @@ import sys from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout -from patman.tools import to_hex_size +from u_boot_pylib import tools +from u_boot_pylib import tout +from u_boot_pylib.tools import to_hex_size class Entry_section(Entry): @@ -172,7 +172,7 @@ class Entry_section(Entry): def IsSpecialSubnode(self, node): """Check if a node is a special one used by the section itself - Some notes are used for hashing / signatures and do not add entries to + Some nodes are used for hashing / signatures and do not add entries to the actual section. Returns: @@ -397,10 +397,13 @@ class Entry_section(Entry): This excludes any padding. If the section is compressed, the compressed data is returned """ - data = self.BuildSectionData(required) - if data is None: - return None - self.SetContents(data) + if not self.build_done: + data = self.BuildSectionData(required) + if data is None: + return None + self.SetContents(data) + else: + data = self.data if self._filename: tools.write_file(tools.get_output_filename(self._filename), data) return data @@ -427,8 +430,11 @@ class Entry_section(Entry): self._SortEntries() self._extend_entries() - data = self.BuildSectionData(True) - self.SetContents(data) + if self.build_done: + self.size = None + else: + data = self.BuildSectionData(True) + self.SetContents(data) self.CheckSize() @@ -810,6 +816,9 @@ class Entry_section(Entry): def LoadData(self, decomp=True): for entry in self._entries.values(): entry.LoadData(decomp) + data = self.ReadData(decomp) + self.contents_size = len(data) + self.ProcessContentsUpdate(data) self.Detail('Loaded data') def GetImage(self): @@ -866,10 +875,15 @@ class Entry_section(Entry): return data def WriteData(self, data, decomp=True): - self.Raise("Replacing sections is not implemented yet") + ok = super().WriteData(data, decomp) + + # The section contents are now fixed and cannot be rebuilt from the + # containing entries. + self.mark_build_done() + return ok def WriteChildData(self, child): - return True + return super().WriteChildData(child) def SetAllowMissing(self, allow_missing): """Set whether a section allows missing external blobs @@ -885,7 +899,7 @@ class Entry_section(Entry): """Set whether a section allows to create a fake blob Args: - allow_fake_blob: True if allowed, False if not allowed + allow_fake: True if allowed, False if not allowed """ super().SetAllowFakeBlob(allow_fake) for entry in self._entries.values(): @@ -909,7 +923,7 @@ class Entry_section(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ for entry in self._entries.values(): entry.CheckFakedBlobs(faked_blobs_list) diff --git a/tools/binman/etype/text.py b/tools/binman/etype/text.py index c55e0233b1..e4deb4abac 100644 --- a/tools/binman/etype/text.py +++ b/tools/binman/etype/text.py @@ -7,7 +7,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_text(Entry): diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 047d310cdf..f7225cecc1 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob_dtb import Entry_blob_dtb -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/u_boot_elf.py b/tools/binman/etype/u_boot_elf.py index 3ec774f38a..f4d86aa176 100644 --- a/tools/binman/etype/u_boot_elf.py +++ b/tools/binman/etype/u_boot_elf.py @@ -9,7 +9,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_elf(Entry_blob): """U-Boot ELF image diff --git a/tools/binman/etype/u_boot_env.py b/tools/binman/etype/u_boot_env.py index c38340b256..c027e93d42 100644 --- a/tools/binman/etype/u_boot_env.py +++ b/tools/binman/etype/u_boot_env.py @@ -8,7 +8,7 @@ import zlib from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_env(Entry_blob): """An entry which contains a U-Boot environment diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 680d198305..1ffeb3911f 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_spl_bss_pad(Entry_blob): """U-Boot SPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_spl_expanded.py b/tools/binman/etype/u_boot_spl_expanded.py index 319f6708fe..fcd0dd19ac 100644 --- a/tools/binman/etype/u_boot_spl_expanded.py +++ b/tools/binman/etype/u_boot_spl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot SPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_bss_pad.py b/tools/binman/etype/u_boot_tpl_bss_pad.py index 47f4b23f35..29c6a95412 100644 --- a/tools/binman/etype/u_boot_tpl_bss_pad.py +++ b/tools/binman/etype/u_boot_tpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_tpl_bss_pad(Entry_blob): """U-Boot TPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_tpl_expanded.py b/tools/binman/etype/u_boot_tpl_expanded.py index 55fde3c8e6..58db4f3755 100644 --- a/tools/binman/etype/u_boot_tpl_expanded.py +++ b/tools/binman/etype/u_boot_tpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot TPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py index c7f3f9dedb..86f9578b71 100644 --- a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py @@ -7,11 +7,11 @@ import struct -from patman import command from binman.entry import Entry from binman.etype.blob import Entry_blob from binman.etype.u_boot_with_ucode_ptr import Entry_u_boot_with_ucode_ptr -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools class Entry_u_boot_tpl_with_ucode_ptr(Entry_u_boot_with_ucode_ptr): """U-Boot TPL with embedded microcode pointer diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index 6945411cf9..97ed7d7eb1 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_ucode(Entry_blob): """U-Boot microcode block diff --git a/tools/binman/etype/u_boot_vpl_bss_pad.py b/tools/binman/etype/u_boot_vpl_bss_pad.py index b2ce2a3135..bba38ccf9e 100644 --- a/tools/binman/etype/u_boot_vpl_bss_pad.py +++ b/tools/binman/etype/u_boot_vpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_vpl_bss_pad(Entry_blob): """U-Boot VPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_vpl_expanded.py b/tools/binman/etype/u_boot_vpl_expanded.py index 92c64f0a65..deff5a3f8c 100644 --- a/tools/binman/etype/u_boot_vpl_expanded.py +++ b/tools/binman/etype/u_boot_vpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot VPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_with_ucode_ptr.py b/tools/binman/etype/u_boot_with_ucode_ptr.py index e275698ceb..41731fd0e1 100644 --- a/tools/binman/etype/u_boot_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_with_ucode_ptr.py @@ -11,8 +11,8 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import command +from u_boot_pylib import tools +from u_boot_pylib import command class Entry_u_boot_with_ucode_ptr(Entry_blob): """U-Boot with embedded microcode pointer diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 04cb7228aa..4adb9a4e9b 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -13,7 +13,7 @@ from binman.entry import EntryArg from binman.etype.collection import Entry_collection from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_vblock(Entry_collection): """An entry which contains a Chromium OS verified boot block diff --git a/tools/binman/etype/x509_cert.py b/tools/binman/etype/x509_cert.py new file mode 100644 index 0000000000..f80a6ec2d1 --- /dev/null +++ b/tools/binman/etype/x509_cert.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2023 Google LLC +# Written by Simon Glass <sjg@chromium.org> +# + +# Support for an X509 certificate, used to sign a set of entries + +from collections import OrderedDict +import os + +from binman.entry import EntryArg +from binman.etype.collection import Entry_collection + +from dtoc import fdt_util +from u_boot_pylib import tools + +class Entry_x509_cert(Entry_collection): + """An entry which contains an X509 certificate + + Properties / Entry arguments: + - content: List of phandles to entries to sign + + Output files: + - input.<unique_name> - input file passed to openssl + - cert.<unique_name> - output file generated by openssl (which is + used as the entry contents) + + openssl signs the provided data, writing the signature in this entry. This + allows verification that the data is genuine + """ + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.openssl = None + + def ReadNode(self): + super().ReadNode() + self._cert_ca = fdt_util.GetString(self._node, 'cert-ca') + self._cert_rev = fdt_util.GetInt(self._node, 'cert-revision-int', 0) + self.key_fname = self.GetEntryArgsOrProps([ + EntryArg('keyfile', str)], required=True)[0] + + def GetCertificate(self, required): + """Get the contents of this entry + + Args: + required: True if the data must be present, False if it is OK to + return None + + Returns: + bytes content of the entry, which is the signed vblock for the + provided data + """ + # Join up the data files to be signed + input_data = self.GetContents(required) + if input_data is None: + return None + + uniq = self.GetUniqueName() + output_fname = tools.get_output_filename('cert.%s' % uniq) + input_fname = tools.get_output_filename('input.%s' % uniq) + config_fname = tools.get_output_filename('config.%s' % uniq) + tools.write_file(input_fname, input_data) + stdout = self.openssl.x509_cert( + cert_fname=output_fname, + input_fname=input_fname, + key_fname=self.key_fname, + cn=self._cert_ca, + revision=self._cert_rev, + config_fname=config_fname) + if stdout is not None: + data = tools.read_file(output_fname) + else: + # Bintool is missing; just use 4KB of zero data + self.record_missing_bintool(self.openssl) + data = tools.get_bytes(0, 4096) + return data + + def ObtainContents(self): + data = self.GetCertificate(False) + if data is None: + return False + self.SetContents(data) + return True + + def ProcessContents(self): + # The blob may have changed due to WriteSymbols() + data = self.GetCertificate(True) + return self.ProcessContentsUpdate(data) + + def AddBintools(self, btools): + super().AddBintools(btools) + self.openssl = self.AddBintool(btools, 'openssl') diff --git a/tools/binman/fdt_test.py b/tools/binman/fdt_test.py index 94347b1a1e..7ef8729546 100644 --- a/tools/binman/fdt_test.py +++ b/tools/binman/fdt_test.py @@ -12,7 +12,7 @@ import unittest from dtoc import fdt from dtoc import fdt_util from dtoc.fdt import FdtScan -from patman import tools +from u_boot_pylib import tools class TestFdt(unittest.TestCase): @classmethod diff --git a/tools/binman/fip_util.py b/tools/binman/fip_util.py index 95eee32bc0..b5caab2d37 100755 --- a/tools/binman/fip_util.py +++ b/tools/binman/fip_util.py @@ -37,8 +37,8 @@ OUR_PATH = os.path.dirname(OUR_FILE) sys.path.insert(2, os.path.join(OUR_PATH, '..')) # pylint: disable=C0413 -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # The TOC header, at the start of the FIP HEADER_FORMAT = '<IIQ' diff --git a/tools/binman/fip_util_test.py b/tools/binman/fip_util_test.py index cf6d0002ec..56aa56f464 100755 --- a/tools/binman/fip_util_test.py +++ b/tools/binman/fip_util_test.py @@ -20,10 +20,10 @@ OUR_PATH = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(2, os.path.join(OUR_PATH, '..')) # pylint: disable=C0413 -from patman import test_util -from patman import tools from binman import bintool from binman import fip_util +from u_boot_pylib import test_util +from u_boot_pylib import tools FIPTOOL = bintool.Bintool.create('fiptool') HAVE_FIPTOOL = FIPTOOL.is_present() diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index 1ce63d1a83..40f2dbfe0f 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -10,7 +10,7 @@ import collections import struct import sys -from patman import tools +from u_boot_pylib import tools # constants imported from lib/fmap.h FMAP_SIGNATURE = b'__FMAP__' @@ -45,6 +45,9 @@ FMAP_AREA_NAMES = ( 'flags', ) +# Flags supported by areas (bits 2:0 are unused so not included here) +FMAP_AREA_PRESERVE = 1 << 3 # Preserved by any firmware updates + # These are the two data structures supported by flashrom, a header (which # appears once at the start) and an area (which is repeated until the end of # the list of areas) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index d74aa90a62..d455ea02ea 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -34,10 +34,10 @@ from dtoc import fdt_util from binman.etype import fdtmap from binman.etype import image_header from binman.image import Image -from patman import command -from patman import test_util -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import test_util +from u_boot_pylib import tools +from u_boot_pylib import tout # Contents of test files, corresponding to different entry types U_BOOT_DATA = b'1234' @@ -1704,7 +1704,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(b'SECTION0', fentry.name) self.assertEqual(0, fentry.offset) self.assertEqual(16, fentry.size) - self.assertEqual(0, fentry.flags) + self.assertEqual(fmap_util.FMAP_AREA_PRESERVE, fentry.flags) fentry = next(fiter) self.assertEqual(b'RO_U_BOOT', fentry.name) @@ -1752,7 +1752,7 @@ class TestFunctional(unittest.TestCase): def _HandleGbbCommand(self, pipe_list): """Fake calls to the futility utility""" - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][-1] # Append our GBB data to the file, which will happen every time the # futility command is called. @@ -1814,7 +1814,7 @@ class TestFunctional(unittest.TestCase): self._hash_data is False, it writes VBLOCK_DATA, else it writes a hash of the input data (here, 'input.vblock'). """ - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][3] with open(fname, 'wb') as fd: if self._hash_data: @@ -4001,9 +4001,17 @@ class TestFunctional(unittest.TestCase): self.assertEqual(expected, data[image_pos:image_pos+size]) def testFitMissing(self): + """Test that binman complains if mkimage is missing""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('162_fit_external.dts', + force_missing_bintools='mkimage') + self.assertIn("Node '/binman/fit': Missing tool: 'mkimage'", + str(e.exception)) + + def testFitMissingOK(self): """Test that binman still produces a FIT image if mkimage is missing""" with test_util.capture_sys_output() as (_, stderr): - self._DoTestFile('162_fit_external.dts', + self._DoTestFile('162_fit_external.dts', allow_missing=True, force_missing_bintools='mkimage') err = stderr.getvalue() self.assertRegex(err, "Image 'image'.*missing bintools.*: mkimage") @@ -5813,13 +5821,61 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(expected_fdtmap, fdtmap) def testReplaceSectionSimple(self): - """Test replacing a simple section with arbitrary data""" + """Test replacing a simple section with same-sized data""" new_data = b'w' * len(COMPRESS_DATA + U_BOOT_DATA) - with self.assertRaises(ValueError) as exc: - self._RunReplaceCmd('section', new_data, - dts='241_replace_section_simple.dts') + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionLarger(self): + """Test replacing a simple section with larger data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + fentry = entries['fdtmap'] + self.assertEqual(entry.offset + entry.size, fentry.offset) + + def testReplaceSectionSmaller(self): + """Test replacing a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + b'\0' + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + # The new size is the same as the old, just with a pad byte at the end + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionSmallerAllow(self): + """Test failing to replace a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + try: + state.SetAllowEntryContraction(True) + with self.assertRaises(ValueError) as exc: + self._RunReplaceCmd('section', new_data, + dts='241_replace_section_simple.dts') + finally: + state.SetAllowEntryContraction(False) + + # Since we have no information about the position of things within the + # section, we cannot adjust the position of /section-u-boot so it ends + # up outside the section self.assertIn( - "Node '/section': Replacing sections is not implemented yet", + "Node '/section/u-boot': Offset 0x24 (36) size 0x4 (4) is outside " + "the section '/section' starting at 0x0 (0) of size 0x27 (39)", str(exc.exception)) def testMkimageImagename(self): @@ -6355,10 +6411,11 @@ fdt fdtmap Extract the devicetree blob from the fdtmap 'tee-os-path': 'missing.bin', } test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) - data = self._DoReadFileDtb( - '276_fit_firmware_loadables.dts', - entry_args=entry_args, - extra_indirs=[test_subdir])[0] + with test_util.capture_sys_output() as (stdout, stderr): + data = self._DoReadFileDtb( + '276_fit_firmware_loadables.dts', + entry_args=entry_args, + extra_indirs=[test_subdir])[0] dtb = fdt.Fdt.FromData(data) dtb.Scan() @@ -6388,6 +6445,128 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(['u-boot', 'atf-2'], fdt_util.GetStringList(node, 'loadables')) + def testTooldir(self): + """Test that we can specify the tooldir""" + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman('--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual('fred', bintool.Bintool.tooldir) + + # Check that the toolpath is updated correctly + self.assertEqual(['fred'], tools.tool_search_paths) + + # Try with a few toolpaths; the tooldir should be at the end + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman( + '--toolpath', 'mary', '--toolpath', 'anna', '--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual(['mary', 'anna', 'fred'], tools.tool_search_paths) + + def testReplaceSectionEntry(self): + """Test replacing an entry in a section""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd('section/blob', + expect_data, dts='241_replace_section_simple.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceSectionDeep(self): + """Test replacing an entry in two levels of sections""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd( + 'section/section/blob', expect_data, + dts='278_replace_section_deep.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + subentries = section.GetEntries() + self.assertIn('section', subentries) + section = subentries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceFitSibling(self): + """Test an image with a FIT inside where we replace its sibling""" + fname = TestFunctional._MakeInputFile('once', b'available once') + self._DoReadFileRealDtb('277_replace_fit_sibling.dts') + os.remove(fname) + + try: + tmpdir, updated_fname = self._SetupImageInTmpdir() + + fname = os.path.join(tmpdir, 'update-blob') + expected = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + tools.write_file(fname, expected) + + self._DoBinman('replace', '-i', updated_fname, 'blob', '-f', fname) + data = tools.read_file(updated_fname) + start = len(U_BOOT_DTB_DATA) + self.assertEqual(expected, data[start:start + len(expected)]) + map_fname = os.path.join(tmpdir, 'image-updated.map') + self.assertFalse(os.path.exists(map_fname)) + finally: + shutil.rmtree(tmpdir) + + def testX509Cert(self): + """Test creating an X509 certificate""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': keyfile, + } + data = self._DoReadFileDtb('279_x509_cert.dts', + entry_args=entry_args)[0] + cert = data[:-4] + self.assertEqual(U_BOOT_DATA, data[-4:]) + + # TODO: verify the signature + + def testX509CertMissing(self): + """Test that binman still produces an image if openssl is missing""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': 'keyfile', + } + with test_util.capture_sys_output() as (_, stderr): + self._DoTestFile('279_x509_cert.dts', + force_missing_bintools='openssl', + entry_args=entry_args) + err = stderr.getvalue() + self.assertRegex(err, "Image 'image'.*missing bintools.*: openssl") + def testPackRockchipTpl(self): """Test that an image with a Rockchip TPL binary can be created""" data = self._DoReadFile('277_rockchip_tpl.dts') diff --git a/tools/binman/image.py b/tools/binman/image.py index 941596320c..8ebf71d61a 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -18,8 +18,8 @@ from binman.etype import image_header from binman.etype import section from dtoc import fdt from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Image(section.Entry_section): """A Image, representing an output from binman diff --git a/tools/binman/image_test.py b/tools/binman/image_test.py index e351fa84ab..bd51c1e55d 100644 --- a/tools/binman/image_test.py +++ b/tools/binman/image_test.py @@ -7,7 +7,7 @@ import unittest from binman.image import Image -from patman.test_util import capture_sys_output +from u_boot_pylib.test_util import capture_sys_output class TestImage(unittest.TestCase): def testInvalidFormat(self): diff --git a/tools/binman/main.py b/tools/binman/main.py index 14432a8d0d..92d2431aea 100755 --- a/tools/binman/main.py +++ b/tools/binman/main.py @@ -34,7 +34,7 @@ sys.pycache_prefix = os.path.relpath(our_path, srctree) sys.path.insert(2, our1_path) from binman import bintool -from patman import test_util +from u_boot_pylib import test_util # Bring in the libfdt module sys.path.insert(2, 'scripts/dtc/pylibfdt') @@ -44,7 +44,7 @@ sys.path.insert(2, os.path.join(srctree, 'build-sandbox_spl/scripts/dtc/pylibfdt from binman import cmdline from binman import control -from patman import test_util +from u_boot_pylib import test_util def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath): """Run the functional tests and any embedded doctests @@ -85,7 +85,7 @@ def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(toolpath): +def RunTestCoverage(toolpath, build_dir): """Run the tests and check that we get 100% coverage""" glob_list = control.GetEntryModules(False) all_set = set([os.path.splitext(os.path.basename(item))[0] @@ -95,8 +95,9 @@ def RunTestCoverage(toolpath): for path in toolpath: extra_args += ' --toolpath %s' % path test_util.run_test_coverage('tools/binman/binman', None, - ['*test*', '*main.py', 'tools/patman/*', 'tools/dtoc/*'], - args.build_dir, all_set, extra_args or None) + ['*test*', '*main.py', 'tools/patman/*', 'tools/dtoc/*', + 'tools/u_boot_pylib/*'], + build_dir, all_set, extra_args or None) def RunBinman(args): """Main entry point to binman once arguments are parsed @@ -116,7 +117,7 @@ def RunBinman(args): if args.cmd == 'test': if args.test_coverage: - RunTestCoverage(args.toolpath) + RunTestCoverage(args.toolpath, args.build_dir) else: ret_code = RunTests(args.debug, args.verbosity, args.processes, args.test_preserve_dirs, args.tests, @@ -140,8 +141,12 @@ def RunBinman(args): return ret_code -if __name__ == "__main__": +def start_binman(): args = cmdline.ParseArgs(sys.argv[1:]) ret_code = RunBinman(args) sys.exit(ret_code) + + +if __name__ == "__main__": + start_binman() diff --git a/tools/binman/pyproject.toml b/tools/binman/pyproject.toml new file mode 100644 index 0000000000..b4b54fbaee --- /dev/null +++ b/tools/binman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "binary-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib", "dtoc"] +description = "Binman firmware-packaging tool" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/package/index.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +binman = "binman.main:start_binman" + +[tool.setuptools.package-data] +patman = ["*.rst"] diff --git a/tools/binman/state.py b/tools/binman/state.py index 56e5bf8bc1..3e78cf3430 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -13,8 +13,8 @@ import threading from dtoc import fdt import os -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout OUR_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -306,7 +306,7 @@ def GetUpdateNodes(node, for_repack=False): """Yield all the nodes that need to be updated in all device trees The property referenced by this node is added to any device trees which - have the given node. Due to removable of unwanted notes, SPL and TPL may + have the given node. Due to removable of unwanted nodes, SPL and TPL may not have this node. Args: diff --git a/tools/binman/test/067_fmap.dts b/tools/binman/test/067_fmap.dts index 9c0e293ac8..24fa6351ec 100644 --- a/tools/binman/test/067_fmap.dts +++ b/tools/binman/test/067_fmap.dts @@ -11,6 +11,7 @@ name-prefix = "ro-"; size = <0x10>; pad-byte = <0x21>; + preserve; u-boot { }; diff --git a/tools/binman/test/277_replace_fit_sibling.dts b/tools/binman/test/277_replace_fit_sibling.dts new file mode 100644 index 0000000000..fc941a8081 --- /dev/null +++ b/tools/binman/test/277_replace_fit_sibling.dts @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot { + }; + + blob { + filename = "compress"; + }; + + fit { + description = "test-desc"; + #address-cells = <1>; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + blob-ext { + filename = "once"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + type = "flat_dt"; + arch = "ppc"; + compression = "none"; + hash-1 { + algo = "crc32"; + }; + u-boot-spl-dtb { + }; + }; + }; + + configurations { + default = "conf-1"; + conf-1 { + description = "Boot Linux kernel with FDT blob"; + kernel = "kernel"; + fdt = "fdt-1"; + }; + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/278_replace_section_deep.dts b/tools/binman/test/278_replace_section_deep.dts new file mode 100644 index 0000000000..fba2d7dcf2 --- /dev/null +++ b/tools/binman/test/278_replace_section_deep.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot-dtb { + }; + + section { + section { + blob { + filename = "compress"; + }; + }; + + u-boot { + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/279_x509_cert.dts b/tools/binman/test/279_x509_cert.dts new file mode 100644 index 0000000000..7123817271 --- /dev/null +++ b/tools/binman/test/279_x509_cert.dts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + x509-cert { + cert-ca = "IOT2050 Firmware Signature"; + cert-revision-int = <0>; + content = <&u_boot>; + }; + + u_boot: u-boot { + }; + }; +}; diff --git a/tools/binman/test/key.key b/tools/binman/test/key.key new file mode 100644 index 0000000000..9de3be14da --- /dev/null +++ b/tools/binman/test/key.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCSDLMHq1Jw3U+G +H2wutSGrT4Xhs5Yy7uhR/rDOiuKTW3zkVdfSIliye3Nnwrl/nNUFzEJ+4t/AiDaJ +Qk5KddTAJnOkw5SYBvFsTDhMR4HH6AyfzaaVl+AAGOg4LXwZzGYKncgOY5u6ZyMB +SzHxozJmmoqYaCIi4Iv2VZRZw1YPBoT6sv38RQSET5ci/g+89Sfb85ZPHPu6PLlz +ZTufG+yzAhIDsIvNpt2YlCnQ1TqoZxXsztxN1bKIP68xvlAQHSAB8+x4y0tYPE1I +UT1DK22FMgz5iyBp6ksFaqI06fITtJjPKG13z8sXXgb4/rJ5I0lhsn1ySsHQ0zLw +/CX4La2/VMA0Bw6GLFRhu/rOycqKfmwLm25bExV8xL6lwFohxbzBQgYr93ujGFyQ +AXBDOphvZcdXP3CHAcEViVRjrsBWNz8wyf7X8h2FIU16kAd30WuspjmnGuvRZ6Gn +SNDVO2tbEKvwkg6liYWy4IXtWcvooMtkhYyvFudcxRPgxEUTQ00biYfJ59ukqD7I +hyT7pq1bZDCVnAt6dUUPWZutrbBacsyITs01hyiPxvAAQ7XRoInmW1DLqHZ+gCJU +YJ0TaiAI8AmnjypMWRUo19l0zIgPdva8EJ+mz+kKFsZszo1nwuxQL7oUSUCb0hfB +2k3WxNthBi3QpspUKPtKweIg9ITtIwIDAQABAoICAA9dS6ZGZTVfauLKwnhFcOXT +R1vfpzDzhjg+CX6pCL4E1WY2C67dEySvrQvg5d/hcV2bR/GOT4izK72T3qWhsMCI +KwlN0/+MV3CTsiaALUyJAm77VQeOwy9vb1qdml0ibie2wpmU7AiXmgykSvxHNWGq +52KyLckqgz7mcOVikdah0nKHSwXzgs6iit1RCfnQdqGChjELdQX6Jm5X24ZZCzUn +xhpiQ8reP5iyGZYRIIsf0SQo/O8pSI9h173tbgHL9paOATYR+Pqu2Vh+x2meE3b8 +NXY5Jy9NSRgoSCk15VQiXyMH90Av+YcbSrN+I+tvhWREQUM5Txt3ZHgKprntoEYE +XLHAr9cvmIzLNeApt2z/g4t80xFBpIvTG3+SV/rthmq0KGCLW2kPkdujOiIwdNFF +6fJ6ikphKAbx2NgUY+6AM5AoOh5QPMqvCdsPwO21YG1WoxmiUpNTaYMlR1fDofr/ +A/z2bFH4SiJPkHXRT2KBiJh4ZZWNzP6hOqGy+jreOpWh5IAyn7cKx6t3I28Q9df0 +tK/1PLgR8WWu6G4uHtF5lKL+LgqFCTbSu9JtLQVQntD7Qyd98sF5o23QQWyA19uU +TVGxtkVaP1y7v+gtC+xMTW9MbGIeJiqMZuZ3xXJVvUNg1/2BDd+VAfPCOq6xGHC7 +s9MFqwUsLCAFFebXC8oVAoIBAQDKGc/o21Ags2t61IJaJjU7YwrsRywhZR+vUz5F +xtqH4jt9AkdWpDkKbO7xNMQ2OFdnobq5mkM+iW6Jvc1fi4gm1HDyP296nPKZdFrJ +UgGfTxOhxFLp7gsJ2F0GX5eDJYvqUTBeYB3wrQkCc+t7fLg2oS+gKGIIn2CP07Mx +Bist3eCcDvL6QIxYS43u+ptTyAItyUYn8KwvCxlIEfjxowsxfhRWuU+Mr4A4hfGB +64xSI1YU1AYZLMucOtK/mmlscfO8isdcyfea0GJn4VLRnNvAKL5g627IdErWHs3u +KgYWAXtVKzHrf4hO8dpVgIzO69wAsqZEvKYGmTJhfyvBN9DdAoIBAQC5AA7s2XOX +raVymhPwEy4I/2w9NuMFmTavOREBp/gA9uaWBdqAWn1rRJiJ5plgdcnOBFPSGBnc +thkuWBRqkklQ0YPKhNBT48CZGBN7VUsvyTZD1+IXLW1TmY5UGT0p6/dAYkoJHnvX +TAHl1tfmeHxVCJWV6Shf5LfJJwsAiykxzetkzmeaycy2s9GKCnkc2uFxyhKnfM0/ +SLwTuXQIJvHuErTYA4jjVOG9EGYW2/uKScPBLpB1YTliAUIvByDy6suCN5pVZGT8 +xVLTYec9lXjhfyhysOAjhD3w77Jh7Exft91fEK50k2ZkqYYnh+mYZcnR52msVSBS +3YL9kK/9dNX/AoIBABcEaZFzqOSQiqUqns31nApvdUcDtBr5kWo+aNE5nJntQiky +oT1U5soxLeV6xP4H3KyI1uNcllwA+v3lCAbhtVf2ygZNAz1LsrWXct+K33RtZSb/ +XRIXclpksfOP34moNQ8yv/d/qulGS8hju2YNBk3yfaIX91JUFINM8ROcSD6pDnO3 +oCSwRUupDzkwgZBBLz5Xtg3Gc1XIRdDXeyrKDvRMD7Tw1gaH1mqZlq/dS9XvAFbO +7wLe/zGD4YzA4VDgiYnnpF0FA5Y2NX7vQqds3fo8qbIQHkXmOL+6Mmn1j0viT1Gb +4cuYcsXK9brXMTI/2oaZ0iXx9la6C+reuPUAjmECggEBAInEvlips0hgW1ZV4cUm +M2El/dA0YKoZqDyjDcQi9zCYra1JXKe7O603XzVK0iugbBGM7XMG2bOgtG3r0ABx +QkH6VN/rOk1OzW31HQT6xswmVs/9I/TIsqLQNsrwJLlkbTO4PpQ97FGv27Xy4cNT +NJwKkYMbKCMJa8hT2ACmoZ3iUIs4nrUJ1Pa2QLRBCmJvqfYYWv35lcur+cvijsNH +ZWE68wvuzfEllBo87RnW5qLcPfhOGewf5CDU+RmWgHYGXllx2PAAnKgUtpKOVStq +daPQEyoeCDzKzWnwxvHfjBy4CxYxkQllf5o1GJ+1ukLwgnRbljltB25OYa89IaJp +cLcCggEAa5vbegzMKYPjR3zcVjnvhRsLXQi1vMtbUqOQ5wYMwGIef4v3QHNoF7EA +aNpWQ/qgCTQUzl3qoQCkRiVmVBBr60Fs5y7sfA92eBxQIV5hxJftH3vmiKqeWeqm +ila9DNw84MNAIqI2u6R3K/ur9fkSswDr3nzvFjuheW5V/M/6zAUtJZXr4iUih929 +uhf2dn6pSLR+epJ5023CVaI2zwz+U6PDEATKy9HjeKab3tQMHxQkT/5IWcLqrVTs +0rMobIgONzQqYDi2sO05YvgNBxvX3pUvqNlthcOtauT8BoE6wxLYm7ZcWYLPn15A +wR0+2mDpx+HDyu76q3M+KxXG2U8sJg== +-----END PRIVATE KEY----- diff --git a/tools/binman/test/key.pem b/tools/binman/test/key.pem new file mode 100644 index 0000000000..7a7b84a8bb --- /dev/null +++ b/tools/binman/test/key.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1kCFB/17qhcvpyKhG+jfS2c0qG1yjruMA0GCSqGSIb3DQEBCwUAMHUx +CzAJBgNVBAYTAk5aMRMwEQYDVQQIDApDYW50ZXJidXJ5MRUwEwYDVQQHDAxDaHJp +c3RjaHVyY2gxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUG +A1UEAwwOTXkgQ29tbW9uIE5hbWUwHhcNMjMwMjEzMDM1MzMzWhcNMjQwMjEzMDM1 +MzMzWjB1MQswCQYDVQQGEwJOWjETMBEGA1UECAwKQ2FudGVyYnVyeTEVMBMGA1UE +BwwMQ2hyaXN0Y2h1cmNoMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxFzAVBgNVBAMMDk15IENvbW1vbiBOYW1lMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEAkgyzB6tScN1Phh9sLrUhq0+F4bOWMu7oUf6wzorik1t85FXX +0iJYsntzZ8K5f5zVBcxCfuLfwIg2iUJOSnXUwCZzpMOUmAbxbEw4TEeBx+gMn82m +lZfgABjoOC18GcxmCp3IDmObumcjAUsx8aMyZpqKmGgiIuCL9lWUWcNWDwaE+rL9 +/EUEhE+XIv4PvPUn2/OWTxz7ujy5c2U7nxvsswISA7CLzabdmJQp0NU6qGcV7M7c +TdWyiD+vMb5QEB0gAfPseMtLWDxNSFE9QytthTIM+YsgaepLBWqiNOnyE7SYzyht +d8/LF14G+P6yeSNJYbJ9ckrB0NMy8Pwl+C2tv1TANAcOhixUYbv6zsnKin5sC5tu +WxMVfMS+pcBaIcW8wUIGK/d7oxhckAFwQzqYb2XHVz9whwHBFYlUY67AVjc/MMn+ +1/IdhSFNepAHd9FrrKY5pxrr0Wehp0jQ1TtrWxCr8JIOpYmFsuCF7VnL6KDLZIWM +rxbnXMUT4MRFE0NNG4mHyefbpKg+yIck+6atW2QwlZwLenVFD1mbra2wWnLMiE7N +NYcoj8bwAEO10aCJ5ltQy6h2foAiVGCdE2ogCPAJp48qTFkVKNfZdMyID3b2vBCf +ps/pChbGbM6NZ8LsUC+6FElAm9IXwdpN1sTbYQYt0KbKVCj7SsHiIPSE7SMCAwEA +ATANBgkqhkiG9w0BAQsFAAOCAgEAJAJoia6Vq4vXP/0bCgW3o9TOMmFYhI/xPxoh +Gd7was9R7BOrMGO+/3E7DZtjycZYL0r9nOtr9S/BBreuZ4vkk/PSoGaSnG8ST4jC +Ajk7ew/32RGOgA/oIzgKj1SPkBtvW+x+76sjUkGKsxmABBUhycIY7K0U8McTTfJ7 +gJ164VXmdG7qFMWmRy4Ry9QGXkDsbMSOZ485X7zbphjK5OZXEujP7GMUgg1lP479 +NqC1g+1m/A3PIB767lVYA7APQsrckHdRqOTkK9TYRQ3mvyE2wruhqE6lx8G/UyFh +RZjZ3lh2bx07UWIlyMabnGDMrM4FCnesqVyVAc8VAbkdXkeJI9r6DdFw+dzIY0P1 +il+MlYpZNwRyNv2W5SCPilyuhuPOSrSnsSHx64puCIvwG/4xA30Jw8nviJuyGSef +7uE+W7SD9E/hQHi/S9KRsYVoo7a6X9ADiwNsRNzVnuqc7K3mv/C5E9s6uFTNoObe +fUBA7pL3Fmvc5pYatxTFI85ajBpe/la6AA+7HX/8PXEphmp6GhFCcfsq+DL03vTM +DqIJL1i/JXggwqvvdcfaSeMDIOIzO89yUGGwwuj9rqMeEY99qDtljgy1EljjrB5i +0j4Jg4O0OEd2KIOD7nz4do1tLNlRcpysDZeXIiwAI7Dd3wWMsgpOQxs0zqWyqDVq +mCKa5Tw= +-----END CERTIFICATE----- diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index c2a69027f8..d81752e994 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -19,10 +19,10 @@ import time from buildman import builderthread from buildman import toolchain -from patman import command from patman import gitutil -from patman import terminal -from patman.terminal import tprint +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib.terminal import tprint # This indicates an new int or hex Kconfig property with no default # It hangs the build since the 'conf' tool cannot proceed without valid input. @@ -194,6 +194,8 @@ class Builder: work_in_output: Use the output directory as the work directory and don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs + no_lto (bool): True to set the NO_LTO flag when building + reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds Private members: _base_board_dict: Last-summarised Dict of boards @@ -253,7 +255,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False): + allow_missing=False, no_lto=False, reproducible_builds=False): """Create a new Builder object Args: @@ -292,6 +294,7 @@ class Builder: C=val to set the value of C (val must have quotes if C is a string Kconfig allow_missing: Run build with BINMAN_ALLOW_MISSING=1 + no_lto (bool): True to set the NO_LTO flag when building """ self.toolchains = toolchains @@ -331,6 +334,8 @@ class Builder: self.adjust_cfg = adjust_cfg self.allow_missing = allow_missing self._ide = False + self.no_lto = no_lto + self.reproducible_builds = reproducible_builds if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 680efae02d..879ff138ad 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -10,8 +10,8 @@ import sys import threading from buildman import cfgutil -from patman import command from patman import gitutil +from u_boot_pylib import command RETURN_CODE_RETRY = -1 BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl'] @@ -255,6 +255,10 @@ class BuilderThread(threading.Thread): args.append('KCFLAGS=-Werror') if self.builder.allow_missing: args.append('BINMAN_ALLOW_MISSING=1') + if self.builder.no_lto: + args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) @@ -273,14 +277,19 @@ class BuilderThread(threading.Thread): # If we need to reconfigure, do that now cfg_file = os.path.join(out_dir, '.config') + cmd_list = [] if do_config or adjust_cfg: config_out = '' if self.mrproper: result = self.Make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out += result.combined + cmd_list.append([self.builder.gnu_make, 'mrproper', + *args]) result = self.Make(commit, brd, 'config', cwd, *(args + config_args), env=env) + cmd_list.append([self.builder.gnu_make] + args + + config_args) config_out += result.combined do_config = False # No need to configure next time if adjust_cfg: @@ -290,6 +299,7 @@ class BuilderThread(threading.Thread): args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) + cmd_list.append([self.builder.gnu_make] + args) if (result.return_code == 2 and ('Some images are invalid' in result.stderr)): # This is handled later by the check for output in @@ -303,6 +313,7 @@ class BuilderThread(threading.Thread): result.stderr = result.stderr.replace(src_dir + '/', '') if self.builder.verbose_build: result.stdout = config_out + result.stdout + result.cmd_list = cmd_list else: result.return_code = 1 result.stderr = 'No tool chain for %s\n' % brd.arch @@ -378,6 +389,12 @@ class BuilderThread(threading.Thread): with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: for var in sorted(env.keys()): fd.write(b'%s="%s"' % (var, env[var])) + + with open(os.path.join(build_dir, 'out-cmd'), 'w', + encoding='utf-8') as fd: + for cmd in result.cmd_list: + print(' '.join(cmd), file=fd) + lines = [] for fname in BASE_ELF_FILENAMES: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 2a83cb7e4f..c8b0db3d8b 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the final binary. This information varies each time U-Boot is built. This causes various files to be rebuilt even if no source changes are made, which in turn requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. +be avoided by turning off the timestamp feature. This can be achieved using +the `-r` flag, which enables reproducible builds by setting +`SOURCE_DATE_EPOCH=0` when building. Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code:: - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + ./tools/buildman/buildman -Pr tegra Checking configuration @@ -1108,6 +1109,8 @@ and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a value for 'altbootcmd', but lost one for ' altbootcmd'. The -U option uses the u-boot.env files which are produced by a build. +Internally, buildman writes out an out-env file into the build directory for +later comparison. Building with clang @@ -1121,6 +1124,20 @@ toolchain. For example: buildman -O clang-7 --board sandbox +Building without LTO +-------------------- + +Link-time optimisation (LTO) is designed to reduce code size by globally +optimising the U-Boot build. Unfortunately this can dramatically slow down +builds. This is particularly noticeable when running a lot of builds. + +Use the -L (--no-lto) flag to disable LTO. + +.. code-block:: bash + + buildman -L --board sandbox + + Doing a simple build -------------------- @@ -1298,6 +1315,14 @@ You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, since it may be dropped altogether in future. +Checking the command +-------------------- + +Buildman writes out the toolchain information to a `toolchain` file within the +output directory. It also writes the commands used to build U-Boot in an +`out-cmd` file. You can check these if you suspect something strange is +happening. + TODO ---- diff --git a/tools/buildman/cfgutil.py b/tools/buildman/cfgutil.py index ab74a8ef06..a340e01cb6 100644 --- a/tools/buildman/cfgutil.py +++ b/tools/buildman/cfgutil.py @@ -7,7 +7,7 @@ import re -from patman import tools +from u_boot_pylib import tools RE_LINE = re.compile(r'(# )?CONFIG_([A-Z0-9_]+)(=(.*)| is not set)') RE_CFG = re.compile(r'(~?)(CONFIG_)?([A-Z0-9_]+)(=.*)?') diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index c485994e9f..a9cda24957 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -3,6 +3,11 @@ # from optparse import OptionParser +import os +import pathlib + +BUILDMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = os.path.exists(BUILDMAN_DIR / "test.py") def ParseArgs(): """Parse command line arguments from sys.argv[] @@ -71,6 +76,8 @@ def ParseArgs(): default=False, help="Don't convert y to 1 in configs") parser.add_option('-l', '--list-error-boards', action='store_true', default=False, help='Show a list of boards next to each error/warning') + parser.add_option('-L', '--no-lto', action='store_true', + default=False, help='Disable Link-time Optimisation (LTO) for builds') parser.add_option('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_option('-m', '--mrproper', action='store_true', @@ -95,18 +102,21 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-r', '--reproducible-builds', action='store_true', + help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_option('-R', '--regen-board-list', action='store_true', help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', default=False, help='Show image size variation in summary') - parser.add_option('--skip-net-tests', action='store_true', default=False, - help='Skip tests which need the network') parser.add_option('--step', type='int', default=1, help='Only build every n commits (0=just first and last)') - parser.add_option('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') + if HAS_TESTS: + parser.add_option('--skip-net-tests', action='store_true', default=False, + help='Skip tests which need the network') + parser.add_option('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') parser.add_option('-T', '--threads', type='int', default=None, help='Number of builder threads to use (0=single-thread)') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 87e7d0e201..35f44c0cf3 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -3,6 +3,7 @@ # import multiprocessing +import importlib.resources import os import shutil import subprocess @@ -13,12 +14,12 @@ from buildman import bsettings from buildman import cfgutil from buildman import toolchain from buildman.builder import Builder -from patman import command from patman import gitutil from patman import patchstream -from patman import terminal -from patman import tools -from patman.terminal import tprint +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib.terminal import tprint def GetPlural(count): """Returns a plural 's' if count is not 1""" @@ -152,9 +153,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, global builder if options.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), - 'README.rst')) + with importlib.resources.path('buildman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 gitutil.setup() @@ -261,9 +261,9 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, count += 1 # Build upstream commit also if not count: - str = ("No commits found to process in branch '%s': " + msg = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) - sys.exit(col.build(col.RED, str)) + sys.exit(col.build(col.RED, msg)) if options.work_in_output: if len(selected) != 1: sys.exit(col.build(col.RED, @@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, shutil.rmtree(output_dir) adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg) + # Drop LOCALVERSION_AUTO since it changes the version string on every commit + if options.reproducible_builds: + # If these are mentioned, leave the local version alone + if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: + print('Not dropping LOCALVERSION_AUTO for reproducible build') + else: + adjust_cfg['LOCALVERSION_AUTO'] = '~' + builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing) + allow_missing=allow_missing, no_lto=options.no_lto, + reproducible_builds=options.reproducible_builds) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 559e4edf74..ebd78f225e 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -14,11 +14,11 @@ from buildman import bsettings from buildman import cmdline from buildman import control from buildman import toolchain -from patman import command from patman import gitutil -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools settings_data = ''' # Buildman settings file @@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase): kwargs: Arguments to pass to command.run_pipe() """ self._make_calls += 1 + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] if stage == 'mrproper': return command.CommandResult(return_code=0) elif stage == 'config': + fname = os.path.join(cwd or '', out_dir, '.config') + tools.write_file(fname, b'CONFIG_SOMETHING=1') return command.CommandResult(return_code=0, combined='Test configuration complete') elif stage == 'build': stderr = '' - out_dir = '' - for arg in args: - if arg.startswith('O='): - out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot') @@ -723,3 +725,57 @@ Some images are invalid''' control.get_allow_missing(False, False, 2, True)) self.assertEqual(False, control.get_allow_missing(False, True, 2, True)) + + def check_command(self, *extra_args): + """Run a command with the extra arguments and return the commands used + + Args: + extra_args (list of str): List of extra arguments + + Returns: + list of str: Lines returned in the out-cmd file + """ + self._RunControl('-o', self._output_dir, *extra_args) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + cmd_fname = os.path.join(board0_dir, 'out-cmd') + self.assertTrue(os.path.exists(cmd_fname)) + data = tools.read_file(cmd_fname) + + config_fname = os.path.join(board0_dir, '.config') + self.assertTrue(os.path.exists(config_fname)) + cfg_data = tools.read_file(config_fname) + + return data.splitlines(), cfg_data + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + lines = self.check_command()[0] + self.assertEqual(2, len(lines)) + self.assertRegex(lines[0], b'make O=/.*board0_defconfig') + self.assertRegex(lines[0], b'make O=/.*-s.*') + + def testNoLto(self): + """Test that the --no-lto flag works""" + lines = self.check_command('-L')[0] + self.assertIn(b'NO_LTO=1', lines[0]) + + def testReproducible(self): + """Test that the -r flag works""" + lines, cfg_data = self.check_command('-r') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +# CONFIG_LOCALVERSION_AUTO is not set +''', cfg_data) + + with test_util.capture_sys_output() as (stdout, stderr): + lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +CONFIG_LOCALVERSION=y +''', cfg_data) + self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue()) diff --git a/tools/buildman/main.py b/tools/buildman/main.py index 67c560c48d..5e1f68d823 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -25,8 +25,8 @@ from buildman import control from buildman import toolchain from patman import patchstream from patman import gitutil -from patman import terminal -from patman import test_util +from u_boot_pylib import terminal +from u_boot_pylib import test_util def RunTests(skip_net_tests, verboose, args): from buildman import func_test @@ -46,17 +46,22 @@ def RunTests(skip_net_tests, verboose, args): return (0 if result.wasSuccessful() else 1) -options, args = cmdline.ParseArgs() +def run_buildman(): + options, args = cmdline.ParseArgs() -if not options.debug: - sys.tracebacklimit = 0 + if not options.debug: + sys.tracebacklimit = 0 -# Run our meagre tests -if options.test: - RunTests(options.skip_net_tests, options.verbose, args) + # Run our meagre tests + if cmdline.HAS_TESTS and options.test: + RunTests(options.skip_net_tests, options.verbose, args) -# Build selected commits for selected boards -else: - bsettings.Setup(options.config_file) - ret_code = control.DoBuildman(options, args) - sys.exit(ret_code) + # Build selected commits for selected boards + else: + bsettings.Setup(options.config_file) + ret_code = control.DoBuildman(options, args) + sys.exit(ret_code) + + +if __name__ == "__main__": + run_buildman() diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml new file mode 100644 index 0000000000..4d75e772ee --- /dev/null +++ b/tools/buildman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "buildman" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib", "patch-manager"] +description = "Buildman build tool for U-Boot" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/build/buildman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +buildman = "buildman.main:run_buildman" + +[tool.setuptools.package-data] +buildman = ["*.rst"] diff --git a/tools/buildman/test.py b/tools/buildman/test.py index daf5467503..9fa6445b79 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -17,10 +17,10 @@ from buildman import cfgutil from buildman import control from buildman import toolchain from patman import commit -from patman import command -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools use_network = True diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index ea1ad1bcb8..8f9130bdcd 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -11,9 +11,9 @@ import tempfile import urllib.request, urllib.error, urllib.parse from buildman import bsettings -from patman import command -from patman import terminal -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, PRIORITY_CALC) = list(range(4)) @@ -156,9 +156,8 @@ class Toolchain: Returns: Value of that environment variable or arguments """ - wrapper = self.GetWrapper() if which == VAR_CROSS_COMPILE: - return wrapper + os.path.join(self.path, self.cross) + return self.GetWrapper() + self.cross elif which == VAR_PATH: return self.path elif which == VAR_ARCH: diff --git a/tools/concurrencytest/.gitignore b/tools/concurrencytest/.gitignore deleted file mode 100644 index 0d20b6487c..0000000000 --- a/tools/concurrencytest/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc diff --git a/tools/concurrencytest/README.md b/tools/concurrencytest/README.md deleted file mode 100644 index 2d7fe75df5..0000000000 --- a/tools/concurrencytest/README.md +++ /dev/null @@ -1,74 +0,0 @@ -concurrencytest -=============== - - - -Python testtools extension for running unittest suites concurrently. - ----- - -Install from PyPI: -``` -pip install concurrencytest -``` - ----- - -Requires: - -* [testtools](https://pypi.python.org/pypi/testtools) : `pip install testtools` -* [python-subunit](https://pypi.python.org/pypi/python-subunit) : `pip install python-subunit` - ----- - -Example: - -```python -import time -import unittest - -from concurrencytest import ConcurrentTestSuite, fork_for_tests - - -class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - -# Load tests from SampleTestCase defined above -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -runner = unittest.TextTestRunner() - -# Run tests sequentially -runner.run(suite) - -# Run same tests across 4 processes -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) -runner.run(concurrent_suite) -``` -Output: - -``` -.... ----------------------------------------------------------------------- -Ran 4 tests in 2.003s - -OK -.... ----------------------------------------------------------------------- -Ran 4 tests in 0.504s - -OK -``` diff --git a/tools/concurrencytest/__init__.py b/tools/concurrencytest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/tools/concurrencytest/__init__.py +++ /dev/null diff --git a/tools/concurrencytest/concurrencytest.py b/tools/concurrencytest/concurrencytest.py deleted file mode 100644 index 1c4f03f37e..0000000000 --- a/tools/concurrencytest/concurrencytest.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0+ -# -# Modified by: Corey Goldberg, 2013 -# -# Original code from: -# Bazaar (bzrlib.tests.__init__.py, v2.6, copied Jun 01 2013) -# Copyright (C) 2005-2011 Canonical Ltd - -"""Python testtools extension for running unittest suites concurrently. - -The `testtools` project provides a ConcurrentTestSuite class, but does -not provide a `make_tests` implementation needed to use it. - -This allows you to parallelize a test run across a configurable number -of worker processes. While this can speed up CPU-bound test runs, it is -mainly useful for IO-bound tests that spend most of their time waiting for -data to arrive from someplace else and can benefit from cocncurrency. - -Unix only. -""" - -import os -import sys -import traceback -import unittest -from itertools import cycle -from multiprocessing import cpu_count - -from subunit import ProtocolTestCase, TestProtocolClient -from subunit.test_results import AutoTimingTestResultDecorator - -from testtools import ConcurrentTestSuite, iterate_tests -from testtools.content import TracebackContent, text_content - - -_all__ = [ - 'ConcurrentTestSuite', - 'fork_for_tests', - 'partition_tests', -] - - -CPU_COUNT = cpu_count() - - -class BufferingTestProtocolClient(TestProtocolClient): - """A TestProtocolClient which can buffer the test outputs - - This class captures the stdout and stderr output streams of the - tests as it runs them, and includes the output texts in the subunit - stream as additional details. - - Args: - stream: A file-like object to write a subunit stream to - buffer (bool): True to capture test stdout/stderr outputs and - include them in the test details - """ - def __init__(self, stream, buffer=True): - super().__init__(stream) - self.buffer = buffer - - def _addOutcome(self, outcome, test, error=None, details=None, - error_permitted=True): - """Report a test outcome to the subunit stream - - The parent class uses this function as a common implementation - for various methods that report successes, errors, failures, etc. - - This version automatically upgrades the error tracebacks to the - new 'details' format by wrapping them in a Content object, so - that we can include the captured test output in the test result - details. - - Args: - outcome: A string describing the outcome - used as the - event name in the subunit stream. - test: The test case whose outcome is to be reported - error: Standard unittest positional argument form - an - exc_info tuple. - details: New Testing-in-python drafted API; a dict from - string to subunit.Content objects. - error_permitted: If True then one and only one of error or - details must be supplied. If False then error must not - be supplied and details is still optional. - """ - if details is None: - details = {} - - # Parent will raise an exception if error_permitted is False but - # error is not None. We want that exception in that case, so - # don't touch error when error_permitted is explicitly False. - if error_permitted and error is not None: - # Parent class prefers error over details - details['traceback'] = TracebackContent(error, test) - error_permitted = False - error = None - - if self.buffer: - stdout = sys.stdout.getvalue() - if stdout: - details['stdout'] = text_content(stdout) - - stderr = sys.stderr.getvalue() - if stderr: - details['stderr'] = text_content(stderr) - - return super()._addOutcome(outcome, test, error=error, - details=details, error_permitted=error_permitted) - - -def fork_for_tests(concurrency_num=CPU_COUNT, buffer=False): - """Implementation of `make_tests` used to construct `ConcurrentTestSuite`. - - :param concurrency_num: number of processes to use. - """ - if buffer: - test_protocol_client_class = BufferingTestProtocolClient - else: - test_protocol_client_class = TestProtocolClient - - def do_fork(suite): - """Take suite and start up multiple runners by forking (Unix only). - - :param suite: TestSuite object. - - :return: An iterable of TestCase-like objects which can each have - run(result) called on them to feed tests to result. - """ - result = [] - test_blocks = partition_tests(suite, concurrency_num) - # Clear the tests from the original suite so it doesn't keep them alive - suite._tests[:] = [] - for process_tests in test_blocks: - process_suite = unittest.TestSuite(process_tests) - # Also clear each split list so new suite has only reference - process_tests[:] = [] - c2pread, c2pwrite = os.pipe() - pid = os.fork() - if pid == 0: - try: - stream = os.fdopen(c2pwrite, 'wb') - os.close(c2pread) - # Leave stderr and stdout open so we can see test noise - # Close stdin so that the child goes away if it decides to - # read from stdin (otherwise its a roulette to see what - # child actually gets keystrokes for pdb etc). - sys.stdin.close() - subunit_result = AutoTimingTestResultDecorator( - test_protocol_client_class(stream) - ) - process_suite.run(subunit_result) - except: - # Try and report traceback on stream, but exit with error - # even if stream couldn't be created or something else - # goes wrong. The traceback is formatted to a string and - # written in one go to avoid interleaving lines from - # multiple failing children. - try: - stream.write(traceback.format_exc()) - finally: - os._exit(1) - os._exit(0) - else: - os.close(c2pwrite) - stream = os.fdopen(c2pread, 'rb') - # If we don't pass the second argument here, it defaults - # to sys.stdout.buffer down the line. But if we don't - # pass it *now*, it may be resolved after sys.stdout is - # replaced with a StringIO (to capture tests' outputs) - # which doesn't have a buffer attribute and can end up - # occasionally causing a 'broken-runner' error. - test = ProtocolTestCase(stream, sys.stdout.buffer) - result.append(test) - return result - return do_fork - - -def partition_tests(suite, count): - """Partition suite into count lists of tests.""" - # This just assigns tests in a round-robin fashion. On one hand this - # splits up blocks of related tests that might run faster if they shared - # resources, but on the other it avoids assigning blocks of slow tests to - # just one partition. So the slowest partition shouldn't be much slower - # than the fastest. - partitions = [list() for _ in range(count)] - tests = iterate_tests(suite) - for partition, test in zip(cycle(partitions), tests): - partition.append(test) - return partitions - - -if __name__ == '__main__': - import time - - class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - # Load tests from SampleTestCase defined above - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - runner = unittest.TextTestRunner() - - # Run tests sequentially - runner.run(suite) - - # Run same tests across 4 processes - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) - runner.run(concurrent_suite) diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 33e2bd2add..fdcb0c7f3d 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -186,6 +186,27 @@ RUN git clone https://gitlab.com/qemu-project/qemu.git /tmp/qemu && \ make -j$(nproc) all install && \ rm -rf /tmp/qemu +# Build QEMU supporting Nokia n900 emulation +RUN mkdir -p /opt/nokia && \ + cd /tmp && \ + git clone https://git.linaro.org/qemu/qemu-linaro.git && \ + cd /tmp/qemu-linaro && \ + git checkout 8f8d8e0796efe1a6f34cdd83fb798f3c41217ec1 && \ + ./configure --enable-system --target-list=arm-softmmu \ + --python=/usr/bin/python2.7 --disable-sdl --disable-gtk \ + --disable-curses --audio-drv-list= --audio-card-list= \ + --disable-werror --disable-xen --disable-xen-pci-passthrough \ + --disable-brlapi --disable-vnc --disable-curl --disable-slirp \ + --disable-kvm --disable-user --disable-linux-user --disable-bsd-user \ + --disable-guest-base --disable-uuid --disable-vde --disable-linux-aio \ + --disable-cap-ng --disable-attr --disable-blobs --disable-docs \ + --disable-spice --disable-libiscsi --disable-smartcard-nss \ + --disable-usb-redir --disable-guest-agent --disable-seccomp \ + --disable-glusterfs --disable-nptl --disable-fdt && \ + make -j$(nproc) && \ + cp /tmp/qemu-linaro/arm-softmmu/qemu-system-arm /opt/nokia && \ + rm -rf /tmp/qemu-linaro + # Build genimage (required by some targets to generate disk images) RUN wget -O - https://github.com/pengutronix/genimage/releases/download/v14/genimage-14.tar.xz | tar -C /tmp -xJ && \ cd /tmp/genimage-14 && \ @@ -229,6 +250,16 @@ RUN mkdir /tmp/trace && \ sudo make install && \ rm -rf /tmp/trace +# Files to run Nokia RX-51 (aka N900) tests +RUN mkdir -p /opt/nokia && \ + cd /opt/nokia && \ + wget https://raw.githubusercontent.com/pali/u-boot-maemo/master/debian/u-boot-gen-combined && \ + chmod 0755 u-boot-gen-combined && \ + wget http://repository.maemo.org/qemu-n900/qemu-n900.tar.gz && \ + wget http://repository.maemo.org/pool/maemo5.0/free/k/kernel/kernel_2.6.28-20103103+0m5_armel.deb && \ + wget http://repository.maemo.org/pool/maemo5.0/free/g/glibc/libc6_2.5.1-1eglibc27+0m5_armel.deb && \ + wget http://repository.maemo.org/pool/maemo5.0/free/b/busybox/busybox_1.10.2.legal-1osso30+0m5_armel.deb + # Create our user/group RUN echo uboot ALL=NOPASSWD: ALL > /etc/sudoers.d/uboot RUN useradd -m -U uboot diff --git a/tools/dtoc/README.rst b/tools/dtoc/README.rst new file mode 100644 index 0000000000..92b39759ed --- /dev/null +++ b/tools/dtoc/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Devicetree-to-C generator +========================= + +This is a Python program and associated utilities, which supports converting +devicetree files into C code. It generates header files containing struct +definitions, as well as C files containing the data. It does not require any +modification of the devicetree files. + +Some high-level libraries are provided for working with devicetree. These may +be useful in other projects. + +This package also includes some U-Boot-specific features, such as creating +`struct udevice` and `struct uclass` entries for devicetree nodes. diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index a69a7889ce..39f416cfd8 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -35,9 +35,9 @@ PROP_IGNORE_LIST = [ 'linux,phandle', "status", 'phandle', - 'u-boot,dm-pre-reloc', - 'u-boot,dm-tpl', - 'u-boot,dm-spl', + 'bootph-all', + 'bootph-pre-sram', + 'bootph-pre-ram', ] # C type declarations for the types we support @@ -442,7 +442,7 @@ class DtbPlatdata(): """ parent = node.parent if parent and not parent.props: - raise ValueError("Parent node '%s' has no properties - do you need u-boot,dm-spl or similar?" % + raise ValueError("Parent node '%s' has no properties - do you need bootph-pre-ram or similar?" % parent.path) num_addr, num_size = 2, 2 if parent: @@ -754,7 +754,7 @@ class DtbPlatdata(): # This might indicate that the parent node is not in the # SPL/TPL devicetree but the child is. For example if we are # dealing with of-platdata in TPL, the parent has a - # u-boot,dm-tpl tag but the child has u-boot,dm-pre-reloc. In + # bootph-pre-sram tag but the child has bootph-all. In # this case the child node exists in TPL but the parent does # not. raise ValueError("Node '%s' requires parent node '%s' but it is not in the valid list" % diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index d933972918..a8e05349a7 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -12,7 +12,7 @@ import sys from dtoc import fdt_util import libfdt from libfdt import QUIET_NOTFOUND -from patman import tools +from u_boot_pylib import tools # This deals with a device tree, presenting it as an assortment of Node and # Prop objects, representing nodes and properties, respectively. This file diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index f34316632a..f1f70568cf 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -13,8 +13,8 @@ import struct import sys import tempfile -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools def fdt32_to_cpu(val): """Convert a device tree cell to an integer diff --git a/tools/dtoc/main.py b/tools/dtoc/main.py index 5508759d4d..6c91450410 100755 --- a/tools/dtoc/main.py +++ b/tools/dtoc/main.py @@ -23,6 +23,7 @@ see doc/driver-model/of-plat.rst from argparse import ArgumentParser import os +import pathlib import sys # Bring in the patman libraries @@ -35,7 +36,10 @@ sys.path.insert(0, os.path.join(our_path, '../../build-sandbox_spl/scripts/dtc/pylibfdt')) from dtoc import dtb_platdata -from patman import test_util +from u_boot_pylib import test_util + +DTOC_DIR = pathlib.Path(__file__).parent +HAVE_TESTS = (DTOC_DIR / 'test_dtoc.py').exists() def run_tests(processes, args): """Run all the test we have for dtoc @@ -61,54 +65,62 @@ def run_tests(processes, args): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(): +def RunTestCoverage(build_dir): """Run the tests and check that we get 100% coverage""" sys.argv = [sys.argv[0]] test_util.run_test_coverage('tools/dtoc/dtoc', '/main.py', - ['tools/patman/*.py', '*/fdt*', '*test*'], args.build_dir) - - -if __name__ != '__main__': - sys.exit(1) - -epilog = '''Generate C code from devicetree files. See of-plat.rst for details''' - -parser = ArgumentParser(epilog=epilog) -parser.add_argument('-B', '--build-dir', type=str, default='b', - help='Directory containing the build output') -parser.add_argument('-c', '--c-output-dir', action='store', - help='Select output directory for C files') -parser.add_argument('-C', '--h-output-dir', action='store', - help='Select output directory for H files (defaults to --c-output-di)') -parser.add_argument('-d', '--dtb-file', action='store', - help='Specify the .dtb input file') -parser.add_argument('-i', '--instantiate', action='store_true', default=False, - help='Instantiate devices to avoid needing device_bind()') -parser.add_argument('--include-disabled', action='store_true', - help='Include disabled nodes') -parser.add_argument('-o', '--output', action='store', - help='Select output filename') -parser.add_argument('-p', '--phase', type=str, - help='set phase of U-Boot this invocation is for (spl/tpl)') -parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') -parser.add_argument('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') -parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') -parser.add_argument('files', nargs='*') -args = parser.parse_args() - -# Run our meagre tests -if args.test: - ret_code = run_tests(args.processes, args) - sys.exit(ret_code) - -elif args.test_coverage: - RunTestCoverage() - -else: - dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, - args.output, - [args.c_output_dir, args.h_output_dir], - args.phase, instantiate=args.instantiate) + ['tools/patman/*.py', 'tools/u_boot_pylib/*','*/fdt*', '*test*'], + build_dir) + + +def run_dtoc(): + epilog = 'Generate C code from devicetree files. See of-plat.rst for details' + + parser = ArgumentParser(epilog=epilog) + parser.add_argument('-B', '--build-dir', type=str, default='b', + help='Directory containing the build output') + parser.add_argument('-c', '--c-output-dir', action='store', + help='Select output directory for C files') + parser.add_argument( + '-C', '--h-output-dir', action='store', + help='Select output directory for H files (defaults to --c-output-di)') + parser.add_argument('-d', '--dtb-file', action='store', + help='Specify the .dtb input file') + parser.add_argument( + '-i', '--instantiate', action='store_true', default=False, + help='Instantiate devices to avoid needing device_bind()') + parser.add_argument('--include-disabled', action='store_true', + help='Include disabled nodes') + parser.add_argument('-o', '--output', action='store', + help='Select output filename') + parser.add_argument( + '-p', '--phase', type=str, + help='set phase of U-Boot this invocation is for (spl/tpl)') + parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + if HAVE_TESTS: + parser.add_argument('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') + parser.add_argument( + '-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + parser.add_argument('files', nargs='*') + args = parser.parse_args() + + # Run our meagre tests + if HAVE_TESTS and args.test: + ret_code = run_tests(args.processes, args) + sys.exit(ret_code) + + elif HAVE_TESTS and args.test_coverage: + RunTestCoverage(args.build_dir) + + else: + dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, + args.output, + [args.c_output_dir, args.h_output_dir], + args.phase, instantiate=args.instantiate) + + +if __name__ == '__main__': + run_dtoc() diff --git a/tools/dtoc/pyproject.toml b/tools/dtoc/pyproject.toml new file mode 100644 index 0000000000..77fe4da215 --- /dev/null +++ b/tools/dtoc/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dtoc" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib"] +description = "Devicetree-to-C generator" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/driver-model/of-plat.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +dtoc = "dtoc.main:run_dtoc" diff --git a/tools/dtoc/test/dtoc_test_add_prop.dts b/tools/dtoc/test/dtoc_test_add_prop.dts index fa296e5552..8225de36d2 100644 --- a/tools/dtoc/test/dtoc_test_add_prop.dts +++ b/tools/dtoc/test/dtoc_test_add_prop.dts @@ -11,13 +11,13 @@ #address-cells = <1>; #size-cells = <1>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intval = <1>; }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intarray = <5>; }; diff --git a/tools/dtoc/test/dtoc_test_addr32.dts b/tools/dtoc/test/dtoc_test_addr32.dts index 239045497c..3e7dc56729 100644 --- a/tools/dtoc/test/dtoc_test_addr32.dts +++ b/tools/dtoc/test/dtoc_test_addr32.dts @@ -12,13 +12,13 @@ #size-cells = <1>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x98765432 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr32_64.dts b/tools/dtoc/test/dtoc_test_addr32_64.dts index 7599d5b0a5..7ce16feef1 100644 --- a/tools/dtoc/test/dtoc_test_addr32_64.dts +++ b/tools/dtoc/test/dtoc_test_addr32_64.dts @@ -12,19 +12,19 @@ #size-cells = <2>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x5678 0x0>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x98765432 0x10987654>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = <0x12345678 0x98765432 0x10987654 2 0 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr64.dts b/tools/dtoc/test/dtoc_test_addr64.dts index 263d251386..5f8c23f04b 100644 --- a/tools/dtoc/test/dtoc_test_addr64.dts +++ b/tools/dtoc/test/dtoc_test_addr64.dts @@ -12,19 +12,19 @@ #size-cells = <2>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = /bits/ 64 <0x1234 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = /bits/ 64 <0x1234567890123456 0x9876543210987654>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = /bits/ 64 <0x1234567890123456 0x9876543210987654 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_addr64_32.dts b/tools/dtoc/test/dtoc_test_addr64_32.dts index 85e4f5fdae..bfbfd87b8d 100644 --- a/tools/dtoc/test/dtoc_test_addr64_32.dts +++ b/tools/dtoc/test/dtoc_test_addr64_32.dts @@ -12,19 +12,19 @@ #size-cells = <1>; test1 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test1"; reg = <0x1234 0x0 0x5678>; }; test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test2"; reg = <0x12345678 0x90123456 0x98765432>; }; test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "test3"; reg = <0x12345678 0x90123456 0x98765432 0 2 3>; }; diff --git a/tools/dtoc/test/dtoc_test_alias_bad.dts b/tools/dtoc/test/dtoc_test_alias_bad.dts index d4f502ad0a..69761f9114 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_alias_bad_path.dts b/tools/dtoc/test/dtoc_test_alias_bad_path.dts index 0beca4f0d0..6f566fe4ab 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad_path.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad_path.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_alias_bad_uc.dts b/tools/dtoc/test/dtoc_test_alias_bad_uc.dts index ae64f5b3b2..5d23c63a63 100644 --- a/tools/dtoc/test/dtoc_test_alias_bad_uc.dts +++ b/tools/dtoc/test/dtoc_test_alias_bad_uc.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_aliases.dts b/tools/dtoc/test/dtoc_test_aliases.dts index ae33716863..018b834046 100644 --- a/tools/dtoc/test/dtoc_test_aliases.dts +++ b/tools/dtoc/test/dtoc_test_aliases.dts @@ -9,13 +9,13 @@ / { spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "compat1", "compat2.1-fred", "compat3"; intval = <1>; }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "compat1", "simple_bus"; intval = <1>; }; diff --git a/tools/dtoc/test/dtoc_test_driver_alias.dts b/tools/dtoc/test/dtoc_test_driver_alias.dts index da7973b2e5..22369a4406 100644 --- a/tools/dtoc/test/dtoc_test_driver_alias.dts +++ b/tools/dtoc/test/dtoc_test_driver_alias.dts @@ -9,7 +9,7 @@ / { gpio_a: gpios@0 { - u-boot,dm-pre-reloc; + bootph-all; gpio-controller; compatible = "sandbox_gpio_alias"; #gpio-cells = <1>; diff --git a/tools/dtoc/test/dtoc_test_inst.dts b/tools/dtoc/test/dtoc_test_inst.dts index b8177fcef5..9689be391b 100644 --- a/tools/dtoc/test/dtoc_test_inst.dts +++ b/tools/dtoc/test/dtoc_test_inst.dts @@ -18,20 +18,20 @@ }; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; intval = <1>; }; i2c: i2c { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,i2c"; intval = <3>; }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; diff --git a/tools/dtoc/test/dtoc_test_invalid_driver.dts b/tools/dtoc/test/dtoc_test_invalid_driver.dts index 914ac3e899..042a325913 100644 --- a/tools/dtoc/test/dtoc_test_invalid_driver.dts +++ b/tools/dtoc/test/dtoc_test_invalid_driver.dts @@ -9,7 +9,7 @@ / { spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "invalid"; }; }; diff --git a/tools/dtoc/test/dtoc_test_noparent.dts b/tools/dtoc/test/dtoc_test_noparent.dts index e976dd2b8a..0efb17e0cb 100644 --- a/tools/dtoc/test/dtoc_test_noparent.dts +++ b/tools/dtoc/test/dtoc_test_noparent.dts @@ -12,18 +12,18 @@ #size-cells = <1>; i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-tpl; + bootph-pre-sram; #address-cells = <1>; #size-cells = <0>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; #address-cells = <1>; #size-cells = <0>; status = "disabled"; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_noprops.dts b/tools/dtoc/test/dtoc_test_noprops.dts index e6fdd11b83..75296beb31 100644 --- a/tools/dtoc/test/dtoc_test_noprops.dts +++ b/tools/dtoc/test/dtoc_test_noprops.dts @@ -13,7 +13,7 @@ i2c@0 { pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_phandle.dts b/tools/dtoc/test/dtoc_test_phandle.dts index d9aa433503..74a146b9a3 100644 --- a/tools/dtoc/test/dtoc_test_phandle.dts +++ b/tools/dtoc/test/dtoc_test_phandle.dts @@ -9,34 +9,34 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #clock-cells = <0>; }; phandle_1: phandle2-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <1>; #clock-cells = <1>; }; phandle_2: phandle3-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <2>; #clock-cells = <2>; }; phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle &phandle_1 11 &phandle_2 12 13 &phandle>; phandle-name-offset = <&phandle_2>, "fred", <123>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_bad.dts b/tools/dtoc/test/dtoc_test_phandle_bad.dts index a3ddc59585..94cfada95b 100644 --- a/tools/dtoc/test/dtoc_test_phandle_bad.dts +++ b/tools/dtoc/test/dtoc_test_phandle_bad.dts @@ -9,7 +9,7 @@ / { phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <20>; /* Invalid phandle */ }; diff --git a/tools/dtoc/test/dtoc_test_phandle_bad2.dts b/tools/dtoc/test/dtoc_test_phandle_bad2.dts index fe25f565fb..4d24b96ce6 100644 --- a/tools/dtoc/test/dtoc_test_phandle_bad2.dts +++ b/tools/dtoc/test/dtoc_test_phandle_bad2.dts @@ -9,13 +9,13 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts b/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts index 241743e73e..6ad8006266 100644 --- a/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts +++ b/tools/dtoc/test/dtoc_test_phandle_cd_gpios.dts @@ -9,33 +9,33 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #gpio-cells = <0>; }; phandle_1: phandle2-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <1>; #gpio-cells = <1>; }; phandle_2: phandle3-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <2>; #gpio-cells = <2>; }; phandle-source { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; cd-gpios = <&phandle &phandle_1 11 &phandle_2 12 13 &phandle>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; cd-gpios = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_reorder.dts b/tools/dtoc/test/dtoc_test_phandle_reorder.dts index aa71d56f27..573a4f6396 100644 --- a/tools/dtoc/test/dtoc_test_phandle_reorder.dts +++ b/tools/dtoc/test/dtoc_test_phandle_reorder.dts @@ -10,13 +10,13 @@ / { phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; #clock-cells = <0>; }; diff --git a/tools/dtoc/test/dtoc_test_phandle_single.dts b/tools/dtoc/test/dtoc_test_phandle_single.dts index aacd0b15fa..1b1763932c 100644 --- a/tools/dtoc/test/dtoc_test_phandle_single.dts +++ b/tools/dtoc/test/dtoc_test_phandle_single.dts @@ -9,14 +9,14 @@ / { phandle: phandle-target { - u-boot,dm-pre-reloc; + bootph-all; compatible = "target"; intval = <0>; #clock-cells = <0>; }; phandle-source2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "source"; clocks = <&phandle>; }; diff --git a/tools/dtoc/test/dtoc_test_simple.dts b/tools/dtoc/test/dtoc_test_simple.dts index aef07efeae..08f667ee5a 100644 --- a/tools/dtoc/test/dtoc_test_simple.dts +++ b/tools/dtoc/test/dtoc_test_simple.dts @@ -11,7 +11,7 @@ #address-cells = <1>; #size-cells = <1>; spl-test { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; boolval; maybe-empty-int = <>; @@ -27,7 +27,7 @@ }; spl-test2 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; intval = <3>; intarray = <5>; @@ -40,7 +40,7 @@ }; spl-test3 { - u-boot,dm-pre-reloc; + bootph-all; compatible = "sandbox,spl-test"; stringarray = "one"; longbytearray = [09 0a 0b 0c 0d 0e 0f 10]; @@ -49,12 +49,12 @@ i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-pre-reloc; + bootph-all; #address-cells = <1>; #size-cells = <0>; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; }; diff --git a/tools/dtoc/test/dtoc_test_single_reg.dts b/tools/dtoc/test/dtoc_test_single_reg.dts index 804b67855b..035937cfbf 100644 --- a/tools/dtoc/test/dtoc_test_single_reg.dts +++ b/tools/dtoc/test/dtoc_test_single_reg.dts @@ -13,12 +13,12 @@ i2c@0 { compatible = "sandbox,i2c"; - u-boot,dm-pre-reloc; + bootph-all; #address-cells = <1>; #size-cells = <0>; pmic@9 { compatible = "sandbox,pmic"; - u-boot,dm-pre-reloc; + bootph-all; reg = <9>; low-power; diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index c62fcbac83..597c93e8a8 100755 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -13,6 +13,7 @@ import collections import copy import glob import os +import pathlib import struct import unittest @@ -25,10 +26,11 @@ from dtoc.dtb_platdata import get_value from dtoc.dtb_platdata import tab_to from dtoc.src_scan import conv_name_to_c from dtoc.src_scan import get_compat_name -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools -OUR_PATH = os.path.dirname(os.path.realpath(__file__)) +DTOC_DIR = pathlib.Path(__file__).parent +TEST_DATA_DIR = DTOC_DIR / 'test/' HEADER = '''/* @@ -91,7 +93,7 @@ def get_dtb_file(dts_fname, capture_stderr=False): Returns: str: Filename of compiled file in output directory """ - return fdt_util.EnsureCompiled(os.path.join(OUR_PATH, 'test', dts_fname), + return fdt_util.EnsureCompiled(str(TEST_DATA_DIR / dts_fname), capture_stderr=capture_stderr) diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 3b8ee00d4e..32fa69cbb0 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -30,8 +30,8 @@ from dtoc import fdt_util from dtoc.fdt_util import fdt32_to_cpu, fdt64_to_cpu from dtoc.fdt import Type, BytesToValue import libfdt -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools #pylint: disable=protected-access @@ -132,10 +132,10 @@ class TestFdt(unittest.TestCase): """Tests obtaining a list of properties""" node = self.dtb.GetNode('/spl-test') props = self.dtb.GetProps(node) - self.assertEqual(['boolval', 'bytearray', 'byteval', 'compatible', - 'int64val', 'intarray', 'intval', 'longbytearray', - 'maybe-empty-int', 'notstring', 'stringarray', - 'stringval', 'u-boot,dm-pre-reloc'], + self.assertEqual(['boolval', 'bootph-all', 'bytearray', 'byteval', + 'compatible', 'int64val', 'intarray', 'intval', + 'longbytearray', 'maybe-empty-int', 'notstring', + 'stringarray', 'stringval', ], sorted(props.keys())) def test_check_error(self): @@ -814,7 +814,8 @@ def run_test_coverage(build_dir): build_dir (str): Directory containing the build output """ test_util.run_test_coverage('tools/dtoc/test_fdt.py', None, - ['tools/patman/*.py', '*test_fdt.py'], build_dir) + ['tools/patman/*.py', 'tools/u_boot_pylib/*', '*test_fdt.py'], + build_dir) def run_tests(names, processes): diff --git a/tools/dtoc/test_src_scan.py b/tools/dtoc/test_src_scan.py index f93cd7f5a3..64b740841c 100644 --- a/tools/dtoc/test_src_scan.py +++ b/tools/dtoc/test_src_scan.py @@ -15,8 +15,8 @@ import unittest from unittest import mock from dtoc import src_scan -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools OUR_PATH = os.path.dirname(os.path.realpath(__file__)) diff --git a/tools/kwbimage.c b/tools/kwbimage.c index 6abb9f2d5c..309657a563 100644 --- a/tools/kwbimage.c +++ b/tools/kwbimage.c @@ -927,6 +927,71 @@ done: return ret; } +static int image_fill_xip_header(void *image, struct image_tool_params *params) +{ + struct main_hdr_v1 *main_hdr = image; /* kwbimage v0 and v1 have same XIP members */ + int version = kwbimage_version(image); + uint32_t srcaddr = le32_to_cpu(main_hdr->srcaddr); + uint32_t startaddr = 0; + + if (main_hdr->blockid != IBR_HDR_SPI_ID) { + fprintf(stderr, "XIP is supported only for SPI images\n"); + return 0; + } + + if (version == 0 && + params->addr >= 0xE8000000 && params->addr < 0xEFFFFFFF && + params->ep >= 0xE8000000 && params->ep < 0xEFFFFFFF) { + /* Load and Execute address is in SPI address space (kwbimage v0) */ + startaddr = 0xE8000000; + } else if (version != 0 && + params->addr >= 0xD4000000 && params->addr < 0xD7FFFFFF && + params->ep >= 0xD4000000 && params->ep < 0xD7FFFFFF) { + /* Load and Execute address is in SPI address space (kwbimage v1) */ + startaddr = 0xD4000000; + } else if (version != 0 && + params->addr >= 0xD8000000 && params->addr < 0xDFFFFFFF && + params->ep >= 0xD8000000 && params->ep < 0xDFFFFFFF) { + /* Load and Execute address is in Device bus space (kwbimage v1) */ + startaddr = 0xD8000000; + } else if (params->addr != 0x0) { + /* Load address is non-zero */ + if (version == 0) + fprintf(stderr, "XIP Load Address or XIP Entry Point is not in SPI address space\n"); + else + fprintf(stderr, "XIP Load Address or XIP Entry Point is not in SPI nor in Device bus address space\n"); + return 0; + } + + /* + * For XIP destaddr must be set to 0xFFFFFFFF and + * execaddr relative to the start of XIP memory address space. + */ + main_hdr->destaddr = cpu_to_le32(0xFFFFFFFF); + + if (startaddr == 0) { + /* + * mkimage's --load-address 0x0 means that binary is Position + * Independent and in this case mkimage's --entry-point address + * is relative offset from beginning of the data part of image. + */ + main_hdr->execaddr = cpu_to_le32(srcaddr + params->ep); + } else { + /* The lowest possible load address is after the header at srcaddr. */ + if (params->addr - startaddr < srcaddr) { + fprintf(stderr, + "Invalid XIP Load Address 0x%08x.\n" + "The lowest address for this configuration is 0x%08x.\n", + params->addr, (unsigned)(startaddr + srcaddr)); + return 0; + } + main_hdr->srcaddr = cpu_to_le32(params->addr - startaddr); + main_hdr->execaddr = cpu_to_le32(params->ep - startaddr); + } + + return 1; +} + static size_t image_headersz_align(size_t headersz, uint8_t blockid) { /* @@ -959,10 +1024,10 @@ static size_t image_headersz_v0(int *hasext) *hasext = 1; } - return image_headersz_align(headersz, image_get_bootfrom()); + return headersz; } -static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, +static void *image_create_v0(size_t *dataoff, struct image_tool_params *params, int payloadsz) { struct image_cfg_element *e; @@ -972,10 +1037,11 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, int has_ext = 0; /* - * Calculate the size of the header and the size of the + * Calculate the size of the header and the offset of the * payload */ headersz = image_headersz_v0(&has_ext); + *dataoff = image_headersz_align(headersz, image_get_bootfrom()); image = malloc(headersz); if (!image) { @@ -990,7 +1056,7 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, /* Fill in the main header */ main_hdr->blocksize = cpu_to_le32(payloadsz); - main_hdr->srcaddr = cpu_to_le32(headersz); + main_hdr->srcaddr = cpu_to_le32(*dataoff); main_hdr->ext = has_ext; main_hdr->version = 0; main_hdr->destaddr = cpu_to_le32(params->addr); @@ -1009,31 +1075,26 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, e = image_find_option(IMAGE_CFG_NAND_BADBLK_LOCATION); if (e) main_hdr->nandbadblklocation = e->nandbadblklocation; - main_hdr->checksum = image_checksum8(image, - sizeof(struct main_hdr_v0)); /* - * For SATA srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 1. + * For SATA srcaddr is specified in number of sectors. * This expects the sector size to be 512 bytes. - * Header size is already aligned. */ if (main_hdr->blockid == IBR_HDR_SATA_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512 + 1); - - /* - * For SDIO srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 0. - * This expects sector size to be 512 bytes. - * Header size is already aligned. - */ - if (main_hdr->blockid == IBR_HDR_SDIO_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512); + main_hdr->srcaddr = cpu_to_le32(le32_to_cpu(main_hdr->srcaddr) / 512); /* For PCIe srcaddr is not used and must be set to 0xFFFFFFFF. */ if (main_hdr->blockid == IBR_HDR_PEX_ID) main_hdr->srcaddr = cpu_to_le32(0xFFFFFFFF); + if (params->xflag) { + if (!image_fill_xip_header(main_hdr, params)) { + free(image); + return NULL; + } + *dataoff = le32_to_cpu(main_hdr->srcaddr); + } + /* Generate the ext header */ if (has_ext) { struct ext_hdr_v0 *ext_hdr; @@ -1059,7 +1120,9 @@ static void *image_create_v0(size_t *imagesz, struct image_tool_params *params, sizeof(struct ext_hdr_v0)); } - *imagesz = headersz; + main_hdr->checksum = image_checksum8(image, + sizeof(struct main_hdr_v0)); + return image; } @@ -1073,10 +1136,6 @@ static size_t image_headersz_v1(int *hasext) int cfgi; int ret; - /* - * Calculate the size of the header and the size of the - * payload - */ headersz = sizeof(struct main_hdr_v1); if (image_get_csk_index() >= 0) { @@ -1172,7 +1231,7 @@ static size_t image_headersz_v1(int *hasext) if (count > 0) headersz += sizeof(struct register_set_hdr_v1) + 8 * count + 4; - return image_headersz_align(headersz, image_get_bootfrom()); + return headersz; } static int add_binary_header_v1(uint8_t **cur, uint8_t **next_ext, @@ -1331,16 +1390,14 @@ static int kwb_sign_csk_with_kak(struct image_tool_params *params, return 0; } -static int add_secure_header_v1(struct image_tool_params *params, uint8_t *ptr, - int payloadsz, size_t headersz, uint8_t *image, +static int add_secure_header_v1(struct image_tool_params *params, uint8_t *image_ptr, + size_t image_size, uint8_t *header_ptr, size_t headersz, struct secure_hdr_v1 *secure_hdr) { struct image_cfg_element *e_jtagdelay; struct image_cfg_element *e_boxid; struct image_cfg_element *e_flashid; RSA *csk = NULL; - unsigned char *image_ptr; - size_t image_size; struct sig_v1 tmp_sig; bool specialized_img = image_get_spezialized_img(); @@ -1366,14 +1423,11 @@ static int add_secure_header_v1(struct image_tool_params *params, uint8_t *ptr, if (kwb_sign_csk_with_kak(params, secure_hdr, csk)) return 1; - image_ptr = ptr + headersz; - image_size = payloadsz - headersz; - - if (kwb_sign_and_verify(csk, image_ptr, image_size, + if (kwb_sign_and_verify(csk, image_ptr, image_size - 4, &secure_hdr->imgsig, "image") < 0) return 1; - if (kwb_sign_and_verify(csk, image, headersz, &tmp_sig, "header") < 0) + if (kwb_sign_and_verify(csk, header_ptr, headersz, &tmp_sig, "header") < 0) return 1; secure_hdr->hdrsig = tmp_sig; @@ -1399,12 +1453,11 @@ static void finish_register_set_header_v1(uint8_t **cur, uint8_t **next_ext, *datai = 0; } -static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, +static void *image_create_v1(size_t *dataoff, struct image_tool_params *params, uint8_t *ptr, int payloadsz) { struct image_cfg_element *e; struct main_hdr_v1 *main_hdr; - struct opt_hdr_v1 *ohdr; struct register_set_hdr_v1 *register_set_hdr; struct secure_hdr_v1 *secure_hdr = NULL; size_t headersz; @@ -1415,12 +1468,13 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, uint8_t delay; /* - * Calculate the size of the header and the size of the + * Calculate the size of the header and the offset of the * payload */ headersz = image_headersz_v1(&hasext); if (headersz == 0) return NULL; + *dataoff = image_headersz_align(headersz, image_get_bootfrom()); image = malloc(headersz); if (!image) { @@ -1442,7 +1496,7 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, main_hdr->headersz_msb = (headersz & 0xFFFF0000) >> 16; main_hdr->destaddr = cpu_to_le32(params->addr); main_hdr->execaddr = cpu_to_le32(params->ep); - main_hdr->srcaddr = cpu_to_le32(headersz); + main_hdr->srcaddr = cpu_to_le32(*dataoff); main_hdr->ext = hasext; main_hdr->version = 1; main_hdr->blockid = image_get_bootfrom(); @@ -1470,27 +1524,24 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, main_hdr->flags = e->debug ? 0x1 : 0; /* - * For SATA srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 1. + * For SATA srcaddr is specified in number of sectors. * This expects the sector size to be 512 bytes. - * Header size is already aligned. */ if (main_hdr->blockid == IBR_HDR_SATA_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512 + 1); - - /* - * For SDIO srcaddr is specified in number of sectors starting from - * sector 0. The main header is stored at sector number 0. - * This expects sector size to be 512 bytes. - * Header size is already aligned. - */ - if (main_hdr->blockid == IBR_HDR_SDIO_ID) - main_hdr->srcaddr = cpu_to_le32(headersz / 512); + main_hdr->srcaddr = cpu_to_le32(le32_to_cpu(main_hdr->srcaddr) / 512); /* For PCIe srcaddr is not used and must be set to 0xFFFFFFFF. */ if (main_hdr->blockid == IBR_HDR_PEX_ID) main_hdr->srcaddr = cpu_to_le32(0xFFFFFFFF); + if (params->xflag) { + if (!image_fill_xip_header(main_hdr, params)) { + free(image); + return NULL; + } + *dataoff = le32_to_cpu(main_hdr->srcaddr); + } + if (image_get_csk_index() >= 0) { /* * only reserve the space here; we fill the header later since @@ -1552,19 +1603,10 @@ static void *image_create_v1(size_t *imagesz, struct image_tool_params *params, &datai, delay); } - if (secure_hdr && add_secure_header_v1(params, ptr, payloadsz + headersz, - headersz, image, secure_hdr)) + if (secure_hdr && add_secure_header_v1(params, ptr + *dataoff, payloadsz, + image, headersz, secure_hdr)) return NULL; - *imagesz = headersz; - - /* Fill the real header size without padding into the main header */ - headersz = sizeof(*main_hdr); - for_each_opt_hdr_v1 (ohdr, main_hdr) - headersz += opt_hdr_v1_size(ohdr); - main_hdr->headersz_lsb = cpu_to_le16(headersz & 0xFFFF); - main_hdr->headersz_msb = (headersz & 0xFFFF0000) >> 16; - /* Calculate and set the header checksum */ main_hdr->checksum = image_checksum8(main_hdr, headersz); @@ -1835,7 +1877,7 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, FILE *fcfg; void *image = NULL; int version; - size_t headersz = 0; + size_t dataoff = 0; size_t datasz; uint32_t checksum; struct stat s; @@ -1845,7 +1887,9 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, * Do not use sbuf->st_size as it contains size with padding. * We need original image data size, so stat original file. */ - if (stat(params->datafile, &s)) { + if (params->skipcpy) { + s.st_size = 0; + } else if (stat(params->datafile, &s)) { fprintf(stderr, "Could not stat data file %s: %s\n", params->datafile, strerror(errno)); exit(EXIT_FAILURE); @@ -1886,11 +1930,11 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, */ case -1: case 0: - image = image_create_v0(&headersz, params, datasz + 4); + image = image_create_v0(&dataoff, params, datasz + 4); break; case 1: - image = image_create_v1(&headersz, params, ptr, datasz + 4); + image = image_create_v1(&dataoff, params, ptr, datasz + 4); break; default: @@ -1908,12 +1952,12 @@ static void kwbimage_set_header(void *ptr, struct stat *sbuf, int ifd, free(image_cfg); /* Build and add image data checksum */ - checksum = cpu_to_le32(image_checksum32((uint8_t *)ptr + headersz, + checksum = cpu_to_le32(image_checksum32((uint8_t *)ptr + dataoff, datasz)); - memcpy((uint8_t *)ptr + headersz + datasz, &checksum, sizeof(uint32_t)); + memcpy((uint8_t *)ptr + dataoff + datasz, &checksum, sizeof(uint32_t)); /* Finally copy the header into the image area */ - memcpy(ptr, image, headersz); + memcpy(ptr, image, kwbheader_size(image)); free(image); } @@ -1933,9 +1977,9 @@ static void kwbimage_print_header(const void *ptr) printf("BIN Img Size: "); genimg_print_size(opt_hdr_v1_size(ohdr) - 12 - 4 * ohdr->data[0]); - printf("BIN Img Offs: %08x\n", - (unsigned)((uint8_t *)ohdr - (uint8_t *)mhdr) + - 8 + 4 * ohdr->data[0]); + printf("BIN Img Offs: "); + genimg_print_size(((uint8_t *)ohdr - (uint8_t *)mhdr) + + 8 + 4 * ohdr->data[0]); } } @@ -1947,9 +1991,20 @@ static void kwbimage_print_header(const void *ptr) } printf("Data Size: "); - genimg_print_size(mhdr->blocksize - sizeof(uint32_t)); - printf("Load Address: %08x\n", mhdr->destaddr); - printf("Entry Point: %08x\n", mhdr->execaddr); + genimg_print_size(le32_to_cpu(mhdr->blocksize) - sizeof(uint32_t)); + printf("Data Offset: "); + if (mhdr->blockid == IBR_HDR_SATA_ID) + printf("%u Sector%s (LBA)\n", le32_to_cpu(mhdr->srcaddr), + le32_to_cpu(mhdr->srcaddr) != 1 ? "s" : ""); + else + genimg_print_size(le32_to_cpu(mhdr->srcaddr)); + if (mhdr->blockid == IBR_HDR_SPI_ID && le32_to_cpu(mhdr->destaddr) == 0xFFFFFFFF) { + printf("Load Address: XIP\n"); + printf("Execute Offs: %08x\n", le32_to_cpu(mhdr->execaddr)); + } else { + printf("Load Address: %08x\n", le32_to_cpu(mhdr->destaddr)); + printf("Entry Point: %08x\n", le32_to_cpu(mhdr->execaddr)); + } } static int kwbimage_check_image_types(uint8_t type) @@ -2028,23 +2083,9 @@ static int kwbimage_verify_header(unsigned char *ptr, int image_size, /* * For SATA srcaddr is specified in number of sectors. - * The main header is must be stored at sector number 1. - * This expects that sector size is 512 bytes and recalculates - * data offset to bytes relative to the main header. + * This expects that sector size is 512 bytes. */ - if (blockid == IBR_HDR_SATA_ID) { - if (offset < 1) - return -FDT_ERR_BADSTRUCTURE; - offset -= 1; - offset *= 512; - } - - /* - * For SDIO srcaddr is specified in number of sectors. - * This expects that sector size is 512 bytes and recalculates - * data offset to bytes. - */ - if (blockid == IBR_HDR_SDIO_ID) + if (blockid == IBR_HDR_SATA_ID) offset *= 512; /* @@ -2067,6 +2108,8 @@ static int kwbimage_verify_header(unsigned char *ptr, int image_size, return 0; } +static int kwbimage_align_size(int bootfrom, int alloc_len, struct stat s); + static int kwbimage_generate(struct image_tool_params *params, struct image_type_params *tparams) { @@ -2085,7 +2128,9 @@ static int kwbimage_generate(struct image_tool_params *params, exit(EXIT_FAILURE); } - if (stat(params->datafile, &s)) { + if (params->skipcpy) { + s.st_size = 0; + } else if (stat(params->datafile, &s)) { fprintf(stderr, "Could not stat data file %s: %s\n", params->datafile, strerror(errno)); exit(EXIT_FAILURE); @@ -2141,6 +2186,8 @@ static int kwbimage_generate(struct image_tool_params *params, exit(EXIT_FAILURE); } + alloc_len = image_headersz_align(alloc_len, image_get_bootfrom()); + free(image_cfg); hdr = malloc(alloc_len); @@ -2155,6 +2202,22 @@ static int kwbimage_generate(struct image_tool_params *params, tparams->hdr = hdr; /* + * This function should return aligned size of the datafile. + * When skipcpy is set (datafile is skipped) then return value of this + * function is ignored, so we have to put required kwbimage aligning + * into the preallocated header size. + */ + if (params->skipcpy) { + tparams->header_size += kwbimage_align_size(bootfrom, alloc_len, s); + return 0; + } else { + return kwbimage_align_size(bootfrom, alloc_len, s); + } +} + +static int kwbimage_align_size(int bootfrom, int alloc_len, struct stat s) +{ + /* * The resulting image needs to be 4-byte aligned. At least * the Marvell hdrparser tool complains if its unaligned. * After the image data is stored 4-byte checksum. @@ -2182,6 +2245,7 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) struct ext_hdr_v0 *ehdr0; struct bin_hdr_v0 *bhdr0; struct opt_hdr_v1 *ohdr; + int regset_count; int params_count; unsigned offset; int is_v0_ext; @@ -2215,12 +2279,18 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) fprintf(f, "NAND_ECC_MODE %s\n", image_nand_ecc_mode_name(mhdr0->nandeccmode)); if (mhdr->blockid == IBR_HDR_NAND_ID) - fprintf(f, "NAND_PAGE_SIZE 0x%x\n", (unsigned)mhdr->nandpagesize); + fprintf(f, "NAND_PAGE_SIZE 0x%x\n", (unsigned)le16_to_cpu(mhdr->nandpagesize)); - if (version != 0 && mhdr->blockid == IBR_HDR_NAND_ID) - fprintf(f, "NAND_BLKSZ 0x%x\n", (unsigned)mhdr->nandblocksize); + if (mhdr->blockid == IBR_HDR_NAND_ID && (version != 0 || is_v0_ext || mhdr->nandblocksize != 0)) { + if (mhdr->nandblocksize != 0) /* block size explicitly set in 64 kB unit */ + fprintf(f, "NAND_BLKSZ 0x%x\n", (unsigned)mhdr->nandblocksize * 64*1024); + else if (le16_to_cpu(mhdr->nandpagesize) > 512) + fprintf(f, "NAND_BLKSZ 0x10000\n"); /* large page NAND flash = 64 kB block size */ + else + fprintf(f, "NAND_BLKSZ 0x4000\n"); /* small page NAND flash = 16 kB block size */ + } - if (mhdr->blockid == IBR_HDR_NAND_ID && (mhdr->nandbadblklocation != 0 || is_v0_ext)) + if (mhdr->blockid == IBR_HDR_NAND_ID && (version != 0 || is_v0_ext)) fprintf(f, "NAND_BADBLK_LOCATION 0x%x\n", (unsigned)mhdr->nandbadblklocation); if (version == 0 && mhdr->blockid == IBR_HDR_SATA_ID) @@ -2266,18 +2336,20 @@ static int kwbimage_generate_config(void *ptr, struct image_tool_params *params) cur_idx++; } else if (ohdr->headertype == OPT_HDR_V1_REGISTER_TYPE) { regset_hdr = (struct register_set_hdr_v1 *)ohdr; - for (i = 0; - i < opt_hdr_v1_size(ohdr) - sizeof(struct opt_hdr_v1) - - sizeof(regset_hdr->data[0].last_entry); - i++) + if (opt_hdr_v1_size(ohdr) > sizeof(*ohdr)) + regset_count = (opt_hdr_v1_size(ohdr) - sizeof(*ohdr)) / + sizeof(regset_hdr->data[0].entry); + else + regset_count = 0; + for (i = 0; i < regset_count; i++) fprintf(f, "DATA 0x%08x 0x%08x\n", le32_to_cpu(regset_hdr->data[i].entry.address), le32_to_cpu(regset_hdr->data[i].entry.value)); - if (opt_hdr_v1_size(ohdr) - sizeof(struct opt_hdr_v1) >= - sizeof(regset_hdr->data[0].last_entry)) { - if (regset_hdr->data[0].last_entry.delay) + if (regset_count > 0) { + if (regset_hdr->data[regset_count-1].last_entry.delay != + REGISTER_SET_HDR_OPT_DELAY_SDRAM_SETUP) fprintf(f, "DATA_DELAY %u\n", - (unsigned)regset_hdr->data[0].last_entry.delay); + (unsigned)regset_hdr->data[regset_count-1].last_entry.delay); else fprintf(f, "DATA_DELAY SDRAM_SETUP\n"); } @@ -2403,12 +2475,7 @@ static int kwbimage_extract_subimage(void *ptr, struct image_tool_params *params /* Extract data image when -p is not specified or when '-p 0' is specified */ offset = le32_to_cpu(mhdr->srcaddr); - if (mhdr->blockid == IBR_HDR_SATA_ID) { - offset -= 1; - offset *= 512; - } - - if (mhdr->blockid == IBR_HDR_SDIO_ID) + if (mhdr->blockid == IBR_HDR_SATA_ID) offset *= 512; if (mhdr->blockid == IBR_HDR_PEX_ID && offset == 0xFFFFFFFF) @@ -2455,9 +2522,6 @@ static int kwbimage_extract_subimage(void *ptr, struct image_tool_params *params return imagetool_save_subimage(params->outfile, image, size); } -/* - * Report Error if xflag is set in addition to default - */ static int kwbimage_check_params(struct image_tool_params *params) { if (!params->lflag && !params->iflag && !params->pflag && @@ -2468,10 +2532,9 @@ static int kwbimage_check_params(struct image_tool_params *params) return 1; } - return (params->dflag && (params->fflag || params->lflag)) || - (params->fflag && (params->dflag || params->lflag)) || - (params->lflag && (params->dflag || params->fflag)) || - (params->xflag); + return (params->dflag && (params->fflag || params->lflag || params->skipcpy)) || + (params->fflag) || + (params->lflag && (params->dflag || params->fflag)); } /* diff --git a/tools/kwboot.c b/tools/kwboot.c index da4fe32da2..7c666486f3 100644 --- a/tools/kwboot.c +++ b/tools/kwboot.c @@ -15,6 +15,12 @@ * Processor, and High-Definition Video Decoder: Functional Specifications" * August 3, 2011. Chapter 5 "BootROM Firmware" * https://web.archive.org/web/20120130172443/https://www.marvell.com/application-processors/armada-500/assets/Armada-510-Functional-Spec.pdf + * - "88F6665, 88F6660, 88F6658, 88F6655, 88F6655F, 88F6650, 88F6650F, 88F6610, + * and 88F6610F Avanta LP Family Integrated Single/Dual CPU Ecosystem for + * Gateway (GW), Home Gateway Unit (HGU), and Single Family Unit (SFU) + * Functional Specifications" Doc. No. MV-S108952-00, Rev. A. November 7, 2013. + * Chapter 7 "Boot Flow" + * CONFIDENTIAL, no public documentation available * - "88F6710, 88F6707, and 88F6W11: ARMADA(R) 370 SoC: Functional Specifications" * May 26, 2014. Chapter 6 "BootROM Firmware". * https://web.archive.org/web/20140617183701/https://www.marvell.com/embedded-processors/armada-300/assets/ARMADA370-FunctionalSpec-datasheet.pdf @@ -22,6 +28,15 @@ * Multi-Core ARMv7 Based SoC Processors: Functional Specifications" * May 29, 2014. Chapter 6 "BootROM Firmware". * https://web.archive.org/web/20180829171131/https://www.marvell.com/embedded-processors/armada-xp/assets/ARMADA-XP-Functional-SpecDatasheet.pdf + * - "BobCat2 Control and Management Subsystem Functional Specifications" + * Doc. No. MV-S109400-00, Rev. A. December 4, 2014. + * Chapter 1.6 BootROM Firmware + * CONFIDENTIAL, no public documentation available + * - "AlleyCat3 and PONCat3 Highly Integrated 1/10 Gigabit Ethernet Switch + * Control and Management Subsystem: Functional Specifications" + * Doc. No. MV-S109693-00, Rev. A. May 20, 2014. + * Chapter 1.6 BootROM Firmware + * CONFIDENTIAL, no public documentation available * - "ARMADA(R) 375 Value-Performance Dual Core CPU System on Chip: Functional * Specifications" Doc. No. MV-S109377-00, Rev. A. September 18, 2013. * Chapter 7 "Boot Sequence" @@ -35,6 +50,72 @@ * System on Chip Functional Specifications" Doc. No. MV-S109896-00, Rev. B. * December 22, 2015. Chapter 7 "Boot Flow" * CONFIDENTIAL, no public documentation available + * - "Marvell boot image parser", Marvell U-Boot 2013.01, version 18.06. September 17, 2015. + * https://github.com/MarvellEmbeddedProcessors/u-boot-marvell/blob/u-boot-2013.01-armada-18.06/tools/marvell/doimage_mv/hdrparser.c + * - "Marvell doimage Tool", Marvell U-Boot 2013.01, version 18.06. August 30, 2015. + * https://github.com/MarvellEmbeddedProcessors/u-boot-marvell/blob/u-boot-2013.01-armada-18.06/tools/marvell/doimage_mv/doimage.c + * + * Storage location / offset of different image types: + * - IBR_HDR_SPI_ID (0x5A): + * SPI image can be stored at any 2 MB aligned offset in the first 16 MB of + * SPI-NOR or parallel-NOR. Despite the type name it really can be stored on + * parallel-NOR and cannot be stored on other SPI devices, like SPI-NAND. + * So it should have been named NOR image, not SPI image. This image type + * supports XIP - Execute In Place directly from NOR memory. + * + * - IBR_HDR_NAND_ID (0x8B): + * NAND image can be stored either at any 2 MB aligned offset in the first + * 16 MB of SPI-NAND or at any blocksize aligned offset in the first 64 MB + * of parallel-NAND. + * + * - IBR_HDR_PEX_ID (0x9C): + * PEX image is used for booting from PCI Express device. Source address + * stored in image is ignored by BootROM. It is not the BootROM who parses + * or loads data part of the PEX image. BootROM just configures SoC to the + * PCIe endpoint mode and let the PCIe device on the other end of the PCIe + * link (which must be in Root Complex mode) to load kwbimage into SoC's + * memory and tell BootROM physical address. + * + * - IBR_HDR_UART_ID (0x69): + * UART image can be transfered via xmodem protocol over first UART. + * + * - IBR_HDR_I2C_ID (0x4D): + * It is unknown for what kind of storage is used this image. It is not + * specified in any document from References section. + * + * - IBR_HDR_SATA_ID (0x78): + * SATA image can be stored at sector 1 (after the MBR table), sector 34 + * (after the GPT table) or at any next sector which is aligned to 2 MB and + * is in the first 16 MB of SATA disk. Note that source address in SATA image + * is stored in sector unit and not in bytes like for any other images. + * Unfortunately sector size is disk specific, in most cases it is 512 bytes + * but there are also Native 4K SATA disks which have 4096 bytes long sectors. + * + * - IBR_HDR_SDIO_ID (0xAE): + * SDIO image can be stored on different medias: + * - SD(SC) card + * - SDHC/SDXC card + * - eMMC HW boot partition + * - eMMC user data partition / MMC card + * It cannot be stored on SDIO card despite the image name. + * + * For SD(SC)/SDHC/SDXC cards, image can be stored at the same locations as + * the SATA image (sector 1, sector 34 or any 2 MB aligned sector) but within + * the first 64 MB. SDHC and SDXC cards have fixed 512 bytes long sector size. + * Old SD(SC) cards unfortunately can have also different sector sizes, mostly + * 1024 bytes long sector sizes and also can be changed at runtime. + * + * For MMC-compatible devices, image can be stored at offset 0 or at offset + * 2 MB. If MMC device supports HW boot partitions then image must be stored + * on the HW partition as is configured in the EXT_CSC register (it can be + * either boot or user data). + * + * Note that source address for SDIO image is stored in byte unit, like for + * any other images (except SATA). Marvell Functional Specifications for + * A38x and A39x SoCs say that source address is in sector units, but this + * is purely incorrect information. A385 BootROM really expects source address + * for SDIO images in bytes and also Marvell tools generate SDIO image with + * source address in byte units. */ #include "kwbimage.h" @@ -1699,6 +1780,47 @@ kwboot_img_is_secure(void *img) return 0; } +static int +kwboot_img_has_ddr_init(void *img) +{ + const struct register_set_hdr_v1 *rhdr; + const struct main_hdr_v0 *hdr0; + struct opt_hdr_v1 *ohdr; + u32 ohdrsz; + int last; + + /* + * kwbimage v0 image headers contain DDR init code either in + * extension header or in binary code header. + */ + if (kwbimage_version(img) == 0) { + hdr0 = img; + return hdr0->ext || hdr0->bin; + } + + /* + * kwbimage v1 image headers contain DDR init code either in binary + * code header or in a register set list header with SDRAM_SETUP. + */ + for_each_opt_hdr_v1 (ohdr, img) { + if (ohdr->headertype == OPT_HDR_V1_BINARY_TYPE) + return 1; + if (ohdr->headertype == OPT_HDR_V1_REGISTER_TYPE) { + rhdr = (const struct register_set_hdr_v1 *)ohdr; + ohdrsz = opt_hdr_v1_size(ohdr); + if (ohdrsz >= sizeof(*ohdr) + sizeof(rhdr->data[0].last_entry)) { + ohdrsz -= sizeof(*ohdr) + sizeof(rhdr->data[0].last_entry); + last = ohdrsz / sizeof(rhdr->data[0].entry); + if (rhdr->data[last].last_entry.delay == + REGISTER_SET_HDR_OPT_DELAY_SDRAM_SETUP) + return 1; + } + } + } + + return 0; +} + static void * kwboot_img_grow_data_right(void *img, size_t *size, size_t grow) { @@ -1854,10 +1976,26 @@ _inject_baudrate_change_code(void *img, size_t *size, int for_data, } } +static const char * +kwboot_img_type(uint8_t blockid) +{ + switch (blockid) { + case IBR_HDR_I2C_ID: return "I2C"; + case IBR_HDR_SPI_ID: return "SPI"; + case IBR_HDR_NAND_ID: return "NAND"; + case IBR_HDR_SATA_ID: return "SATA"; + case IBR_HDR_PEX_ID: return "PEX"; + case IBR_HDR_UART_ID: return "UART"; + case IBR_HDR_SDIO_ID: return "SDIO"; + default: return "unknown"; + } +} + static int kwboot_img_patch(void *img, size_t *size, int baudrate) { struct main_hdr_v1 *hdr; + struct opt_hdr_v1 *ohdr; uint32_t srcaddr; uint8_t csum; size_t hdrsz; @@ -1866,8 +2004,10 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) hdr = img; - if (*size < sizeof(struct main_hdr_v1)) + if (*size < sizeof(struct main_hdr_v1)) { + fprintf(stderr, "Invalid image header size\n"); goto err; + } image_ver = kwbimage_version(img); if (image_ver != 0 && image_ver != 1) { @@ -1877,24 +2017,23 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) hdrsz = kwbheader_size(hdr); - if (*size < hdrsz) + if (*size < hdrsz) { + fprintf(stderr, "Invalid image header size\n"); goto err; + } + + kwboot_printv("Detected kwbimage v%d with %s boot signature\n", image_ver, kwboot_img_type(hdr->blockid)); csum = kwboot_hdr_csum8(hdr) - hdr->checksum; - if (csum != hdr->checksum) + if (csum != hdr->checksum) { + fprintf(stderr, "Image has invalid header checksum stored in image header\n"); goto err; + } srcaddr = le32_to_cpu(hdr->srcaddr); switch (hdr->blockid) { case IBR_HDR_SATA_ID: - if (srcaddr < 1) - goto err; - - hdr->srcaddr = cpu_to_le32((srcaddr - 1) * 512); - break; - - case IBR_HDR_SDIO_ID: hdr->srcaddr = cpu_to_le32(srcaddr * 512); break; @@ -1906,18 +2045,48 @@ kwboot_img_patch(void *img, size_t *size, int baudrate) case IBR_HDR_SPI_ID: if (hdr->destaddr == cpu_to_le32(0xFFFFFFFF)) { kwboot_printv("Patching destination and execution addresses from SPI/NOR XIP area to DDR area 0x00800000\n"); - hdr->destaddr = cpu_to_le32(0x00800000); - hdr->execaddr = cpu_to_le32(0x00800000); + hdr->destaddr = cpu_to_le32(0x00800000 + le32_to_cpu(hdr->srcaddr)); + hdr->execaddr = cpu_to_le32(0x00800000 + le32_to_cpu(hdr->execaddr)); } break; } - if (hdrsz > le32_to_cpu(hdr->srcaddr) || - *size < le32_to_cpu(hdr->srcaddr) + le32_to_cpu(hdr->blocksize)) + if (hdrsz > le32_to_cpu(hdr->srcaddr)) { + fprintf(stderr, "Image has invalid data offset stored in image header\n"); goto err; + } - if (kwboot_img_csum32(img) != *kwboot_img_csum32_ptr(img)) + if (*size < le32_to_cpu(hdr->srcaddr) + le32_to_cpu(hdr->blocksize)) { + fprintf(stderr, "Image has invalid data size stored in image header\n"); goto err; + } + + for_each_opt_hdr_v1 (ohdr, hdr) { + if (!opt_hdr_v1_valid_size(ohdr, (const uint8_t *)hdr + hdrsz)) { + fprintf(stderr, "Invalid optional image header\n"); + goto err; + } + } + + /* + * The 32-bit data checksum is optional for UART image. If it is not + * present (checksum detected as invalid) then grow data part of the + * image for the checksum, so it can be inserted there. + */ + if (kwboot_img_csum32(img) != *kwboot_img_csum32_ptr(img)) { + if (hdr->blockid != IBR_HDR_UART_ID) { + fprintf(stderr, "Image has invalid data checksum\n"); + goto err; + } + kwboot_img_grow_data_right(img, size, sizeof(uint32_t)); + } + + if (!kwboot_img_has_ddr_init(img) && + (le32_to_cpu(hdr->destaddr) < 0x40000000 || + le32_to_cpu(hdr->destaddr) + le32_to_cpu(hdr->blocksize) > 0x40034000)) { + fprintf(stderr, "Image does not contain DDR init code needed for UART booting\n"); + goto err; + } is_secure = kwboot_img_is_secure(img); @@ -2182,6 +2351,7 @@ main(int argc, char **argv) KWBOOT_XM_BLKSZ + sizeof(kwboot_baud_code) + sizeof(kwboot_baud_code_data_jump) + + sizeof(uint32_t) + KWBOOT_XM_BLKSZ; if (imgpath) { diff --git a/tools/mkimage.c b/tools/mkimage.c index af7b0e09b3..a92d9d5ca5 100644 --- a/tools/mkimage.c +++ b/tools/mkimage.c @@ -599,7 +599,12 @@ int main(int argc, char **argv) exit (retval); } - if ((params.type != IH_TYPE_MULTI) && (params.type != IH_TYPE_SCRIPT)) { + if (!params.skipcpy && params.type != IH_TYPE_MULTI && params.type != IH_TYPE_SCRIPT) { + if (!params.datafile) { + fprintf(stderr, "%s: Option -d with image data file was not specified\n", + params.cmdname); + exit(EXIT_FAILURE); + } dfd = open(params.datafile, O_RDONLY | O_BINARY); if (dfd < 0) { fprintf(stderr, "%s: Can't open %s: %s\n", @@ -860,7 +865,9 @@ copy_file (int ifd, const char *datafile, int pad) exit (EXIT_FAILURE); } - if (params.xflag) { + if (params.xflag && + (((params.type > IH_TYPE_INVALID) && (params.type < IH_TYPE_FLATDT)) || + (params.type == IH_TYPE_KERNEL_NOLOAD) || (params.type == IH_TYPE_FIRMWARE_IVT))) { unsigned char *p = NULL; /* * XIP: do not append the struct legacy_img_hdr at the diff --git a/tools/patman/__init__.py b/tools/patman/__init__.py index 1b98ec7fee..08eeffdf6d 100644 --- a/tools/patman/__init__.py +++ b/tools/patman/__init__.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0+ -__all__ = ['checkpatch', 'command', 'commit', 'control', 'cros_subprocess', - 'func_test', 'get_maintainer', 'gitutil', '__main__', 'patchstream', - 'project', 'series', 'setup', 'settings', 'terminal', - 'test_checkpatch', 'test_util', 'tools', 'tout'] +__all__ = ['checkpatch', 'commit', 'control', 'func_test', 'get_maintainer', + 'gitutil', '__main__', 'patchstream', 'project', 'series', + 'settings','setup', 'status', 'test_checkpatch', 'test_settings'] diff --git a/tools/patman/__main__.py b/tools/patman/__main__.py index 749e6348b6..48ffbc8ead 100755 --- a/tools/patman/__main__.py +++ b/tools/patman/__main__.py @@ -24,10 +24,9 @@ from patman import func_test from patman import gitutil from patman import project from patman import settings -from patman import terminal -from patman import test_util -from patman import test_checkpatch -from patman import tools +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools epilog = '''Create patches from commits in a branch, check them and email them as specified by tags you place in the commits. Use -n to do a dry run first.''' @@ -146,11 +145,12 @@ if not args.debug: # Run our meagre tests if args.cmd == 'test': from patman import func_test + from patman import test_checkpatch result = test_util.run_test_suites( 'patman', False, False, False, None, None, None, [test_checkpatch.TestPatch, func_test.TestFunctional, - 'gitutil', 'settings', 'terminal']) + 'gitutil', 'settings']) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index d1b902dd96..e03cac115e 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -3,13 +3,14 @@ # import collections +import concurrent.futures import os import re import sys -from patman import command from patman import gitutil -from patman import terminal +from u_boot_pylib import command +from u_boot_pylib import terminal EMACS_PREFIX = r'(?:[0-9]{4}.*\.patch:[0-9]+: )?' TYPE_NAME = r'([A-Z_]+:)?' @@ -244,26 +245,31 @@ def check_patches(verbose, args, use_tree): error_count, warning_count, check_count = 0, 0, 0 col = terminal.Color() - for fname in args: - result = check_patch(fname, verbose, use_tree=use_tree) - if not result.ok: - error_count += result.errors - warning_count += result.warnings - check_count += result.checks - print('%d errors, %d warnings, %d checks for %s:' % (result.errors, - result.warnings, result.checks, col.build(col.BLUE, fname))) - if (len(result.problems) != result.errors + result.warnings + - result.checks): - print("Internal error: some problems lost") - # Python seems to get confused by this - # pylint: disable=E1133 - for item in result.problems: - sys.stderr.write( - get_warning_msg(col, item.get('type', '<unknown>'), - item.get('file', '<unknown>'), - item.get('line', 0), item.get('msg', 'message'))) - print - #print(stdout) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + futures = [] + for fname in args: + f = executor.submit(check_patch, fname, verbose, use_tree=use_tree) + futures.append(f) + + for fname, f in zip(args, futures): + result = f.result() + if not result.ok: + error_count += result.errors + warning_count += result.warnings + check_count += result.checks + print('%d errors, %d warnings, %d checks for %s:' % (result.errors, + result.warnings, result.checks, col.build(col.BLUE, fname))) + if (len(result.problems) != result.errors + result.warnings + + result.checks): + print("Internal error: some problems lost") + # Python seems to get confused by this + # pylint: disable=E1133 + for item in result.problems: + sys.stderr.write( + get_warning_msg(col, item.get('type', '<unknown>'), + item.get('file', '<unknown>'), + item.get('line', 0), item.get('msg', 'message'))) + print if error_count or warning_count or check_count: str = 'checkpatch.pl found %d error(s), %d warning(s), %d checks(s)' color = col.GREEN diff --git a/tools/patman/control.py b/tools/patman/control.py index 38e98dab84..916ddf8fcf 100644 --- a/tools/patman/control.py +++ b/tools/patman/control.py @@ -14,7 +14,7 @@ import sys from patman import checkpatch from patman import gitutil from patman import patchstream -from patman import terminal +from u_boot_pylib import terminal def setup(): """Do required setup before doing anything""" @@ -85,7 +85,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree): # Do a few checks on the series series.DoChecks() - # Check the patches, and run them through 'git am' just to be sure + # Check the patches if run_checkpatch: ok = checkpatch.check_patches(verbose, patch_files, use_tree) else: diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index c25a47bdeb..42ac4ed77b 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -23,9 +23,9 @@ from patman import patchstream from patman.patchstream import PatchStream from patman.series import Series from patman import settings -from patman import terminal -from patman import tools -from patman.test_util import capture_sys_output +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib.test_util import capture_sys_output import pygit2 from patman import status @@ -240,6 +240,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Change log missing for v3', next(lines)) self.assertEqual('Change log for unknown version v4', next(lines)) self.assertEqual("Alias 'pci' not found", next(lines)) + while next(lines) != 'Cc processing complete': + pass self.assertIn('Dry run', next(lines)) self.assertEqual('', next(lines)) self.assertIn('Send a total of %d patches' % count, next(lines)) diff --git a/tools/patman/get_maintainer.py b/tools/patman/get_maintainer.py index f7011be1e4..8df3d124ba 100644 --- a/tools/patman/get_maintainer.py +++ b/tools/patman/get_maintainer.py @@ -7,8 +7,8 @@ import os import shlex import shutil -from patman import command from patman import gitutil +from u_boot_pylib import command def find_get_maintainer(script_file_name): diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 5e742102c2..6700057359 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -5,9 +5,9 @@ import os import sys -from patman import command from patman import settings -from patman import terminal +from u_boot_pylib import command +from u_boot_pylib import terminal # True to use --no-decorate - we check this in setup() use_no_decorate = True diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index fb6a6036f3..f91669a940 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -14,10 +14,10 @@ import queue import shutil import tempfile -from patman import command from patman import commit from patman import gitutil from patman.series import Series +from u_boot_pylib import command # Tags that we detect and remove RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' diff --git a/tools/patman/patman.rst b/tools/patman/patman.rst index 6113962fb4..038b651ee8 100644 --- a/tools/patman/patman.rst +++ b/tools/patman/patman.rst @@ -41,6 +41,18 @@ In Linux and U-Boot this will also call get_maintainer.pl on each of your patches automatically (unless you use -m to disable this). +Installation +------------ + +You can install patman using:: + + pip install patch-manager + +The name is chosen since patman conflicts with an existing package. + +If you are using patman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/patman/patman`. + How to use this tool -------------------- diff --git a/tools/patman/pyproject.toml b/tools/patman/pyproject.toml new file mode 100644 index 0000000000..c5dc7c7e27 --- /dev/null +++ b/tools/patman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "patch-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib"] +description = "Patman patch manager" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/patman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +patman = "patman.__main__:run_patman" + +[tool.setuptools.package-data] +patman = ["*.rst"] diff --git a/tools/patman/series.py b/tools/patman/series.py index 2eeeef71dc..6866e1dbd0 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -5,14 +5,17 @@ from __future__ import print_function import collections +import concurrent.futures import itertools import os +import sys +import time from patman import get_maintainer from patman import gitutil from patman import settings -from patman import terminal -from patman import tools +from u_boot_pylib import terminal +from u_boot_pylib import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', @@ -234,6 +237,49 @@ class Series(dict): str = 'Change log exists, but no version is set' print(col.build(col.RED, str)) + def GetCcForCommit(self, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, + all_skips): + """Get the email CCs to use with a particular commit + + Uses subject tags and get_maintainers.pl script to find people to cc + on a patch + + Args: + commit (Commit): Commit to process + process_tags (bool): Process tags as if they were aliases + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: + True/False to call the get_maintainers to CC maintainers + List of maintainers to include (for testing) + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl + script (or compatible). + all_skips (set of str): Updated to include the set of bouncing email + addresses that were dropped from the output. This is essentially + a return value from this function. + + Returns: + list of str: List of email addresses to cc + """ + cc = [] + if process_tags: + cc += gitutil.build_email_list(commit.tags, + warn_on_error=warn_on_error) + cc += gitutil.build_email_list(commit.cc_list, + warn_on_error=warn_on_error) + if type(add_maintainers) == type(cc): + cc += add_maintainers + elif add_maintainers: + cc += get_maintainer.get_maintainer(get_maintainer_script, + commit.patch) + all_skips |= set(cc) & set(settings.bounces) + cc = list(set(cc) - set(settings.bounces)) + if limit is not None: + cc = cc[:limit] + return cc + def MakeCcFile(self, process_tags, cover_fname, warn_on_error, add_maintainers, limit, get_maintainer_script): """Make a cc file for us to use for per-commit Cc automation @@ -241,15 +287,15 @@ class Series(dict): Also stores in self._generated_cc to make ShowActions() faster. Args: - process_tags: Process tags as if they were aliases - cover_fname: If non-None the name of the cover letter. - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - add_maintainers: Either: + process_tags (bool): Process tags as if they were aliases + cover_fname (str): If non-None the name of the cover letter. + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: True/False to call the get_maintainers to CC maintainers List of maintainers to include (for testing) - limit: Limit the length of the Cc list (None if no limit) - get_maintainer_script: The file name of the get_maintainer.pl + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl script (or compatible). Return: Filename of temp file created @@ -259,28 +305,42 @@ class Series(dict): fname = '/tmp/patman.%d' % os.getpid() fd = open(fname, 'w', encoding='utf-8') all_ccs = [] + all_skips = set() + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + for i, commit in enumerate(self.commits): + commit.seq = i + commit.future = executor.submit( + self.GetCcForCommit, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, all_skips) + + # Show progress any commits that are taking forever + lastlen = 0 + while True: + left = [commit for commit in self.commits + if not commit.future.done()] + if not left: + break + names = ', '.join(f'{c.seq + 1}:{c.subject}' + for c in left[:2]) + out = f'\r{len(left)} remaining: {names}'[:79] + spaces = ' ' * (lastlen - len(out)) + if lastlen: # Don't print anything the first time + print(out, spaces, end='') + sys.stdout.flush() + lastlen = len(out) + time.sleep(.25) + print(f'\rdone{" " * lastlen}\r', end='') + print('Cc processing complete') + for commit in self.commits: - cc = [] - if process_tags: - cc += gitutil.build_email_list(commit.tags, - warn_on_error=warn_on_error) - cc += gitutil.build_email_list(commit.cc_list, - warn_on_error=warn_on_error) - if type(add_maintainers) == type(cc): - cc += add_maintainers - elif add_maintainers: - - cc += get_maintainer.get_maintainer(get_maintainer_script, - commit.patch) - for x in set(cc) & set(settings.bounces): - print(col.build(col.YELLOW, 'Skipping "%s"' % x)) - cc = list(set(cc) - set(settings.bounces)) - if limit is not None: - cc = cc[:limit] + cc = commit.future.result() all_ccs += cc print(commit.patch, '\0'.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc + for x in sorted(all_skips): + print(col.build(col.YELLOW, f'Skipping "{x}"')) + if cover_fname: cover_cc = gitutil.build_email_list(self.get('cover_cc', '')) cover_cc = list(set(cover_cc + all_ccs)) diff --git a/tools/patman/status.py b/tools/patman/status.py index 47ed6d61d4..5fb436e08f 100644 --- a/tools/patman/status.py +++ b/tools/patman/status.py @@ -18,8 +18,8 @@ import requests from patman import patchstream from patman.patchstream import PatchStream -from patman import terminal -from patman import tout +from u_boot_pylib import terminal +from u_boot_pylib import tout # Patches which are part of a multi-patch series are shown with a prefix like # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last diff --git a/tools/patman/test_checkpatch.py b/tools/patman/test_checkpatch.py index 4c2ab6e590..a8bb364e42 100644 --- a/tools/patman/test_checkpatch.py +++ b/tools/patman/test_checkpatch.py @@ -452,6 +452,12 @@ index 0000000..2234c87 self.check_strl("cat"); self.check_strl("cpy"); + def test_schema(self): + """Check for uses of strn(cat|cpy)""" + pm = PatchMaker() + pm.add_line('arch/sandbox/dts/sandbox.dtsi', '\tu-boot,dm-pre-proper;') + self.check_single_message(pm, 'PRE_SCHEMA', 'error') + if __name__ == "__main__": unittest.main() gitutil.RunTests() diff --git a/tools/patman/test_settings.py b/tools/patman/test_settings.py index c768a2fc64..06b7cbc3ab 100644 --- a/tools/patman/test_settings.py +++ b/tools/patman/test_settings.py @@ -10,7 +10,7 @@ import sys import tempfile from patman import settings -from patman import tools +from u_boot_pylib import tools @contextlib.contextmanager diff --git a/tools/rmboard.py b/tools/rmboard.py index ae25632127..0c56b149e0 100755 --- a/tools/rmboard.py +++ b/tools/rmboard.py @@ -28,7 +28,7 @@ import os import re import sys -from patman import command +from u_boot_pylib import command def rm_kconfig_include(path): """Remove a path from Kconfig files diff --git a/tools/u_boot_pylib/LICENSE b/tools/u_boot_pylib/LICENSE new file mode 100644 index 0000000000..d159169d10 --- /dev/null +++ b/tools/u_boot_pylib/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tools/u_boot_pylib/README.rst b/tools/u_boot_pylib/README.rst new file mode 100644 index 0000000000..93858f5571 --- /dev/null +++ b/tools/u_boot_pylib/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +# U-Boot Python Library +===================== + +This is a Python library used by various U-Boot tools, including patman, +buildman and binman. + +The module can be installed with pip:: + + pip install u_boot_pylib + +or via setup.py:: + + ./setup.py install [--user] diff --git a/tools/u_boot_pylib/__init__.py b/tools/u_boot_pylib/__init__.py new file mode 100644 index 0000000000..63c88e85ec --- /dev/null +++ b/tools/u_boot_pylib/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ + +__all__ = ['command', 'cros_subprocess','terminal', 'test_util', 'tools', + 'tout'] diff --git a/tools/u_boot_pylib/__main__.py b/tools/u_boot_pylib/__main__.py new file mode 100755 index 0000000000..8f98d7bd9f --- /dev/null +++ b/tools/u_boot_pylib/__main__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2023 Google LLC +# + +import os +import sys + +if __name__ == "__main__": + # Allow 'from u_boot_pylib import xxx to work' + our_path = os.path.dirname(os.path.realpath(__file__)) + sys.path.append(os.path.join(our_path, '..')) + + # Run tests + from u_boot_pylib import terminal + from u_boot_pylib import test_util + + result = test_util.run_test_suites( + 'u_boot_pylib', False, False, False, None, None, None, + ['terminal']) + + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/command.py b/tools/u_boot_pylib/command.py index 92c453b5c1..9bbfc5bdd8 100644 --- a/tools/patman/command.py +++ b/tools/u_boot_pylib/command.py @@ -4,7 +4,7 @@ import os -from patman import cros_subprocess +from u_boot_pylib import cros_subprocess """Shell command ease-ups for Python.""" diff --git a/tools/patman/cros_subprocess.py b/tools/u_boot_pylib/cros_subprocess.py index cd614f38a6..cd614f38a6 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/u_boot_pylib/cros_subprocess.py diff --git a/tools/u_boot_pylib/pyproject.toml b/tools/u_boot_pylib/pyproject.toml new file mode 100644 index 0000000000..3f33caf6f8 --- /dev/null +++ b/tools/u_boot_pylib/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "u_boot_pylib" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +description = "U-Boot python library" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" diff --git a/tools/patman/terminal.py b/tools/u_boot_pylib/terminal.py index 40d79f8ac0..40d79f8ac0 100644 --- a/tools/patman/terminal.py +++ b/tools/u_boot_pylib/terminal.py diff --git a/tools/patman/test_util.py b/tools/u_boot_pylib/test_util.py index 0f6d1aa902..e7564e10c9 100644 --- a/tools/patman/test_util.py +++ b/tools/u_boot_pylib/test_util.py @@ -11,15 +11,14 @@ import os import sys import unittest -from patman import command +from u_boot_pylib import command from io import StringIO -buffer_outputs = True use_concurrent = True try: - from concurrencytest.concurrencytest import ConcurrentTestSuite - from concurrencytest.concurrencytest import fork_for_tests + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests except: use_concurrent = False @@ -120,7 +119,6 @@ class FullTextTestResult(unittest.TextTestResult): 0: Print nothing 1: Print a dot per test 2: Print test names - 3: Print test names, and buffered outputs for failing tests """ def __init__(self, stream, descriptions, verbosity): self.verbosity = verbosity @@ -140,39 +138,12 @@ class FullTextTestResult(unittest.TextTestResult): self.printErrorList('XFAIL', self.expectedFailures) self.printErrorList('XPASS', unexpected_successes) - def addError(self, test, err): - """Called when an error has occurred.""" - super().addError(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addFailure(self, test, err): - """Called when a test has failed.""" - super().addFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSubTest(self, test, subtest, err): - """Called at the end of a subtest.""" - super().addSubTest(test, subtest, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSuccess(self, test): - """Called when a test has completed successfully""" - super().addSuccess(test) - # Don't print stdout/stderr for successful tests - self._mirrorOutput = False - def addSkip(self, test, reason): """Called when a test is skipped.""" # Add empty line to keep spacing consistent with other results if not reason.endswith('\n'): reason += '\n' super().addSkip(test, reason) - self._mirrorOutput &= self.verbosity >= 3 - - def addExpectedFailure(self, test, err): - """Called when an expected failure/error occurred.""" - super().addExpectedFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, @@ -208,14 +179,12 @@ def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, runner = unittest.TextTestRunner( stream=sys.stdout, verbosity=(1 if verbosity is None else verbosity), - buffer=False if test_name else buffer_outputs, resultclass=FullTextTestResult, ) if use_concurrent and processes != 1: suite = ConcurrentTestSuite(suite, - fork_for_tests(processes or multiprocessing.cpu_count(), - buffer=False if test_name else buffer_outputs)) + fork_for_tests(processes or multiprocessing.cpu_count())) for module in class_and_module_list: if isinstance(module, str) and (not test_name or test_name == module): diff --git a/tools/patman/tools.py b/tools/u_boot_pylib/tools.py index 2ac814d476..187725b501 100644 --- a/tools/patman/tools.py +++ b/tools/u_boot_pylib/tools.py @@ -11,8 +11,8 @@ import sys import tempfile import urllib.request -from patman import command -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tout # Output directly (generally this is temporary) outdir = None diff --git a/tools/patman/tout.py b/tools/u_boot_pylib/tout.py index ff0fd92afc..6bd2806f88 100644 --- a/tools/patman/tout.py +++ b/tools/u_boot_pylib/tout.py @@ -6,7 +6,7 @@ import sys -from patman import terminal +from u_boot_pylib import terminal # Output verbosity levels that we support ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(6) diff --git a/tools/u_boot_pylib/u_boot_pylib b/tools/u_boot_pylib/u_boot_pylib new file mode 120000 index 0000000000..5a427d1942 --- /dev/null +++ b/tools/u_boot_pylib/u_boot_pylib @@ -0,0 +1 @@ +__main__.py
\ No newline at end of file |