diff options
Diffstat (limited to 'tools/buildman')
| -rw-r--r-- | tools/buildman/README | 40 | ||||
| -rw-r--r-- | tools/buildman/board.py | 289 | ||||
| -rw-r--r-- | tools/buildman/boards.py | 752 | ||||
| -rw-r--r-- | tools/buildman/builder.py | 140 | ||||
| -rw-r--r-- | tools/buildman/builderthread.py | 6 | ||||
| -rw-r--r-- | tools/buildman/cmdline.py | 7 | ||||
| -rw-r--r-- | tools/buildman/control.py | 56 | ||||
| -rw-r--r-- | tools/buildman/func_test.py | 59 | ||||
| -rw-r--r-- | tools/buildman/test.py | 45 | ||||
| -rw-r--r-- | tools/buildman/toolchain.py | 8 |
10 files changed, 954 insertions, 448 deletions
diff --git a/tools/buildman/README b/tools/buildman/README index 49438cb909..a8357a804b 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -128,10 +128,10 @@ Selecting which boards to build =============================== Buildman lets you build all boards, or a subset. Specify the subset by passing -command-line arguments that list the desired board name, architecture name, -SOC name, or anything else in the boards.cfg file. Multiple arguments are -allowed. Each argument will be interpreted as a regular expression, so -behaviour is a superset of exact or substring matching. Examples are: +command-line arguments that list the desired build target, architecture, +CPU, board name, vendor, SoC or options. Multiple arguments are allowed. Each +argument will be interpreted as a regular expression, so behaviour is a superset +of exact or substring matching. Examples are: * 'tegra20' All boards with a Tegra20 SoC * 'tegra' All boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...) @@ -1054,7 +1054,6 @@ between one commit and the next. For example: $ buildman -b squash brppt1 -sU -boards.cfg is up to date. Nothing to do. Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) 01: Migrate bootlimit to Kconfig 02: Squashed commit of the following: @@ -1092,6 +1091,21 @@ This will write the full build into /tmp/build including object files. You must specify the output directory with -o when using -w. +Support for IDEs (Integrated Development Environments) +====================================================== + +Normally buildman summarises the output and shows information indicating the +meaning of each line of output. For example a '+' symbol appears at the start of +each error line. Also, buildman prints information about what it is about to do, +along with a summary at the end. + +When using buildman from an IDE, it is helpful to drop this behaviour. Use the +-I/--ide option for that. You might find -W helpful also so that warnings do +not cause the build to fail: + + buildman -o /tmp/build --board sandbox -wWI + + Changing the configuration ========================== @@ -1294,6 +1308,19 @@ Some options you might like are: break anything. But note this does not check bisectability! +Using boards.cfg +================ + +This file is no-longer needed by buildman but it is still generated in the +working directory. This helps avoid a delay on every build, since scanning all +the Kconfig files takes a few seconds. Use the -R flag to force regeneration +of the file - in that case buildman exits after writing the file. with exit code +2 if there was an error in the maintainer files. + +You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, +since it may be dropped altogether in future. + + TODO ==== @@ -1304,9 +1331,6 @@ scope for more though, e.g.: - 'hunting' for problems, perhaps by building a few boards for each arch, or checking commits for changed files and building only boards which use those files -- using the same git repo for all threads instead of cloning it. Currently - it uses about 500MB per thread, so on a 64-thread machine this is 32GB for - the build. Credits diff --git a/tools/buildman/board.py b/tools/buildman/board.py index 447aaabea8..8ef905b8ce 100644 --- a/tools/buildman/board.py +++ b/tools/buildman/board.py @@ -1,78 +1,12 @@ # SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2012 The Chromium OS Authors. -from collections import OrderedDict -import re -class Expr: - """A single regular expression for matching boards to build""" - - def __init__(self, expr): - """Set up a new Expr object. - - Args: - expr: String cotaining regular expression to store - """ - self._expr = expr - self._re = re.compile(expr) - - def Matches(self, props): - """Check if any of the properties match the regular expression. - - Args: - props: List of properties to check - Returns: - True if any of the properties match the regular expression - """ - for prop in props: - if self._re.match(prop): - return True - return False - - def __str__(self): - return self._expr - -class Term: - """A list of expressions each of which must match with properties. - - This provides a list of 'AND' expressions, meaning that each must - match the board properties for that board to be built. - """ - def __init__(self): - self._expr_list = [] - self._board_count = 0 - - def AddExpr(self, expr): - """Add an Expr object to the list to check. - - Args: - expr: New Expr object to add to the list of those that must - match for a board to be built. - """ - self._expr_list.append(Expr(expr)) - - def __str__(self): - """Return some sort of useful string describing the term""" - return '&'.join([str(expr) for expr in self._expr_list]) - - def Matches(self, props): - """Check if any of the properties match this term - - Each of the expressions in the term is checked. All must match. - - Args: - props: List of properties to check - Returns: - True if all of the expressions in the Term match, else False - """ - for expr in self._expr_list: - if not expr.Matches(props): - return False - return True +"""A single board which can be selected and built""" class Board: """A particular board that we can build""" - def __init__(self, status, arch, cpu, soc, vendor, board_name, target, options): + def __init__(self, status, arch, cpu, soc, vendor, board_name, target, cfg_name): """Create a new board type. Args: @@ -83,7 +17,7 @@ class Board: vendor: Name of vendor (e.g. armltd) board_name: Name of board (e.g. integrator) target: Target name (use make <target>_defconfig to configure) - options: board-specific options (e.g. integratorcp:CM1136) + cfg_name: Config name """ self.target = target self.arch = arch @@ -91,220 +25,7 @@ class Board: self.board_name = board_name self.vendor = vendor self.soc = soc - self.options = options + self.cfg_name = cfg_name self.props = [self.target, self.arch, self.cpu, self.board_name, - self.vendor, self.soc, self.options] + self.vendor, self.soc, self.cfg_name] self.build_it = False - - -class Boards: - """Manage a list of boards.""" - def __init__(self): - # Use a simple list here, sinc OrderedDict requires Python 2.7 - self._boards = [] - - def AddBoard(self, board): - """Add a new board to the list. - - The board's target member must not already exist in the board list. - - Args: - board: board to add - """ - self._boards.append(board) - - def ReadBoards(self, fname): - """Read a list of boards from a board file. - - Create a board object for each and add it to our _boards list. - - Args: - fname: Filename of boards.cfg file - """ - with open(fname, 'r', encoding='utf-8') as fd: - for line in fd: - if line[0] == '#': - continue - fields = line.split() - if not fields: - continue - for upto in range(len(fields)): - if fields[upto] == '-': - fields[upto] = '' - while len(fields) < 8: - fields.append('') - if len(fields) > 8: - fields = fields[:8] - - board = Board(*fields) - self.AddBoard(board) - - - def GetList(self): - """Return a list of available boards. - - Returns: - List of Board objects - """ - return self._boards - - def GetDict(self): - """Build a dictionary containing all the boards. - - Returns: - Dictionary: - key is board.target - value is board - """ - board_dict = OrderedDict() - for board in self._boards: - board_dict[board.target] = board - return board_dict - - def GetSelectedDict(self): - """Return a dictionary containing the selected boards - - Returns: - List of Board objects that are marked selected - """ - board_dict = OrderedDict() - for board in self._boards: - if board.build_it: - board_dict[board.target] = board - return board_dict - - def GetSelected(self): - """Return a list of selected boards - - Returns: - List of Board objects that are marked selected - """ - return [board for board in self._boards if board.build_it] - - def GetSelectedNames(self): - """Return a list of selected boards - - Returns: - List of board names that are marked selected - """ - return [board.target for board in self._boards if board.build_it] - - def _BuildTerms(self, args): - """Convert command line arguments to a list of terms. - - This deals with parsing of the arguments. It handles the '&' - operator, which joins several expressions into a single Term. - - For example: - ['arm & freescale sandbox', 'tegra'] - - will produce 3 Terms containing expressions as follows: - arm, freescale - sandbox - tegra - - The first Term has two expressions, both of which must match for - a board to be selected. - - Args: - args: List of command line arguments - Returns: - A list of Term objects - """ - syms = [] - for arg in args: - for word in arg.split(): - sym_build = [] - for term in word.split('&'): - if term: - sym_build.append(term) - sym_build.append('&') - syms += sym_build[:-1] - terms = [] - term = None - oper = None - for sym in syms: - if sym == '&': - oper = sym - elif oper: - term.AddExpr(sym) - oper = None - else: - if term: - terms.append(term) - term = Term() - term.AddExpr(sym) - if term: - terms.append(term) - return terms - - def SelectBoards(self, args, exclude=[], boards=None): - """Mark boards selected based on args - - Normally either boards (an explicit list of boards) or args (a list of - terms to match against) is used. It is possible to specify both, in - which case they are additive. - - If boards and args are both empty, all boards are selected. - - Args: - args: List of strings specifying boards to include, either named, - or by their target, architecture, cpu, vendor or soc. If - empty, all boards are selected. - exclude: List of boards to exclude, regardless of 'args' - boards: List of boards to build - - Returns: - Tuple - Dictionary which holds the list of boards which were selected - due to each argument, arranged by argument. - List of errors found - """ - result = OrderedDict() - warnings = [] - terms = self._BuildTerms(args) - - result['all'] = [] - for term in terms: - result[str(term)] = [] - - exclude_list = [] - for expr in exclude: - exclude_list.append(Expr(expr)) - - found = [] - for board in self._boards: - matching_term = None - build_it = False - if terms: - match = False - for term in terms: - if term.Matches(board.props): - matching_term = str(term) - build_it = True - break - elif boards: - if board.target in boards: - build_it = True - found.append(board.target) - else: - build_it = True - - # Check that it is not specifically excluded - for expr in exclude_list: - if expr.Matches(board.props): - build_it = False - break - - if build_it: - board.build_it = True - if matching_term: - result[matching_term].append(board.target) - result['all'].append(board.target) - - if boards: - remaining = set(boards) - set(found) - if remaining: - warnings.append('Boards not found: %s\n' % ', '.join(remaining)) - - return result, warnings diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py new file mode 100644 index 0000000000..8a0971aa40 --- /dev/null +++ b/tools/buildman/boards.py @@ -0,0 +1,752 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. +# Author: Simon Glass <sjg@chromium.org> +# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> + +"""Maintains a list of boards and allows them to be selected""" + +from collections import OrderedDict +import errno +import fnmatch +import glob +import multiprocessing +import os +import re +import sys +import tempfile +import time + +from buildman import board +from buildman import kconfiglib + + +### constant variables ### +OUTPUT_FILE = 'boards.cfg' +CONFIG_DIR = 'configs' +SLEEP_TIME = 0.03 +COMMENT_BLOCK = f'''# +# List of boards +# Automatically generated by {__file__}: don't edit +# +# Status, Arch, CPU, SoC, Vendor, Board, Target, Config, Maintainers + +''' + + +def try_remove(fname): + """Remove a file ignoring 'No such file or directory' error. + + Args: + fname (str): Filename to remove + + Raises: + OSError: output file exists but could not be removed + """ + try: + os.remove(fname) + except OSError as exception: + # Ignore 'No such file or directory' error + if exception.errno != errno.ENOENT: + raise + + +def output_is_new(output): + """Check if the output file is up to date. + + Looks at defconfig and Kconfig files to make sure none is newer than the + output file. Also ensures that the boards.cfg does not mention any removed + boards. + + Args: + output (str): Filename to check + + Returns: + True if the given output file exists and is newer than any of + *_defconfig, MAINTAINERS and Kconfig*. False otherwise. + + Raises: + OSError: output file exists but could not be opened + """ + # pylint: disable=too-many-branches + try: + ctime = os.path.getctime(output) + except OSError as exception: + if exception.errno == errno.ENOENT: + # return False on 'No such file or directory' error + return False + raise + + for (dirpath, _, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + for (dirpath, _, filenames) in os.walk('.'): + for filename in filenames: + if (fnmatch.fnmatch(filename, '*~') or + not fnmatch.fnmatch(filename, 'Kconfig*') and + not filename == 'MAINTAINERS'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + # Detect a board that has been removed since the current board database + # was generated + with open(output, encoding="utf-8") as inf: + for line in inf: + if 'Options,' in line: + return False + if line[0] == '#' or line == '\n': + continue + defconfig = line.split()[6] + '_defconfig' + if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): + return False + + return True + + +class Expr: + """A single regular expression for matching boards to build""" + + def __init__(self, expr): + """Set up a new Expr object. + + Args: + expr (str): String cotaining regular expression to store + """ + self._expr = expr + self._re = re.compile(expr) + + def matches(self, props): + """Check if any of the properties match the regular expression. + + Args: + props (list of str): List of properties to check + Returns: + True if any of the properties match the regular expression + """ + for prop in props: + if self._re.match(prop): + return True + return False + + def __str__(self): + return self._expr + +class Term: + """A list of expressions each of which must match with properties. + + This provides a list of 'AND' expressions, meaning that each must + match the board properties for that board to be built. + """ + def __init__(self): + self._expr_list = [] + self._board_count = 0 + + def add_expr(self, expr): + """Add an Expr object to the list to check. + + Args: + expr (Expr): New Expr object to add to the list of those that must + match for a board to be built. + """ + self._expr_list.append(Expr(expr)) + + def __str__(self): + """Return some sort of useful string describing the term""" + return '&'.join([str(expr) for expr in self._expr_list]) + + def matches(self, props): + """Check if any of the properties match this term + + Each of the expressions in the term is checked. All must match. + + Args: + props (list of str): List of properties to check + Returns: + True if all of the expressions in the Term match, else False + """ + for expr in self._expr_list: + if not expr.matches(props): + return False + return True + + +class KconfigScanner: + + """Kconfig scanner.""" + + ### constant variable only used in this class ### + _SYMBOL_TABLE = { + 'arch' : 'SYS_ARCH', + 'cpu' : 'SYS_CPU', + 'soc' : 'SYS_SOC', + 'vendor' : 'SYS_VENDOR', + 'board' : 'SYS_BOARD', + 'config' : 'SYS_CONFIG_NAME', + # 'target' is added later + } + + def __init__(self): + """Scan all the Kconfig files and create a Kconfig object.""" + # Define environment variables referenced from Kconfig + os.environ['srctree'] = os.getcwd() + os.environ['UBOOTVERSION'] = 'dummy' + os.environ['KCONFIG_OBJDIR'] = '' + self._tmpfile = None + self._conf = kconfiglib.Kconfig(warn=False) + + def __del__(self): + """Delete a leftover temporary file before exit. + + The scan() method of this class creates a temporay file and deletes + it on success. If scan() method throws an exception on the way, + the temporary file might be left over. In that case, it should be + deleted in this destructor. + """ + if self._tmpfile: + try_remove(self._tmpfile) + + def scan(self, defconfig): + """Load a defconfig file to obtain board parameters. + + Args: + defconfig (str): path to the defconfig file to be processed + + Returns: + A dictionary of board parameters. It has a form of: + { + 'arch': <arch_name>, + 'cpu': <cpu_name>, + 'soc': <soc_name>, + 'vendor': <vendor_name>, + 'board': <board_name>, + 'target': <target_name>, + 'config': <config_header_name>, + } + """ + # strip special prefixes and save it in a temporary file + outfd, self._tmpfile = tempfile.mkstemp() + with os.fdopen(outfd, 'w') as outf: + with open(defconfig, encoding='utf-8') as inf: + for line in inf: + colon = line.find(':CONFIG_') + if colon == -1: + outf.write(line) + else: + outf.write(line[colon + 1:]) + + self._conf.load_config(self._tmpfile) + try_remove(self._tmpfile) + self._tmpfile = None + + params = {} + + # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc. + # Set '-' if the value is empty. + for key, symbol in list(self._SYMBOL_TABLE.items()): + value = self._conf.syms.get(symbol).str_value + if value: + params[key] = value + else: + params[key] = '-' + + defconfig = os.path.basename(defconfig) + params['target'], match, rear = defconfig.partition('_defconfig') + assert match and not rear, f'{defconfig} : invalid defconfig' + + # fix-up for aarch64 + if params['arch'] == 'arm' and params['cpu'] == 'armv8': + params['arch'] = 'aarch64' + + return params + + +class MaintainersDatabase: + + """The database of board status and maintainers. + + Properties: + database: dict: + key: Board-target name (e.g. 'snow') + value: tuple: + str: Board status (e.g. 'Active') + str: List of maintainers, separated by : + warnings (list of str): List of warnings due to missing status, etc. + """ + + def __init__(self): + """Create an empty database.""" + self.database = {} + self.warnings = [] + + def get_status(self, target): + """Return the status of the given board. + + The board status is generally either 'Active' or 'Orphan'. + Display a warning message and return '-' if status information + is not found. + + Args: + target (str): Build-target name + + Returns: + str: 'Active', 'Orphan' or '-'. + """ + if not target in self.database: + self.warnings.append(f"WARNING: no status info for '{target}'") + return '-' + + tmp = self.database[target][0] + if tmp.startswith('Maintained'): + return 'Active' + if tmp.startswith('Supported'): + return 'Active' + if tmp.startswith('Orphan'): + return 'Orphan' + self.warnings.append(f"WARNING: {tmp}: unknown status for '{target}'") + return '-' + + def get_maintainers(self, target): + """Return the maintainers of the given board. + + Args: + target (str): Build-target name + + Returns: + str: Maintainers of the board. If the board has two or more + maintainers, they are separated with colons. + """ + if not target in self.database: + self.warnings.append(f"WARNING: no maintainers for '{target}'") + return '' + + return ':'.join(self.database[target][1]) + + def parse_file(self, fname): + """Parse a MAINTAINERS file. + + Parse a MAINTAINERS file and accumulate board status and maintainers + information in the self.database dict. + + Args: + fname (str): MAINTAINERS file to be parsed + """ + targets = [] + maintainers = [] + status = '-' + with open(fname, encoding="utf-8") as inf: + for line in inf: + # Check also commented maintainers + if line[:3] == '#M:': + line = line[1:] + tag, rest = line[:2], line[2:].strip() + if tag == 'M:': + maintainers.append(rest) + elif tag == 'F:': + # expand wildcard and filter by 'configs/*_defconfig' + for item in glob.glob(rest): + front, match, rear = item.partition('configs/') + if not front and match: + front, match, rear = rear.rpartition('_defconfig') + if match and not rear: + targets.append(front) + elif tag == 'S:': + status = rest + elif line == '\n': + for target in targets: + self.database[target] = (status, maintainers) + targets = [] + maintainers = [] + status = '-' + if targets: + for target in targets: + self.database[target] = (status, maintainers) + + +class Boards: + """Manage a list of boards.""" + def __init__(self): + self._boards = [] + + def add_board(self, brd): + """Add a new board to the list. + + The board's target member must not already exist in the board list. + + Args: + brd (Board): board to add + """ + self._boards.append(brd) + + def read_boards(self, fname): + """Read a list of boards from a board file. + + Create a Board object for each and add it to our _boards list. + + Args: + fname (str): Filename of boards.cfg file + """ + with open(fname, 'r', encoding='utf-8') as inf: + for line in inf: + if line[0] == '#': + continue + fields = line.split() + if not fields: + continue + for upto, field in enumerate(fields): + if field == '-': + fields[upto] = '' + while len(fields) < 8: + fields.append('') + if len(fields) > 8: + fields = fields[:8] + + brd = board.Board(*fields) + self.add_board(brd) + + + def get_list(self): + """Return a list of available boards. + + Returns: + List of Board objects + """ + return self._boards + + def get_dict(self): + """Build a dictionary containing all the boards. + + Returns: + Dictionary: + key is board.target + value is board + """ + board_dict = OrderedDict() + for brd in self._boards: + board_dict[brd.target] = brd + return board_dict + + def get_selected_dict(self): + """Return a dictionary containing the selected boards + + Returns: + List of Board objects that are marked selected + """ + board_dict = OrderedDict() + for brd in self._boards: + if brd.build_it: + board_dict[brd.target] = brd + return board_dict + + def get_selected(self): + """Return a list of selected boards + + Returns: + List of Board objects that are marked selected + """ + return [brd for brd in self._boards if brd.build_it] + + def get_selected_names(self): + """Return a list of selected boards + + Returns: + List of board names that are marked selected + """ + return [brd.target for brd in self._boards if brd.build_it] + + @classmethod + def _build_terms(cls, args): + """Convert command line arguments to a list of terms. + + This deals with parsing of the arguments. It handles the '&' + operator, which joins several expressions into a single Term. + + For example: + ['arm & freescale sandbox', 'tegra'] + + will produce 3 Terms containing expressions as follows: + arm, freescale + sandbox + tegra + + The first Term has two expressions, both of which must match for + a board to be selected. + + Args: + args (list of str): List of command line arguments + + Returns: + list of Term: A list of Term objects + """ + syms = [] + for arg in args: + for word in arg.split(): + sym_build = [] + for term in word.split('&'): + if term: + sym_build.append(term) + sym_build.append('&') + syms += sym_build[:-1] + terms = [] + term = None + oper = None + for sym in syms: + if sym == '&': + oper = sym + elif oper: + term.add_expr(sym) + oper = None + else: + if term: + terms.append(term) + term = Term() + term.add_expr(sym) + if term: + terms.append(term) + return terms + + def select_boards(self, args, exclude=None, brds=None): + """Mark boards selected based on args + + Normally either boards (an explicit list of boards) or args (a list of + terms to match against) is used. It is possible to specify both, in + which case they are additive. + + If brds and args are both empty, all boards are selected. + + Args: + args (list of str): List of strings specifying boards to include, + either named, or by their target, architecture, cpu, vendor or + soc. If empty, all boards are selected. + exclude (list of str): List of boards to exclude, regardless of + 'args', or None for none + brds (list of Board): List of boards to build, or None/[] for all + + Returns: + Tuple + Dictionary which holds the list of boards which were selected + due to each argument, arranged by argument. + List of errors found + """ + def _check_board(brd): + """Check whether to include or exclude a board + + Checks the various terms and decide whether to build it or not (the + 'build_it' variable). + + If it is built, add the board to the result[term] list so we know + which term caused it to be built. Add it to result['all'] also. + + Keep a list of boards we found in 'found', so we can report boards + which appear in self._boards but not in brds. + + Args: + brd (Board): Board to check + """ + matching_term = None + build_it = False + if terms: + for term in terms: + if term.matches(brd.props): + matching_term = str(term) + build_it = True + break + elif brds: + if brd.target in brds: + build_it = True + found.append(brd.target) + else: + build_it = True + + # Check that it is not specifically excluded + for expr in exclude_list: + if expr.matches(brd.props): + build_it = False + break + + if build_it: + brd.build_it = True + if matching_term: + result[matching_term].append(brd.target) + result['all'].append(brd.target) + + result = OrderedDict() + warnings = [] + terms = self._build_terms(args) + + result['all'] = [] + for term in terms: + result[str(term)] = [] + + exclude_list = [] + if exclude: + for expr in exclude: + exclude_list.append(Expr(expr)) + + found = [] + for brd in self._boards: + _check_board(brd) + + if brds: + remaining = set(brds) - set(found) + if remaining: + warnings.append(f"Boards not found: {', '.join(remaining)}\n") + + return result, warnings + + @classmethod + def scan_defconfigs_for_multiprocess(cls, queue, defconfigs): + """Scan defconfig files and queue their board parameters + + This function is intended to be passed to multiprocessing.Process() + constructor. + + Args: + queue (multiprocessing.Queue): The resulting board parameters are + written into this. + defconfigs (sequence of str): A sequence of defconfig files to be + scanned. + """ + kconf_scanner = KconfigScanner() + for defconfig in defconfigs: + queue.put(kconf_scanner.scan(defconfig)) + + @classmethod + def read_queues(cls, queues, params_list): + """Read the queues and append the data to the paramers list""" + for que in queues: + while not que.empty(): + params_list.append(que.get()) + + def scan_defconfigs(self, jobs=1): + """Collect board parameters for all defconfig files. + + This function invokes multiple processes for faster processing. + + Args: + jobs (int): The number of jobs to run simultaneously + """ + all_defconfigs = [] + for (dirpath, _, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + all_defconfigs.append(os.path.join(dirpath, filename)) + + total_boards = len(all_defconfigs) + processes = [] + queues = [] + for i in range(jobs): + defconfigs = all_defconfigs[total_boards * i // jobs : + total_boards * (i + 1) // jobs] + que = multiprocessing.Queue(maxsize=-1) + proc = multiprocessing.Process( + target=self.scan_defconfigs_for_multiprocess, + args=(que, defconfigs)) + proc.start() + processes.append(proc) + queues.append(que) + + # The resulting data should be accumulated to this list + params_list = [] + + # Data in the queues should be retrieved preriodically. + # Otherwise, the queues would become full and subprocesses would get stuck. + while any(p.is_alive() for p in processes): + self.read_queues(queues, params_list) + # sleep for a while until the queues are filled + time.sleep(SLEEP_TIME) + + # Joining subprocesses just in case + # (All subprocesses should already have been finished) + for proc in processes: + proc.join() + + # retrieve leftover data + self.read_queues(queues, params_list) + + return params_list + + @classmethod + def insert_maintainers_info(cls, params_list): + """Add Status and Maintainers information to the board parameters list. + + Args: + params_list (list of dict): A list of the board parameters + + Returns: + list of str: List of warnings collected due to missing status, etc. + """ + database = MaintainersDatabase() + for (dirpath, _, filenames) in os.walk('.'): + if 'MAINTAINERS' in filenames: + database.parse_file(os.path.join(dirpath, 'MAINTAINERS')) + + for i, params in enumerate(params_list): + target = params['target'] + params['status'] = database.get_status(target) + params['maintainers'] = database.get_maintainers(target) + params_list[i] = params + return database.warnings + + @classmethod + def format_and_output(cls, params_list, output): + """Write board parameters into a file. + + Columnate the board parameters, sort lines alphabetically, + and then write them to a file. + + Args: + params_list (list of dict): The list of board parameters + output (str): The path to the output file + """ + fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target', + 'config', 'maintainers') + + # First, decide the width of each column + max_length = {f: 0 for f in fields} + for params in params_list: + for field in fields: + max_length[field] = max(max_length[field], len(params[field])) + + output_lines = [] + for params in params_list: + line = '' + for field in fields: + # insert two spaces between fields like column -t would + line += ' ' + params[field].ljust(max_length[field]) + output_lines.append(line.strip()) + + # ignore case when sorting + output_lines.sort(key=str.lower) + + with open(output, 'w', encoding="utf-8") as outf: + outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') + + def ensure_board_list(self, output, jobs=1, force=False, quiet=False): + """Generate a board database file if needed. + + Args: + output (str): The name of the output file + jobs (int): The number of jobs to run simultaneously + force (bool): Force to generate the output even if it is new + quiet (bool): True to avoid printing a message if nothing needs doing + + Returns: + bool: True if all is well, False if there were warnings + """ + if not force and output_is_new(output): + if not quiet: + print(f'{output} is up to date. Nothing to do.') + return True + params_list = self.scan_defconfigs(jobs) + warnings = self.insert_maintainers_info(params_list) + for warn in warnings: + print(warn, file=sys.stderr) + self.format_and_output(params_list, output) + return not warnings diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index aa2ffe16f6..76252b9079 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -108,7 +108,7 @@ u-boot/ source directory boards: List of Board objects which have line in the error/warning output errline: The text of the error line """ -ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline') +ErrLine = collections.namedtuple('ErrLine', 'char,brds,errline') # Possible build outcomes OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4)) @@ -213,6 +213,8 @@ class Builder: threading is not being used _terminated: Thread was terminated due to an error _restarting_config: True if 'Restart config' is detected in output + _ide: Produce output suitable for an Integrated Development Environment, + i.e. dont emit progress information and put errors/warnings on stderr """ class Outcome: """Records a build outcome for a single make invocation @@ -325,6 +327,7 @@ class Builder: self.config_filenames = BASE_CONFIG_FILENAMES self.work_in_output = work_in_output self.adjust_cfg = adjust_cfg + self._ide = False if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES @@ -382,7 +385,7 @@ class Builder: show_detail=False, show_bloat=False, list_error_boards=False, show_config=False, show_environment=False, filter_dtb_warnings=False, - filter_migration_warnings=False): + filter_migration_warnings=False, ide=False): """Setup display options for the builder. Args: @@ -397,6 +400,8 @@ class Builder: compiler filter_migration_warnings: Filter out any warnings about migrating a board to driver model + ide: Create output that can be parsed by an IDE. There is no '+' prefix on + error lines and output on stderr stays on stderr. """ self._show_errors = show_errors self._show_sizes = show_sizes @@ -407,6 +412,7 @@ class Builder: self._show_environment = show_environment self._filter_dtb_warnings = filter_dtb_warnings self._filter_migration_warnings = filter_migration_warnings + self._ide = ide def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -535,8 +541,9 @@ class Builder: line += '%s : ' % self._complete_delay line += target - terminal.print_clear() - tprint(line, newline=False, limit_to_line=True) + if not self._ide: + terminal.print_clear() + tprint(line, newline=False, limit_to_line=True) def _GetOutputDir(self, commit_upto): """Get the name of the output directory for a commit number @@ -662,17 +669,15 @@ class Builder: """ sym = {} for line in fd.readlines(): - try: - if line.strip(): - size, type, name = line[:-1].split() - except: - tprint("Invalid line in file '%s': '%s'" % (fname, line[:-1])) - continue - if type in 'tTdDbB': - # function names begin with '.' on 64-bit powerpc - if '.' in name[1:]: - name = 'static.' + name.split('.')[0] - sym[name] = sym.get(name, 0) + int(size, 16) + line = line.strip() + parts = line.split() + if line and len(parts) == 3: + size, type, name = line.split() + if type in 'tTdDbB': + # function names begin with '.' on 64-bit powerpc + if '.' in name[1:]: + name = 'static.' + name.split('.')[0] + sym[name] = sym.get(name, 0) + int(size, 16) return sym def _ProcessConfig(self, fname): @@ -834,8 +839,9 @@ class Builder: Returns: Tuple: - Dict containing boards which passed building this commit. - keyed by board.target + Dict containing boards which built this commit: + key: board.target + value: Builder.Outcome object List containing a summary of error lines Dict keyed by error line, containing a list of the Board objects with that error @@ -867,11 +873,11 @@ class Builder: config = {} environment = {} - for board in boards_selected.values(): - outcome = self.GetBuildOutcome(commit_upto, board.target, + for brd in boards_selected.values(): + outcome = self.GetBuildOutcome(commit_upto, brd.target, read_func_sizes, read_config, read_environment) - board_dict[board.target] = outcome + board_dict[brd.target] = outcome last_func = None last_was_warning = False for line in outcome.err_lines: @@ -886,29 +892,29 @@ class Builder: if is_warning or (last_was_warning and is_note): if last_func: AddLine(warn_lines_summary, warn_lines_boards, - last_func, board) + last_func, brd) AddLine(warn_lines_summary, warn_lines_boards, - line, board) + line, brd) else: if last_func: AddLine(err_lines_summary, err_lines_boards, - last_func, board) + last_func, brd) AddLine(err_lines_summary, err_lines_boards, - line, board) + line, brd) last_was_warning = is_warning last_func = None - tconfig = Config(self.config_filenames, board.target) + tconfig = Config(self.config_filenames, brd.target) for fname in self.config_filenames: if outcome.config: for key, value in outcome.config[fname].items(): tconfig.Add(fname, key, value) - config[board.target] = tconfig + config[brd.target] = tconfig - tenvironment = Environment(board.target) + tenvironment = Environment(brd.target) if outcome.environment: for key, value in outcome.environment.items(): tenvironment.Add(key, value) - environment[board.target] = tenvironment + environment[brd.target] = tenvironment return (board_dict, err_lines_summary, err_lines_boards, warn_lines_summary, warn_lines_boards, config, environment) @@ -963,9 +969,8 @@ class Builder: board.target """ self._base_board_dict = {} - for board in board_selected: - self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, - {}) + for brd in board_selected: + self._base_board_dict[brd] = Builder.Outcome(0, [], [], {}, {}, {}) self._base_err_lines = [] self._base_warn_lines = [] self._base_err_line_boards = {} @@ -1209,14 +1214,14 @@ class Builder: List of boards with that error line, or [] if the user has not requested such a list """ - boards = [] + brds = [] board_set = set() if self._list_error_boards: - for board in line_boards[line]: - if not board in board_set: - boards.append(board) - board_set.add(board) - return boards + for brd in line_boards[line]: + if not brd in board_set: + brds.append(brd) + board_set.add(brd) + return brds def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, char): @@ -1319,8 +1324,7 @@ class Builder: if err_lines: out_list = [] for line in err_lines: - boards = '' - names = [board.target for board in line.boards] + names = [brd.target for brd in line.brds] board_str = ' '.join(names) if names else '' if board_str: out = self.col.build(colour, line.char + '(') @@ -1369,8 +1373,14 @@ class Builder: better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') + # For the IDE mode, print out all the output + if self._ide: + outcome = board_dict[target] + for line in outcome.err_lines: + sys.stderr.write(line) + # Display results by arch - if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, + elif any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, worse_err, better_err, worse_warn, better_warn)): arch_list = {} self.AddOutcome(board_selected, arch_list, ok_boards, '', @@ -1535,9 +1545,9 @@ class Builder: # Get a list of boards that did not get built, if needed not_built = [] - for board in board_selected: - if not board in board_dict: - not_built.append(board) + for brd in board_selected: + if not brd in board_dict: + not_built.append(brd) if not_built: tprint("Boards not built (%d): %s" % (len(not_built), ', '.join(not_built))) @@ -1746,14 +1756,15 @@ class Builder: self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), commits is not None) self._PrepareOutputSpace() - tprint('\rStarting build...', newline=False) + if not self._ide: + tprint('\rStarting build...', newline=False) self.SetupBuild(board_selected, commits) self.ProcessResult(None) self.thread_exceptions = [] # Create jobs to build all commits for each board for brd in board_selected.values(): job = builderthread.BuilderJob() - job.board = brd + job.brd = brd job.commits = commits job.keep_outputs = keep_outputs job.work_in_output = self.work_in_output @@ -1773,24 +1784,25 @@ class Builder: # Wait until we have processed all output self.out_queue.join() - tprint() - - msg = 'Completed: %d total built' % self.count - if self.already_done: - msg += ' (%d previously' % self.already_done - if self.already_done != self.count: - msg += ', %d newly' % (self.count - self.already_done) - msg += ')' - duration = datetime.now() - self._start_time - if duration > timedelta(microseconds=1000000): - if duration.microseconds >= 500000: - duration = duration + timedelta(seconds=1) - duration = duration - timedelta(microseconds=duration.microseconds) - rate = float(self.count) / duration.total_seconds() - msg += ', duration %s, rate %1.2f' % (duration, rate) - tprint(msg) - if self.thread_exceptions: - tprint('Failed: %d thread exceptions' % len(self.thread_exceptions), - colour=self.col.RED) + if not self._ide: + tprint() + + msg = 'Completed: %d total built' % self.count + if self.already_done: + msg += ' (%d previously' % self.already_done + if self.already_done != self.count: + msg += ', %d newly' % (self.count - self.already_done) + msg += ')' + duration = datetime.now() - self._start_time + if duration > timedelta(microseconds=1000000): + if duration.microseconds >= 500000: + duration = duration + timedelta(seconds=1) + duration = duration - timedelta(microseconds=duration.microseconds) + rate = float(self.count) / duration.total_seconds() + msg += ', duration %s, rate %1.2f' % (duration, rate) + tprint(msg) + if self.thread_exceptions: + tprint('Failed: %d thread exceptions' % len(self.thread_exceptions), + colour=self.col.RED) return (self.fail, self.warned, self.thread_exceptions) diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 7522ff62de..6240e08c76 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -40,7 +40,7 @@ class BuilderJob: """Holds information about a job to be performed by a thread Members: - board: Board object to build + brd: Board object to build commits: List of Commit objects to build keep_outputs: True to save build output files step: 1 to process every commit, n to process every nth commit @@ -48,7 +48,7 @@ class BuilderJob: don't write to a separate output directory. """ def __init__(self): - self.board = None + self.brd = None self.commits = [] self.keep_outputs = False self.step = 1 @@ -491,7 +491,7 @@ class BuilderThread(threading.Thread): Returns: List of Result objects """ - brd = job.board + brd = job.brd work_dir = self.builder.GetThreadDir(self.thread_num) self.toolchain = None if job.commits: diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 8586bdf3b5..b29c1eb5ee 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -59,9 +59,8 @@ def ParseArgs(): parser.add_option('-i', '--in-tree', dest='in_tree', action='store_true', default=False, help='Build in the source tree instead of a separate directory') - # -I will be removed after April 2021 - parser.add_option('-I', '--incremental', action='store_true', - default=False, help='Deprecated, does nothing. See -m') + parser.add_option('-I', '--ide', action='store_true', default=False, + help='Create build output that can be parsed by an IDE') parser.add_option('-j', '--jobs', dest='jobs', type='int', default=None, help='Number of jobs to run at once (passed to make)') parser.add_option('-k', '--keep-outputs', action='store_true', @@ -90,6 +89,8 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-R', '--regen-board-list', action='store_true', + help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 8f4810bc3e..0c75466fbd 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -8,7 +8,7 @@ import shutil import subprocess import sys -from buildman import board +from buildman import boards from buildman import bsettings from buildman import cfgutil from buildman import toolchain @@ -87,7 +87,7 @@ def ShowActions(series, why_selected, boards_selected, builder, options, for warning in board_warnings: print(col.build(col.YELLOW, warning)) -def ShowToolchainPrefix(boards, toolchains): +def ShowToolchainPrefix(brds, toolchains): """Show information about a the tool chain used by one or more boards The function checks that all boards use the same toolchain, then prints @@ -100,9 +100,9 @@ def ShowToolchainPrefix(boards, toolchains): Return: None on success, string error message otherwise """ - boards = boards.GetSelectedDict() + board_selected = brds.get_selected_dict() tc_set = set() - for brd in boards.values(): + for brd in board_selected.values(): tc_set.add(toolchains.Select(brd.arch)) if len(tc_set) != 1: return 'Supplied boards must share one toolchain' @@ -111,7 +111,7 @@ def ShowToolchainPrefix(boards, toolchains): print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) return None -def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, +def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, clean_dir=False, test_thread_exceptions=False): """The main control code for buildman @@ -124,7 +124,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, to execute 'make'. If this is None, the normal function will be used, which calls the 'make' tool with suitable arguments. This setting is useful for tests. - board: Boards() object to use, containing a list of available + brds: Boards() object to use, containing a list of available boards. If this is None it will be created and scanned. clean_dir: Used for tests only, indicates that the existing output_dir should be removed before starting the build @@ -176,32 +176,25 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, print() return 0 - if options.incremental: - print(col.build(col.RED, - 'Warning: -I has been removed. See documentation')) if not options.output_dir: if options.work_in_output: sys.exit(col.build(col.RED, '-w requires that you specify -o')) options.output_dir = '..' # Work out what subset of the boards we are building - if not boards: + if not brds: if not os.path.exists(options.output_dir): os.makedirs(options.output_dir) board_file = os.path.join(options.output_dir, 'boards.cfg') - our_path = os.path.dirname(os.path.realpath(__file__)) - genboardscfg = os.path.join(our_path, '../genboardscfg.py') - if not os.path.exists(genboardscfg): - genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') - status = subprocess.call([genboardscfg, '-q', '-o', board_file]) - if status != 0: - # Older versions don't support -q - status = subprocess.call([genboardscfg, '-o', board_file]) - if status != 0: - sys.exit("Failed to generate boards.cfg") - - boards = board.Boards() - boards.ReadBoards(board_file) + + brds = boards.Boards() + ok = brds.ensure_board_list(board_file, + options.threads or multiprocessing.cpu_count(), + force=options.regen_board_list, + quiet=not options.verbose) + if options.regen_board_list: + return 0 if ok else 2 + brds.read_boards(board_file) exclude = [] if options.exclude: @@ -214,14 +207,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, requested_boards += b.split(',') else: requested_boards = None - why_selected, board_warnings = boards.SelectBoards(args, exclude, - requested_boards) - selected = boards.GetSelected() + why_selected, board_warnings = brds.select_boards(args, exclude, + requested_boards) + selected = brds.get_selected() if not len(selected): sys.exit(col.build(col.RED, 'No matching boards found')) if options.print_prefix: - err = ShowToolchainPrefix(boards, toolchains) + err = ShowToolchainPrefix(brds, toolchains) if err: sys.exit(col.build(col.RED, err)) return 0 @@ -352,7 +345,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, builder.in_tree = options.in_tree # Work out which boards to build - board_selected = boards.GetSelectedDict() + board_selected = brds.get_selected_dict() if series: commits = series.commits @@ -362,8 +355,9 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, else: commits = None - tprint(GetActionSummary(options.summary, commits, board_selected, - options)) + if not options.ide: + tprint(GetActionSummary(options.summary, commits, board_selected, + options)) # We can't show function sizes without board details at present if options.show_bloat: @@ -372,7 +366,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, options.show_errors, options.show_sizes, options.show_detail, options.show_bloat, options.list_error_boards, options.show_config, options.show_environment, options.filter_dtb_warnings, - options.filter_migration_warnings) + options.filter_migration_warnings, options.ide) if options.summary: builder.ShowSummary(commits, board_selected) else: diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index fbf6706644..f12e996634 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -9,6 +9,7 @@ import tempfile import unittest from buildman import board +from buildman import boards from buildman import bsettings from buildman import cmdline from buildman import control @@ -35,7 +36,7 @@ chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot} chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot} ''' -boards = [ +BOARDS = [ ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], @@ -187,14 +188,14 @@ class TestFunctional(unittest.TestCase): self.setupToolchains() self._toolchains.Add('arm-gcc', test=False) self._toolchains.Add('powerpc-gcc', test=False) - self._boards = board.Boards() - for brd in boards: - self._boards.AddBoard(board.Board(*brd)) + self._boards = boards.Boards() + for brd in BOARDS: + self._boards.add_board(board.Board(*brd)) # Directories where the source been cloned self._clone_dirs = [] self._commits = len(commit_shortlog.splitlines()) + 1 - self._total_builds = self._commits * len(boards) + self._total_builds = self._commits * len(BOARDS) # Number of calls to make self._make_calls = 0 @@ -220,13 +221,13 @@ class TestFunctional(unittest.TestCase): return command.run_pipe([[self._buildman_pathname] + list(args)], capture=True, capture_stderr=True) - def _RunControl(self, *args, boards=None, clean_dir=False, + def _RunControl(self, *args, brds=None, clean_dir=False, test_thread_exceptions=False): """Run buildman Args: args: List of arguments to pass - boards: + brds: Boards object clean_dir: Used for tests only, indicates that the existing output_dir should be removed before starting the build test_thread_exceptions: Uses for tests only, True to make the threads @@ -239,7 +240,7 @@ class TestFunctional(unittest.TestCase): sys.argv = [sys.argv[0]] + list(args) options, args = cmdline.ParseArgs() result = control.DoBuildman(options, args, toolchains=self._toolchains, - make_func=self._HandleMake, boards=boards or self._boards, + make_func=self._HandleMake, brds=brds or self._boards, clean_dir=clean_dir, test_thread_exceptions=test_thread_exceptions) self._builder = control.builder @@ -442,7 +443,7 @@ class TestFunctional(unittest.TestCase): def testNoBoards(self): """Test that buildman aborts when there are no boards""" - self._boards = board.Boards() + self._boards = boards.Boards() with self.assertRaises(SystemExit): self._RunControl() @@ -451,7 +452,7 @@ class TestFunctional(unittest.TestCase): self.setupToolchains(); self._RunControl('-o', self._output_dir) lines = terminal.get_print_test_lines() - self.assertIn('Building current source for %d boards' % len(boards), + self.assertIn('Building current source for %d boards' % len(BOARDS), lines[0].text) def testBadBranch(self): @@ -467,7 +468,7 @@ class TestFunctional(unittest.TestCase): # Buildman always builds the upstream commit as well self.assertIn('Building %d commits for %d boards' % - (self._commits, len(boards)), lines[0].text) + (self._commits, len(BOARDS)), lines[0].text) self.assertEqual(self._builder.count, self._total_builds) # Only sandbox should succeed, the others don't have toolchains @@ -476,12 +477,12 @@ class TestFunctional(unittest.TestCase): self.assertEqual(ret_code, 100) for commit in range(self._commits): - for board in self._boards.GetList(): - if board.arch != 'sandbox': - errfile = self._builder.GetErrFile(commit, board.target) + for brd in self._boards.get_list(): + if brd.arch != 'sandbox': + errfile = self._builder.GetErrFile(commit, brd.target) fd = open(errfile) self.assertEqual(fd.readlines(), - ['No tool chain for %s\n' % board.arch]) + ['No tool chain for %s\n' % brd.arch]) fd.close() def testBranch(self): @@ -493,17 +494,17 @@ class TestFunctional(unittest.TestCase): def testCount(self): """Test building a specific number of commitst""" self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir) - self.assertEqual(self._builder.count, 2 * len(boards)) + self.assertEqual(self._builder.count, 2 * len(BOARDS)) self.assertEqual(self._builder.fail, 0) # Each board has a config, and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (1 + 2)) + self.assertEqual(self._make_calls, len(BOARDS) * (1 + 2)) def testIncremental(self): """Test building a branch twice - the second time should do nothing""" self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir) # Each board has a mrproper, config, and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) + self.assertEqual(self._make_calls, len(BOARDS) * (self._commits + 1)) self._make_calls = 0 self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False) self.assertEqual(self._make_calls, 0) @@ -516,19 +517,19 @@ class TestFunctional(unittest.TestCase): self._make_calls = 0 self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False) # Each board has a config and one make per commit - self.assertEqual(self._make_calls, len(boards) * (self._commits + 1)) + self.assertEqual(self._make_calls, len(BOARDS) * (self._commits + 1)) def testForceReconfigure(self): """The -f flag should force a rebuild""" self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) # Each commit has a config and make - self.assertEqual(self._make_calls, len(boards) * self._commits * 2) + self.assertEqual(self._make_calls, len(BOARDS) * self._commits * 2) def testMrproper(self): """The -f flag should force a rebuild""" self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir) # Each board has a mkproper, config and then one make per commit - self.assertEqual(self._make_calls, len(boards) * (self._commits + 2)) + self.assertEqual(self._make_calls, len(BOARDS) * (self._commits + 2)) def testErrors(self): """Test handling of build errors""" @@ -580,10 +581,10 @@ class TestFunctional(unittest.TestCase): def testWorkInOutput(self): """Test the -w option which should write directly to the output dir""" - board_list = board.Boards() - board_list.AddBoard(board.Board(*boards[0])) + board_list = boards.Boards() + board_list.add_board(board.Board(*BOARDS[0])) self._RunControl('-o', self._output_dir, '-w', clean_dir=False, - boards=board_list) + brds=board_list) self.assertTrue( os.path.exists(os.path.join(self._output_dir, 'u-boot'))) self.assertTrue( @@ -599,15 +600,15 @@ class TestFunctional(unittest.TestCase): self.assertFalse( os.path.exists(os.path.join(self._output_dir, 'u-boot'))) - board_list = board.Boards() - board_list.AddBoard(board.Board(*boards[0])) + board_list = boards.Boards() + board_list.add_board(board.Board(*BOARDS[0])) with self.assertRaises(SystemExit) as e: self._RunControl('-b', self._test_branch, '-o', self._output_dir, - '-w', clean_dir=False, boards=board_list) + '-w', clean_dir=False, brds=board_list) self.assertIn("single commit", str(e.exception)) - board_list = board.Boards() - board_list.AddBoard(board.Board(*boards[0])) + board_list = boards.Boards() + board_list.add_board(board.Board(*BOARDS[0])) with self.assertRaises(SystemExit) as e: self._RunControl('-w', clean_dir=False) self.assertIn("specify -o", str(e.exception)) diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 27287438ee..daf5467503 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -10,6 +10,7 @@ import time import unittest from buildman import board +from buildman import boards from buildman import bsettings from buildman import builder from buildman import cfgutil @@ -94,7 +95,7 @@ commits = [ [errors[4]]], ] -boards = [ +BOARDS = [ ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''], ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''], ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''], @@ -131,10 +132,10 @@ class TestBuild(unittest.TestCase): self.commits.append(comm) # Set up boards to build - self.boards = board.Boards() - for brd in boards: - self.boards.AddBoard(board.Board(*brd)) - self.boards.SelectBoards([]) + self.brds = boards.Boards() + for brd in BOARDS: + self.brds.add_board(board.Board(*brd)) + self.brds.select_boards([]) # Add some test settings bsettings.Setup(None) @@ -176,7 +177,7 @@ class TestBuild(unittest.TestCase): result.combined = result.stdout + result.stderr return result - def assertSummary(self, text, arch, plus, boards, outcome=OUTCOME_ERR): + def assertSummary(self, text, arch, plus, brds, outcome=OUTCOME_ERR): col = self._col expected_colour = (col.GREEN if outcome == OUTCOME_OK else col.YELLOW if outcome == OUTCOME_WARN else col.RED) @@ -184,8 +185,8 @@ class TestBuild(unittest.TestCase): # TODO(sjg@chromium.org): If plus is '', we shouldn't need this expect += ' ' + col.build(expected_colour, plus) expect += ' ' - for board in boards: - expect += col.build(expected_colour, ' %s' % board) + for brd in brds: + expect += col.build(expected_colour, ' %s' % brd) self.assertEqual(text, expect) def _SetupTest(self, echo_lines=False, threads=1, **kwdisplay_args): @@ -203,7 +204,7 @@ class TestBuild(unittest.TestCase): build = builder.Builder(self.toolchains, self.base_dir, None, threads, 2, checkout=False, show_unknown=False) build.do_make = self.Make - board_selected = self.boards.GetSelectedDict() + board_selected = self.brds.get_selected_dict() # Build the boards for the pre-defined commits and warnings/errors # associated with each. This calls our Make() to inject the fake output. @@ -217,7 +218,7 @@ class TestBuild(unittest.TestCase): # We should get two starting messages, an update for every commit built # and a summary message - self.assertEqual(count, len(commits) * len(boards) + 3) + self.assertEqual(count, len(commits) * len(BOARDS) + 3) build.SetDisplayOptions(**kwdisplay_args); build.ShowSummary(self.commits, board_selected) if echo_lines: @@ -236,7 +237,7 @@ class TestBuild(unittest.TestCase): filter_dtb_warnings: Adjust the check for output produced with the --filter-dtb-warnings flag """ - def add_line_prefix(prefix, boards, error_str, colour): + def add_line_prefix(prefix, brds, error_str, colour): """Add a prefix to each line of a string The training \n in error_str is removed before processing @@ -253,9 +254,9 @@ class TestBuild(unittest.TestCase): lines = error_str.strip().splitlines() new_lines = [] for line in lines: - if boards: + if brds: expect = self._col.build(colour, prefix + '(') - expect += self._col.build(self._col.MAGENTA, boards, + expect += self._col.build(self._col.MAGENTA, brds, bright=False) expect += self._col.build(colour, ') %s' % line) else: @@ -468,18 +469,18 @@ class TestBuild(unittest.TestCase): def testBoardSingle(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['sandbox']), + self.assertEqual(self.brds.select_boards(['sandbox']), ({'all': ['board4'], 'sandbox': ['board4']}, [])) def testBoardArch(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['arm']), + self.assertEqual(self.brds.select_boards(['arm']), ({'all': ['board0', 'board1'], 'arm': ['board0', 'board1']}, [])) def testBoardArchSingle(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['arm sandbox']), + self.assertEqual(self.brds.select_boards(['arm sandbox']), ({'sandbox': ['board4'], 'all': ['board0', 'board1', 'board4'], 'arm': ['board0', 'board1']}, [])) @@ -487,20 +488,20 @@ class TestBuild(unittest.TestCase): def testBoardArchSingleMultiWord(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['arm', 'sandbox']), + self.assertEqual(self.brds.select_boards(['arm', 'sandbox']), ({'sandbox': ['board4'], 'all': ['board0', 'board1', 'board4'], 'arm': ['board0', 'board1']}, [])) def testBoardSingleAnd(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['Tester & arm']), + self.assertEqual(self.brds.select_boards(['Tester & arm']), ({'Tester&arm': ['board0', 'board1'], 'all': ['board0', 'board1']}, [])) def testBoardTwoAnd(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['Tester', '&', 'arm', + self.assertEqual(self.brds.select_boards(['Tester', '&', 'arm', 'Tester' '&', 'powerpc', 'sandbox']), ({'sandbox': ['board4'], @@ -511,19 +512,19 @@ class TestBuild(unittest.TestCase): def testBoardAll(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards([]), + self.assertEqual(self.brds.select_boards([]), ({'all': ['board0', 'board1', 'board2', 'board3', 'board4']}, [])) def testBoardRegularExpression(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['T.*r&^Po']), + self.assertEqual(self.brds.select_boards(['T.*r&^Po']), ({'all': ['board2', 'board3'], 'T.*r&^Po': ['board2', 'board3']}, [])) def testBoardDuplicate(self): """Test single board selection""" - self.assertEqual(self.boards.SelectBoards(['sandbox sandbox', + self.assertEqual(self.brds.select_boards(['sandbox sandbox', 'sandbox']), ({'all': ['board4'], 'sandbox': ['board4']}, [])) def CheckDirs(self, build, dirname): diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 46a4e5ed40..fea40ba215 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -441,7 +441,7 @@ class Toolchains: args = args[:m.start(0)] + value + args[m.end(0):] return args - def GetMakeArguments(self, board): + def GetMakeArguments(self, brd): """Returns 'make' arguments for a given board The flags are in a section called 'make-flags'. Flags are named @@ -462,13 +462,13 @@ class Toolchains: A special 'target' variable is set to the board target. Args: - board: Board object for the board to check. + brd: Board object for the board to check. Returns: 'make' flags for that board, or '' if none """ - self._make_flags['target'] = board.target + self._make_flags['target'] = brd.target arg_str = self.ResolveReferences(self._make_flags, - self._make_flags.get(board.target, '')) + self._make_flags.get(brd.target, '')) args = re.findall("(?:\".*?\"|\S)+", arg_str) i = 0 while i < len(args): |