diff options
Diffstat (limited to 'tools/buildman/boards.py')
| -rw-r--r-- | tools/buildman/boards.py | 752 |
1 files changed, 752 insertions, 0 deletions
diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py new file mode 100644 index 0000000000..8a0971aa40 --- /dev/null +++ b/tools/buildman/boards.py @@ -0,0 +1,752 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2012 The Chromium OS Authors. +# Author: Simon Glass <sjg@chromium.org> +# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> + +"""Maintains a list of boards and allows them to be selected""" + +from collections import OrderedDict +import errno +import fnmatch +import glob +import multiprocessing +import os +import re +import sys +import tempfile +import time + +from buildman import board +from buildman import kconfiglib + + +### constant variables ### +OUTPUT_FILE = 'boards.cfg' +CONFIG_DIR = 'configs' +SLEEP_TIME = 0.03 +COMMENT_BLOCK = f'''# +# List of boards +# Automatically generated by {__file__}: don't edit +# +# Status, Arch, CPU, SoC, Vendor, Board, Target, Config, Maintainers + +''' + + +def try_remove(fname): + """Remove a file ignoring 'No such file or directory' error. + + Args: + fname (str): Filename to remove + + Raises: + OSError: output file exists but could not be removed + """ + try: + os.remove(fname) + except OSError as exception: + # Ignore 'No such file or directory' error + if exception.errno != errno.ENOENT: + raise + + +def output_is_new(output): + """Check if the output file is up to date. + + Looks at defconfig and Kconfig files to make sure none is newer than the + output file. Also ensures that the boards.cfg does not mention any removed + boards. + + Args: + output (str): Filename to check + + Returns: + True if the given output file exists and is newer than any of + *_defconfig, MAINTAINERS and Kconfig*. False otherwise. + + Raises: + OSError: output file exists but could not be opened + """ + # pylint: disable=too-many-branches + try: + ctime = os.path.getctime(output) + except OSError as exception: + if exception.errno == errno.ENOENT: + # return False on 'No such file or directory' error + return False + raise + + for (dirpath, _, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + for (dirpath, _, filenames) in os.walk('.'): + for filename in filenames: + if (fnmatch.fnmatch(filename, '*~') or + not fnmatch.fnmatch(filename, 'Kconfig*') and + not filename == 'MAINTAINERS'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + # Detect a board that has been removed since the current board database + # was generated + with open(output, encoding="utf-8") as inf: + for line in inf: + if 'Options,' in line: + return False + if line[0] == '#' or line == '\n': + continue + defconfig = line.split()[6] + '_defconfig' + if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): + return False + + return True + + +class Expr: + """A single regular expression for matching boards to build""" + + def __init__(self, expr): + """Set up a new Expr object. + + Args: + expr (str): String cotaining regular expression to store + """ + self._expr = expr + self._re = re.compile(expr) + + def matches(self, props): + """Check if any of the properties match the regular expression. + + Args: + props (list of str): List of properties to check + Returns: + True if any of the properties match the regular expression + """ + for prop in props: + if self._re.match(prop): + return True + return False + + def __str__(self): + return self._expr + +class Term: + """A list of expressions each of which must match with properties. + + This provides a list of 'AND' expressions, meaning that each must + match the board properties for that board to be built. + """ + def __init__(self): + self._expr_list = [] + self._board_count = 0 + + def add_expr(self, expr): + """Add an Expr object to the list to check. + + Args: + expr (Expr): New Expr object to add to the list of those that must + match for a board to be built. + """ + self._expr_list.append(Expr(expr)) + + def __str__(self): + """Return some sort of useful string describing the term""" + return '&'.join([str(expr) for expr in self._expr_list]) + + def matches(self, props): + """Check if any of the properties match this term + + Each of the expressions in the term is checked. All must match. + + Args: + props (list of str): List of properties to check + Returns: + True if all of the expressions in the Term match, else False + """ + for expr in self._expr_list: + if not expr.matches(props): + return False + return True + + +class KconfigScanner: + + """Kconfig scanner.""" + + ### constant variable only used in this class ### + _SYMBOL_TABLE = { + 'arch' : 'SYS_ARCH', + 'cpu' : 'SYS_CPU', + 'soc' : 'SYS_SOC', + 'vendor' : 'SYS_VENDOR', + 'board' : 'SYS_BOARD', + 'config' : 'SYS_CONFIG_NAME', + # 'target' is added later + } + + def __init__(self): + """Scan all the Kconfig files and create a Kconfig object.""" + # Define environment variables referenced from Kconfig + os.environ['srctree'] = os.getcwd() + os.environ['UBOOTVERSION'] = 'dummy' + os.environ['KCONFIG_OBJDIR'] = '' + self._tmpfile = None + self._conf = kconfiglib.Kconfig(warn=False) + + def __del__(self): + """Delete a leftover temporary file before exit. + + The scan() method of this class creates a temporay file and deletes + it on success. If scan() method throws an exception on the way, + the temporary file might be left over. In that case, it should be + deleted in this destructor. + """ + if self._tmpfile: + try_remove(self._tmpfile) + + def scan(self, defconfig): + """Load a defconfig file to obtain board parameters. + + Args: + defconfig (str): path to the defconfig file to be processed + + Returns: + A dictionary of board parameters. It has a form of: + { + 'arch': <arch_name>, + 'cpu': <cpu_name>, + 'soc': <soc_name>, + 'vendor': <vendor_name>, + 'board': <board_name>, + 'target': <target_name>, + 'config': <config_header_name>, + } + """ + # strip special prefixes and save it in a temporary file + outfd, self._tmpfile = tempfile.mkstemp() + with os.fdopen(outfd, 'w') as outf: + with open(defconfig, encoding='utf-8') as inf: + for line in inf: + colon = line.find(':CONFIG_') + if colon == -1: + outf.write(line) + else: + outf.write(line[colon + 1:]) + + self._conf.load_config(self._tmpfile) + try_remove(self._tmpfile) + self._tmpfile = None + + params = {} + + # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc. + # Set '-' if the value is empty. + for key, symbol in list(self._SYMBOL_TABLE.items()): + value = self._conf.syms.get(symbol).str_value + if value: + params[key] = value + else: + params[key] = '-' + + defconfig = os.path.basename(defconfig) + params['target'], match, rear = defconfig.partition('_defconfig') + assert match and not rear, f'{defconfig} : invalid defconfig' + + # fix-up for aarch64 + if params['arch'] == 'arm' and params['cpu'] == 'armv8': + params['arch'] = 'aarch64' + + return params + + +class MaintainersDatabase: + + """The database of board status and maintainers. + + Properties: + database: dict: + key: Board-target name (e.g. 'snow') + value: tuple: + str: Board status (e.g. 'Active') + str: List of maintainers, separated by : + warnings (list of str): List of warnings due to missing status, etc. + """ + + def __init__(self): + """Create an empty database.""" + self.database = {} + self.warnings = [] + + def get_status(self, target): + """Return the status of the given board. + + The board status is generally either 'Active' or 'Orphan'. + Display a warning message and return '-' if status information + is not found. + + Args: + target (str): Build-target name + + Returns: + str: 'Active', 'Orphan' or '-'. + """ + if not target in self.database: + self.warnings.append(f"WARNING: no status info for '{target}'") + return '-' + + tmp = self.database[target][0] + if tmp.startswith('Maintained'): + return 'Active' + if tmp.startswith('Supported'): + return 'Active' + if tmp.startswith('Orphan'): + return 'Orphan' + self.warnings.append(f"WARNING: {tmp}: unknown status for '{target}'") + return '-' + + def get_maintainers(self, target): + """Return the maintainers of the given board. + + Args: + target (str): Build-target name + + Returns: + str: Maintainers of the board. If the board has two or more + maintainers, they are separated with colons. + """ + if not target in self.database: + self.warnings.append(f"WARNING: no maintainers for '{target}'") + return '' + + return ':'.join(self.database[target][1]) + + def parse_file(self, fname): + """Parse a MAINTAINERS file. + + Parse a MAINTAINERS file and accumulate board status and maintainers + information in the self.database dict. + + Args: + fname (str): MAINTAINERS file to be parsed + """ + targets = [] + maintainers = [] + status = '-' + with open(fname, encoding="utf-8") as inf: + for line in inf: + # Check also commented maintainers + if line[:3] == '#M:': + line = line[1:] + tag, rest = line[:2], line[2:].strip() + if tag == 'M:': + maintainers.append(rest) + elif tag == 'F:': + # expand wildcard and filter by 'configs/*_defconfig' + for item in glob.glob(rest): + front, match, rear = item.partition('configs/') + if not front and match: + front, match, rear = rear.rpartition('_defconfig') + if match and not rear: + targets.append(front) + elif tag == 'S:': + status = rest + elif line == '\n': + for target in targets: + self.database[target] = (status, maintainers) + targets = [] + maintainers = [] + status = '-' + if targets: + for target in targets: + self.database[target] = (status, maintainers) + + +class Boards: + """Manage a list of boards.""" + def __init__(self): + self._boards = [] + + def add_board(self, brd): + """Add a new board to the list. + + The board's target member must not already exist in the board list. + + Args: + brd (Board): board to add + """ + self._boards.append(brd) + + def read_boards(self, fname): + """Read a list of boards from a board file. + + Create a Board object for each and add it to our _boards list. + + Args: + fname (str): Filename of boards.cfg file + """ + with open(fname, 'r', encoding='utf-8') as inf: + for line in inf: + if line[0] == '#': + continue + fields = line.split() + if not fields: + continue + for upto, field in enumerate(fields): + if field == '-': + fields[upto] = '' + while len(fields) < 8: + fields.append('') + if len(fields) > 8: + fields = fields[:8] + + brd = board.Board(*fields) + self.add_board(brd) + + + def get_list(self): + """Return a list of available boards. + + Returns: + List of Board objects + """ + return self._boards + + def get_dict(self): + """Build a dictionary containing all the boards. + + Returns: + Dictionary: + key is board.target + value is board + """ + board_dict = OrderedDict() + for brd in self._boards: + board_dict[brd.target] = brd + return board_dict + + def get_selected_dict(self): + """Return a dictionary containing the selected boards + + Returns: + List of Board objects that are marked selected + """ + board_dict = OrderedDict() + for brd in self._boards: + if brd.build_it: + board_dict[brd.target] = brd + return board_dict + + def get_selected(self): + """Return a list of selected boards + + Returns: + List of Board objects that are marked selected + """ + return [brd for brd in self._boards if brd.build_it] + + def get_selected_names(self): + """Return a list of selected boards + + Returns: + List of board names that are marked selected + """ + return [brd.target for brd in self._boards if brd.build_it] + + @classmethod + def _build_terms(cls, args): + """Convert command line arguments to a list of terms. + + This deals with parsing of the arguments. It handles the '&' + operator, which joins several expressions into a single Term. + + For example: + ['arm & freescale sandbox', 'tegra'] + + will produce 3 Terms containing expressions as follows: + arm, freescale + sandbox + tegra + + The first Term has two expressions, both of which must match for + a board to be selected. + + Args: + args (list of str): List of command line arguments + + Returns: + list of Term: A list of Term objects + """ + syms = [] + for arg in args: + for word in arg.split(): + sym_build = [] + for term in word.split('&'): + if term: + sym_build.append(term) + sym_build.append('&') + syms += sym_build[:-1] + terms = [] + term = None + oper = None + for sym in syms: + if sym == '&': + oper = sym + elif oper: + term.add_expr(sym) + oper = None + else: + if term: + terms.append(term) + term = Term() + term.add_expr(sym) + if term: + terms.append(term) + return terms + + def select_boards(self, args, exclude=None, brds=None): + """Mark boards selected based on args + + Normally either boards (an explicit list of boards) or args (a list of + terms to match against) is used. It is possible to specify both, in + which case they are additive. + + If brds and args are both empty, all boards are selected. + + Args: + args (list of str): List of strings specifying boards to include, + either named, or by their target, architecture, cpu, vendor or + soc. If empty, all boards are selected. + exclude (list of str): List of boards to exclude, regardless of + 'args', or None for none + brds (list of Board): List of boards to build, or None/[] for all + + Returns: + Tuple + Dictionary which holds the list of boards which were selected + due to each argument, arranged by argument. + List of errors found + """ + def _check_board(brd): + """Check whether to include or exclude a board + + Checks the various terms and decide whether to build it or not (the + 'build_it' variable). + + If it is built, add the board to the result[term] list so we know + which term caused it to be built. Add it to result['all'] also. + + Keep a list of boards we found in 'found', so we can report boards + which appear in self._boards but not in brds. + + Args: + brd (Board): Board to check + """ + matching_term = None + build_it = False + if terms: + for term in terms: + if term.matches(brd.props): + matching_term = str(term) + build_it = True + break + elif brds: + if brd.target in brds: + build_it = True + found.append(brd.target) + else: + build_it = True + + # Check that it is not specifically excluded + for expr in exclude_list: + if expr.matches(brd.props): + build_it = False + break + + if build_it: + brd.build_it = True + if matching_term: + result[matching_term].append(brd.target) + result['all'].append(brd.target) + + result = OrderedDict() + warnings = [] + terms = self._build_terms(args) + + result['all'] = [] + for term in terms: + result[str(term)] = [] + + exclude_list = [] + if exclude: + for expr in exclude: + exclude_list.append(Expr(expr)) + + found = [] + for brd in self._boards: + _check_board(brd) + + if brds: + remaining = set(brds) - set(found) + if remaining: + warnings.append(f"Boards not found: {', '.join(remaining)}\n") + + return result, warnings + + @classmethod + def scan_defconfigs_for_multiprocess(cls, queue, defconfigs): + """Scan defconfig files and queue their board parameters + + This function is intended to be passed to multiprocessing.Process() + constructor. + + Args: + queue (multiprocessing.Queue): The resulting board parameters are + written into this. + defconfigs (sequence of str): A sequence of defconfig files to be + scanned. + """ + kconf_scanner = KconfigScanner() + for defconfig in defconfigs: + queue.put(kconf_scanner.scan(defconfig)) + + @classmethod + def read_queues(cls, queues, params_list): + """Read the queues and append the data to the paramers list""" + for que in queues: + while not que.empty(): + params_list.append(que.get()) + + def scan_defconfigs(self, jobs=1): + """Collect board parameters for all defconfig files. + + This function invokes multiple processes for faster processing. + + Args: + jobs (int): The number of jobs to run simultaneously + """ + all_defconfigs = [] + for (dirpath, _, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + all_defconfigs.append(os.path.join(dirpath, filename)) + + total_boards = len(all_defconfigs) + processes = [] + queues = [] + for i in range(jobs): + defconfigs = all_defconfigs[total_boards * i // jobs : + total_boards * (i + 1) // jobs] + que = multiprocessing.Queue(maxsize=-1) + proc = multiprocessing.Process( + target=self.scan_defconfigs_for_multiprocess, + args=(que, defconfigs)) + proc.start() + processes.append(proc) + queues.append(que) + + # The resulting data should be accumulated to this list + params_list = [] + + # Data in the queues should be retrieved preriodically. + # Otherwise, the queues would become full and subprocesses would get stuck. + while any(p.is_alive() for p in processes): + self.read_queues(queues, params_list) + # sleep for a while until the queues are filled + time.sleep(SLEEP_TIME) + + # Joining subprocesses just in case + # (All subprocesses should already have been finished) + for proc in processes: + proc.join() + + # retrieve leftover data + self.read_queues(queues, params_list) + + return params_list + + @classmethod + def insert_maintainers_info(cls, params_list): + """Add Status and Maintainers information to the board parameters list. + + Args: + params_list (list of dict): A list of the board parameters + + Returns: + list of str: List of warnings collected due to missing status, etc. + """ + database = MaintainersDatabase() + for (dirpath, _, filenames) in os.walk('.'): + if 'MAINTAINERS' in filenames: + database.parse_file(os.path.join(dirpath, 'MAINTAINERS')) + + for i, params in enumerate(params_list): + target = params['target'] + params['status'] = database.get_status(target) + params['maintainers'] = database.get_maintainers(target) + params_list[i] = params + return database.warnings + + @classmethod + def format_and_output(cls, params_list, output): + """Write board parameters into a file. + + Columnate the board parameters, sort lines alphabetically, + and then write them to a file. + + Args: + params_list (list of dict): The list of board parameters + output (str): The path to the output file + """ + fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target', + 'config', 'maintainers') + + # First, decide the width of each column + max_length = {f: 0 for f in fields} + for params in params_list: + for field in fields: + max_length[field] = max(max_length[field], len(params[field])) + + output_lines = [] + for params in params_list: + line = '' + for field in fields: + # insert two spaces between fields like column -t would + line += ' ' + params[field].ljust(max_length[field]) + output_lines.append(line.strip()) + + # ignore case when sorting + output_lines.sort(key=str.lower) + + with open(output, 'w', encoding="utf-8") as outf: + outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') + + def ensure_board_list(self, output, jobs=1, force=False, quiet=False): + """Generate a board database file if needed. + + Args: + output (str): The name of the output file + jobs (int): The number of jobs to run simultaneously + force (bool): Force to generate the output even if it is new + quiet (bool): True to avoid printing a message if nothing needs doing + + Returns: + bool: True if all is well, False if there were warnings + """ + if not force and output_is_new(output): + if not quiet: + print(f'{output} is up to date. Nothing to do.') + return True + params_list = self.scan_defconfigs(jobs) + warnings = self.insert_maintainers_info(params_list) + for warn in warnings: + print(warn, file=sys.stderr) + self.format_and_output(params_list, output) + return not warnings |