diff --git a/lib/functions/cli/cli-configdump.sh b/lib/functions/cli/cli-configdump.sh index fb7ccb61d..bfb323742 100644 --- a/lib/functions/cli/cli-configdump.sh +++ b/lib/functions/cli/cli-configdump.sh @@ -7,22 +7,23 @@ # This file is a part of the Armbian Build Framework # https://github.com/armbian/build/ -function cli_config_dump_pre_run() { - declare -g CONFIG_DEFS_ONLY='yes' +function cli_config_dump_json_pre_run() { + declare -g -r CONFIG_DEFS_ONLY='yes' # @TODO: This is actually too late (early optimizations in logging etc), so callers should also set it in the environment when using CLI. sorry. } -function cli_config_dump_run() { +function cli_config_dump_json_run() { # configuration etc - it initializes the extension manager - do_capturing_defs config_and_remove_useless < /dev/null # this sets CAPTURED_VARS; the < /dev/null is take away the terminal from stdin - echo "${CAPTURED_VARS}" # to stdout! + do_capturing_defs config_board_and_remove_useless < /dev/null # this sets CAPTURED_VARS_NAMES and CAPTURED_VARS_ARRAY; the < /dev/null is take away the terminal from stdin + + # convert to JSON, using python helper; each var is passed via a command line argument; that way we avoid newline/nul-char separation issues + python3 "${SRC}/lib/tools/configdump2json.py" "--args" "${CAPTURED_VARS_ARRAY[@]}" # to stdout + + return 0 } -function config_and_remove_useless() { - do_logging=no prep_conf_main_build_single # avoid logging during configdump; it's useless +function config_board_and_remove_useless() { + skip_host_config=yes use_board=yes skip_kernel=no do_logging=no prep_conf_main_minimal_ni # avoid logging during configdump; it's useless; skip host config unset FINALDEST - unset HOOK_ORDER HOOK_POINT HOOK_POINT_TOTAL_FUNCS - unset REPO_CONFIG REPO_STORAGE unset DEB_STORAGE - unset RKBIN_DIR unset ROOTPWD } diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh index 21cad2720..38d264d55 100644 --- a/lib/functions/cli/commands.sh +++ b/lib/functions/cli/commands.sh @@ -19,10 +19,12 @@ function armbian_register_commands() { ["requirements"]="requirements" # implemented in cli_requirements_pre_run and cli_requirements_run - ["config-dump"]="config_dump" # implemented in cli_config_dump_pre_run and cli_config_dump_run - ["configdump"]="config_dump" # idem + # Given a board/config/exts, dump out the (non-userspace) JSON of configuration + ["configdump"]="config_dump_json" # implemented in cli_config_dump_json_pre_run and cli_config_dump_json_run + ["config-dump-json"]="config_dump_json" # implemented in cli_config_dump_json_pre_run and cli_config_dump_json_run - ["json-info"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run + ["json-info-boards"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run + ["write-all-boards-branches-json"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["kernel-patches-to-git"]="patch_kernel" # implemented in cli_patch_kernel_pre_run and cli_patch_kernel_run @@ -67,9 +69,6 @@ function armbian_register_commands() { ["generate-dockerfile"]="DOCKERFILE_GENERATE_ONLY='yes'" - ["config-dump"]="CONFIG_DEFS_ONLY='yes'" - ["configdump"]="CONFIG_DEFS_ONLY='yes'" - # artifact shortcuts ["kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ARTIFACT_BUILD_INTERACTIVE='yes' ARTIFACT_IGNORE_CACHE='yes' ${common_cli_artifact_vars}" ["kernel"]="WHAT='kernel' ${common_cli_artifact_vars}" diff --git a/lib/functions/main/config-prepare.sh b/lib/functions/main/config-prepare.sh index 6748f41cc..f33658c07 100644 --- a/lib/functions/main/config-prepare.sh +++ b/lib/functions/main/config-prepare.sh @@ -51,8 +51,10 @@ function prep_conf_main_minimal_ni() { # needed LOG_SECTION="config_early_init" do_with_conditional_logging config_early_init - # needed - check_basic_host + # needed for most stuff, but not for configdump + if [[ "${skip_host_config:-"no"}" != "yes" ]]; then + check_basic_host + fi # needed for BOARD= builds. if [[ "${use_board:-"no"}" == "yes" ]]; then @@ -122,11 +124,18 @@ function config_source_board_file() { # Sanity check: if no board config was sourced, then the board name is invalid [[ ${#sourced_board_configs[@]} -eq 0 ]] && exit_with_error "No such BOARD '${BOARD}'; no board config file found." - LINUXFAMILY="${BOARDFAMILY}" # @TODO: wtf? why? this is (100%?) rewritten by family config! - # this sourced the board config. do_main_configuration will source the family file. + # Otherwise publish it as readonly global + declare -g -r SOURCED_BOARD_CONFIGS_FILENAME_LIST="${sourced_board_configs[*]}" - # Lets make some variables readonly. + # this is (100%?) rewritten by family config! + # answer: this defaults LINUXFAMILY to BOARDFAMILY. that... shouldn't happen, extensions might change it too. + # @TODO: better to check for empty after sourcing family config and running extensions, *warning about it*, and only then default to BOARDFAMILY. + # this sourced the board config. do_main_configuration will source the (BOARDFAMILY) family file. + LINUXFAMILY="${BOARDFAMILY}" + + # Lets make some variables readonly after sourcing the board file. # We don't want anything changing them, it's exclusively for board config. + # @TODO: ok but then we need some way to add packages simply via command line or config. ADD_PACKAGES_IMAGE="foo,bar"? declare -g -r PACKAGE_LIST_BOARD="${PACKAGE_LIST_BOARD}" declare -g -r PACKAGE_LIST_BOARD_REMOVE="${PACKAGE_LIST_BOARD_REMOVE}" diff --git a/lib/tools/common/bash_declare_parser.py b/lib/tools/common/bash_declare_parser.py new file mode 100644 index 000000000..b8853822e --- /dev/null +++ b/lib/tools/common/bash_declare_parser.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2022-2023 Ricardo Pardini +# +# This file is a part of the Armbian Build Framework +# https://github.com/armbian/build/ +# +import logging +import re + +log: logging.Logger = logging.getLogger("bash_declare_parser") + +REGEX_BASH_DECLARE_DOUBLE_QUOTE = r"declare (-[-xr]) (.*?)=\"(.*)\"" +REGEX_BASH_DECLARE_SINGLE_QUOTE = r"declare (-[-xr]) (.*?)=\$'(.*)'" + + +class BashDeclareParser: + def __init__(self, origin: str = 'unknown'): + self.origin = origin + + def parse_one(self, one_declare): + all_keys = {} + count_matches = 0 + + # Now parse it with regex-power! it only parses non-array, non-dictionary values, double-quoted. + for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_DOUBLE_QUOTE, one_declare, re.DOTALL), start=1): + count_matches += 1 + value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_double_quoted(match.group(3))) + all_keys[match.group(2)] = value + + if count_matches == 0: + # try for the single-quoted version + for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_SINGLE_QUOTE, one_declare, re.DOTALL), start=1): + count_matches += 1 + value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_single_quoted(match.group(3))) + all_keys[match.group(2)] = value + + if count_matches == 0: + log.error(f"** No matches found for Bash declare regex (origin: {self.origin}), line ==>{one_declare}<==") + + return all_keys + + def parse_dequoted_value(self, key, value): + if ("_LIST" in key) or ("_DIRS" in key) or ("_ARRAY" in key): + value = self.armbian_value_parse_list(value, " ") + return value + + def armbian_value_parse_double_quoted(self, value: str): + # replace "\\\\n" with actual newline + value = value.replace('\\\\n', "\n") + value = value.replace('\\\\t', "\t") + value = value.replace('\\\"', '"') + return value + + def armbian_value_parse_single_quoted(self, value: str): + value = value.replace('\\n', "\n") + value = value.replace('\n', "\n") + value = value.replace('\\t', "\t") + value = value.replace('\t', "\t") + return value + + def armbian_value_parse_list(self, item_value, delimiter): + ret = [] + for item in item_value.split(delimiter): + ret.append((item)) + # trim whitespace out of every value + ret = list(map(str.strip, ret)) + # filter out empty strings + ret = list(filter(None, ret)) + return ret + + def armbian_value_parse_newline_map(self, item_value): + lines = item_value.split("\n") + ret = [] + for line in lines: + ret.append(self.armbian_value_parse_list(line, ":")) + return ret diff --git a/lib/tools/configdump2json.py b/lib/tools/configdump2json.py new file mode 100644 index 000000000..e11341226 --- /dev/null +++ b/lib/tools/configdump2json.py @@ -0,0 +1,19 @@ +import json +import sys + +from common.bash_declare_parser import BashDeclareParser + +mode = sys.argv[1] + +parser = BashDeclareParser() + +if mode == "--args": + # loop over argv, parse one by one + everything = {} + for arg in sys.argv[2:]: + parsed = parser.parse_one(arg) + everything.update(parsed) + # print(json.dumps(everything, indent=4)) # multiline, indented + print(json.dumps(everything, separators=(',', ':'))) # single line, no indent, compact +else: + raise Exception(f"Unknown mode '{mode}'") diff --git a/lib/tools/info.py b/lib/tools/info.py index 1039ead0e..ccd553bf1 100755 --- a/lib/tools/info.py +++ b/lib/tools/info.py @@ -10,6 +10,7 @@ import concurrent.futures import glob import json +import multiprocessing import os import re import subprocess @@ -32,27 +33,6 @@ def get_all_boards_list_from_armbian(src_path): return ret -def armbian_value_parse_simple(value, armbian_src_path): - # return value.replace(armbian_src_path, "${SRC}") - return value - - -def armbian_value_parse_list(item_value, delimiter, armbian_src_path): - # return map(lambda x: armbian_value_parse_simple(x, armbian_src_path), item_value.split()) - ret = [] - for item in item_value.split(delimiter): - ret.append(armbian_value_parse_simple(item, armbian_src_path)) - return ret - - -def armbian_value_parse_newline_map(item_value, armbian_src_path): - lines = item_value.split("\n") - ret = [] - for line in lines: - ret.append(armbian_value_parse_list(line, ":", armbian_src_path)) - return ret - - def map_to_armbian_params(map_params): ret = [] for param in map_params: @@ -61,59 +41,42 @@ def map_to_armbian_params(map_params): def run_armbian_compile_and_parse(path_to_compile_sh, armbian_src_path, compile_params): - exec_cmd = ([path_to_compile_sh] + ["config-dump"] + map_to_armbian_params(compile_params)) + exec_cmd = ([path_to_compile_sh] + ["config-dump-json"] + map_to_armbian_params(compile_params)) # eprint("Running command: '{}' ", exec_cmd) result = None logs = ["Not available"] try: result = subprocess.run( exec_cmd, - stdout=subprocess.PIPE, check=True, universal_newlines=True, + stdout=subprocess.PIPE, + check=True, + universal_newlines=False, # universal_newlines messes up bash encoding, don't use, instead decode utf8 manually; + bufsize=-1, # full buffering + # Early (pre-param-parsing) optimizations for those in Armbian bash code, so use an ENV (not PARAM) env={ "CONFIG_DEFS_ONLY": "yes", # Dont do anything. Just output vars. - "ANSI_COLOR": "none", # Do not use ANSI colors in logging output + "ANSI_COLOR": "none", # Do not use ANSI colors in logging output, don't write to log files "WRITE_EXTENSIONS_METADATA": "no" # Not interested in ext meta here }, stderr=subprocess.PIPE ) except subprocess.CalledProcessError as e: - lines_stderr = e.stderr.split("\n") - eprint( - "Error calling Armbian: params: {}, return code: {}, stderr: {}".format( - compile_params, e.returncode, - # the last 5 elements of lines_stderr, joined - "; ".join(lines_stderr[-5:]) - ) - ) + # decode utf8 manually, universal_newlines messes up bash encoding + lines_stderr = e.stderr.decode("utf8").split("\n") + eprint("Error calling Armbian: params: {}, return code: {}, stderr: {}".format(compile_params, e.returncode, "; ".join(lines_stderr[-5:]))) return {"in": compile_params, "out": {}, "logs": lines_stderr, "config_ok": False} if result is not None: if result.stderr: - # parse list, split by newline, remove armbian_src_path - logs = armbian_value_parse_list(result.stderr, "\n", armbian_src_path) + # parse list, split by newline + lines = result.stderr.decode("utf8").split("\n") + # trim lines, remove empty ones + logs = [line.strip() for line in lines if line.strip()] - # Now parse it with regex-power! - # regex = r"^declare (..) (.*?)=\"(.*?)\"$" # old multiline version - regex = r"declare (..) (.*?)=\"(.*?)\"" - test_str = result.stdout - matches = re.finditer(regex, test_str, re.DOTALL | re.MULTILINE) - all_keys = {} + # parse the result.stdout as json + parsed = json.loads(result.stdout.decode("utf8")) - for matchNum, match in enumerate(matches, start=1): - flags = match.group(1) - key = match.group(2) - value = match.group(3) - - if ("_LIST" in key) or ("_DIRS" in key): - value = armbian_value_parse_list(value, " ", armbian_src_path) - elif "_TARGET_MAP" in key: - value = armbian_value_parse_newline_map(value, armbian_src_path) - else: - value = armbian_value_parse_simple(value, armbian_src_path) - - all_keys[key] = value - - info = {"in": compile_params, "out": all_keys, "config_ok": True} + info = {"in": compile_params, "out": parsed, "config_ok": True} # info["logs"] = logs return info @@ -133,20 +96,9 @@ if not os.path.exists(compile_sh_full_path): raise Exception("Can't find compile.sh") common_compile_params = { - "BUILD_MINIMAL": "no", - # "DEB_COMPRESS": "none", - # "CLOUD_IMAGE": "yes", - # "CLEAN_LEVEL": "debs", - # "SHOW_LOG": "yes", - # "SKIP_EXTERNAL_TOOLCHAINS": "yes", - # "CONFIG_DEFS_ONLY": "yes", - "KERNEL_CONFIGURE": "no", - # "EXPERT": "yes" } board_compile_params = { - "RELEASE": "jammy", - "BUILD_DESKTOP": "no" } @@ -184,9 +136,7 @@ def get_info_for_one_board(board_file, board_name, common_params, board_info, br # eprint("Running Armbian bash for board '{}'".format(board_name)) try: - parsed = run_armbian_compile_and_parse(compile_sh_full_path, armbian_src_path, - common_params | {"BOARD": board_name}) - # print(json.dumps(parsed, indent=4, sort_keys=True)) + parsed = run_armbian_compile_and_parse(compile_sh_full_path, armbian_src_path, common_params | {"BOARD": board_name}) return parsed | board_info except BaseException as e: eprint("Failed get info for board '{}': '{}'".format(board_name, e)) @@ -209,15 +159,17 @@ if True: raise e # now loop over gathered infos every_info = [] - with concurrent.futures.ProcessPoolExecutor() as executor: # max_workers=32 + # get the number of processor cores on this machine + max_workers = multiprocessing.cpu_count() * 2 # use double the number of cpu cores, that's the sweet spot + eprint(f"Using {max_workers} workers for parallel processing.") + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: every_future = [] for board in all_boards.keys(): board_info = info_for_board[board] for possible_branch in board_info["BOARD_POSSIBLE_BRANCHES"]: all_params = common_compile_params | board_compile_params | {"BRANCH": possible_branch} # eprint("Submitting future for board {} with BRANCH={}".format(board, possible_branch)) - future = executor.submit(get_info_for_one_board, all_boards[board], board, all_params, - board_info, possible_branch) + future = executor.submit(get_info_for_one_board, all_boards[board], board, all_params, board_info, possible_branch) every_future.append(future) eprint(f"Waiting for all {len(every_future)} configurations to be computed... this might take a long time.")