configdump/json-info-boards: revamp, all-JSON now; use_board=yes skip_kernel=no for config; refactor & use new Python bash-declare-to-JSON utility

- use new capture'd vars scheme
- so `./compile.sh BOARD=xxx BRANCH=yyyy config-dump-json | jq .` now works and is consistent/newline tolerant
- introduce internal `skip_host_config=yes` for `prep_conf_main_minimal_ni()` to skip calling `check_basic_host()`

lib/functions/cli/cli-configdump.sh

@@ -7,22 +7,23 @@
 # This file is a part of the Armbian Build Framework
 # https://github.com/armbian/build/
 
-function cli_config_dump_pre_run() {
+function cli_config_dump_json_pre_run() {
-	declare -g CONFIG_DEFS_ONLY='yes'
+	declare -g -r CONFIG_DEFS_ONLY='yes' # @TODO: This is actually too late (early optimizations in logging etc), so callers should also set it in the environment when using CLI. sorry.
 }
 
-function cli_config_dump_run() {
+function cli_config_dump_json_run() {
 	# configuration etc - it initializes the extension manager
-	do_capturing_defs config_and_remove_useless < /dev/null # this sets CAPTURED_VARS; the < /dev/null is take away the terminal from stdin
+	do_capturing_defs config_board_and_remove_useless < /dev/null # this sets CAPTURED_VARS_NAMES and CAPTURED_VARS_ARRAY; the < /dev/null is take away the terminal from stdin
-	echo "${CAPTURED_VARS}"                                 # to stdout!
+
+	# convert to JSON, using python helper; each var is passed via a command line argument; that way we avoid newline/nul-char separation issues
+	python3 "${SRC}/lib/tools/configdump2json.py" "--args" "${CAPTURED_VARS_ARRAY[@]}" # to stdout
+
+	return 0
 }
 
-function config_and_remove_useless() {
+function config_board_and_remove_useless() {
-	do_logging=no prep_conf_main_build_single # avoid logging during configdump; it's useless
+	skip_host_config=yes use_board=yes skip_kernel=no do_logging=no prep_conf_main_minimal_ni # avoid logging during configdump; it's useless; skip host config
 	unset FINALDEST
-	unset HOOK_ORDER HOOK_POINT HOOK_POINT_TOTAL_FUNCS
-	unset REPO_CONFIG REPO_STORAGE
 	unset DEB_STORAGE
-	unset RKBIN_DIR
 	unset ROOTPWD
 }

lib/functions/cli/commands.sh

@@ -19,10 +19,12 @@ function armbian_register_commands() {
 
 		["requirements"]="requirements" # implemented in cli_requirements_pre_run and cli_requirements_run
 
-		["config-dump"]="config_dump" # implemented in cli_config_dump_pre_run and cli_config_dump_run
+		# Given a board/config/exts, dump out the (non-userspace) JSON of configuration
-		["configdump"]="config_dump"  # idem
+		["configdump"]="config_dump_json"              # implemented in cli_config_dump_json_pre_run and cli_config_dump_json_run
+		["config-dump-json"]="config_dump_json"        # implemented in cli_config_dump_json_pre_run and cli_config_dump_json_run
 
-		["json-info"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
+		["json-info-boards"]="json_info"               # implemented in cli_json_info_pre_run and cli_json_info_run
+		["write-all-boards-branches-json"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
 
 		["kernel-patches-to-git"]="patch_kernel" # implemented in cli_patch_kernel_pre_run and cli_patch_kernel_run
 
@@ -67,9 +69,6 @@ function armbian_register_commands() {
 
 		["generate-dockerfile"]="DOCKERFILE_GENERATE_ONLY='yes'"
 
-		["config-dump"]="CONFIG_DEFS_ONLY='yes'"
-		["configdump"]="CONFIG_DEFS_ONLY='yes'"
-
 		# artifact shortcuts
 		["kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ARTIFACT_BUILD_INTERACTIVE='yes' ARTIFACT_IGNORE_CACHE='yes' ${common_cli_artifact_vars}"
 		["kernel"]="WHAT='kernel' ${common_cli_artifact_vars}"

lib/functions/main/config-prepare.sh

@@ -51,8 +51,10 @@ function prep_conf_main_minimal_ni() {
 	# needed
 	LOG_SECTION="config_early_init" do_with_conditional_logging config_early_init
 
-	# needed
+	# needed for most stuff, but not for configdump
-	check_basic_host
+	if [[ "${skip_host_config:-"no"}" != "yes" ]]; then
+		check_basic_host
+	fi
 
 	# needed for BOARD= builds.
 	if [[ "${use_board:-"no"}" == "yes" ]]; then
@@ -122,11 +124,18 @@ function config_source_board_file() {
 	# Sanity check: if no board config was sourced, then the board name is invalid
 	[[ ${#sourced_board_configs[@]} -eq 0 ]] && exit_with_error "No such BOARD '${BOARD}'; no board config file found."
 
-	LINUXFAMILY="${BOARDFAMILY}" # @TODO: wtf? why? this is (100%?) rewritten by family config!
+	# Otherwise publish it as readonly global
-	# this sourced the board config. do_main_configuration will source the family file.
+	declare -g -r SOURCED_BOARD_CONFIGS_FILENAME_LIST="${sourced_board_configs[*]}"
 
-	# Lets make some variables readonly.
+	# this is (100%?) rewritten by family config!
+	# answer: this defaults LINUXFAMILY to BOARDFAMILY. that... shouldn't happen, extensions might change it too.
+	# @TODO: better to check for empty after sourcing family config and running extensions, *warning about it*, and only then default to BOARDFAMILY.
+	# this sourced the board config. do_main_configuration will source the (BOARDFAMILY) family file.
+	LINUXFAMILY="${BOARDFAMILY}"
+
+	# Lets make some variables readonly after sourcing the board file.
 	# We don't want anything changing them, it's exclusively for board config.
+	# @TODO: ok but then we need some way to add packages simply via command line or config. ADD_PACKAGES_IMAGE="foo,bar"?
 	declare -g -r PACKAGE_LIST_BOARD="${PACKAGE_LIST_BOARD}"
 	declare -g -r PACKAGE_LIST_BOARD_REMOVE="${PACKAGE_LIST_BOARD_REMOVE}"
 

lib/tools/common/bash_declare_parser.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2022-2023 Ricardo Pardini <ricardo@pardini.net>
+#
+# This file is a part of the Armbian Build Framework
+# https://github.com/armbian/build/
+#
+import logging
+import re
+
+log: logging.Logger = logging.getLogger("bash_declare_parser")
+
+REGEX_BASH_DECLARE_DOUBLE_QUOTE = r"declare (-[-xr]) (.*?)=\"(.*)\""
+REGEX_BASH_DECLARE_SINGLE_QUOTE = r"declare (-[-xr]) (.*?)=\$'(.*)'"
+
+
+class BashDeclareParser:
+	def __init__(self, origin: str = 'unknown'):
+		self.origin = origin
+
+	def parse_one(self, one_declare):
+		all_keys = {}
+		count_matches = 0
+
+		# Now parse it with regex-power! it only parses non-array, non-dictionary values, double-quoted.
+		for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_DOUBLE_QUOTE, one_declare, re.DOTALL), start=1):
+			count_matches += 1
+			value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_double_quoted(match.group(3)))
+			all_keys[match.group(2)] = value
+
+		if count_matches == 0:
+			# try for the single-quoted version
+			for matchNum, match in enumerate(re.finditer(REGEX_BASH_DECLARE_SINGLE_QUOTE, one_declare, re.DOTALL), start=1):
+				count_matches += 1
+				value = self.parse_dequoted_value(match.group(2), self.armbian_value_parse_single_quoted(match.group(3)))
+				all_keys[match.group(2)] = value
+
+		if count_matches == 0:
+			log.error(f"** No matches found for Bash declare regex (origin: {self.origin}), line ==>{one_declare}<==")
+
+		return all_keys
+
+	def parse_dequoted_value(self, key, value):
+		if ("_LIST" in key) or ("_DIRS" in key) or ("_ARRAY" in key):
+			value = self.armbian_value_parse_list(value, " ")
+		return value
+
+	def armbian_value_parse_double_quoted(self, value: str):
+		# replace "\\\\n" with actual newline
+		value = value.replace('\\\\n', "\n")
+		value = value.replace('\\\\t', "\t")
+		value = value.replace('\\\"', '"')
+		return value
+
+	def armbian_value_parse_single_quoted(self, value: str):
+		value = value.replace('\\n', "\n")
+		value = value.replace('\n', "\n")
+		value = value.replace('\\t', "\t")
+		value = value.replace('\t', "\t")
+		return value
+
+	def armbian_value_parse_list(self, item_value, delimiter):
+		ret = []
+		for item in item_value.split(delimiter):
+			ret.append((item))
+		# trim whitespace out of every value
+		ret = list(map(str.strip, ret))
+		# filter out empty strings
+		ret = list(filter(None, ret))
+		return ret
+
+	def armbian_value_parse_newline_map(self, item_value):
+		lines = item_value.split("\n")
+		ret = []
+		for line in lines:
+			ret.append(self.armbian_value_parse_list(line, ":"))
+		return ret

lib/tools/configdump2json.py

@@ -0,0 +1,19 @@
+import json
+import sys
+
+from common.bash_declare_parser import BashDeclareParser
+
+mode = sys.argv[1]
+
+parser = BashDeclareParser()
+
+if mode == "--args":
+	# loop over argv, parse one by one
+	everything = {}
+	for arg in sys.argv[2:]:
+		parsed = parser.parse_one(arg)
+		everything.update(parsed)
+	# print(json.dumps(everything, indent=4))  # multiline, indented
+	print(json.dumps(everything, separators=(',', ':')))  # single line, no indent, compact
+else:
+	raise Exception(f"Unknown mode '{mode}'")

lib/tools/info.py

@@ -10,6 +10,7 @@
 import concurrent.futures
 import glob
 import json
+import multiprocessing
 import os
 import re
 import subprocess
@@ -32,27 +33,6 @@ def get_all_boards_list_from_armbian(src_path):
 	return ret
 
 
-def armbian_value_parse_simple(value, armbian_src_path):
-	# return value.replace(armbian_src_path, "${SRC}")
-	return value
-
-
-def armbian_value_parse_list(item_value, delimiter, armbian_src_path):
-	# return map(lambda x: armbian_value_parse_simple(x, armbian_src_path), item_value.split())
-	ret = []
-	for item in item_value.split(delimiter):
-		ret.append(armbian_value_parse_simple(item, armbian_src_path))
-	return ret
-
-
-def armbian_value_parse_newline_map(item_value, armbian_src_path):
-	lines = item_value.split("\n")
-	ret = []
-	for line in lines:
-		ret.append(armbian_value_parse_list(line, ":", armbian_src_path))
-	return ret
-
-
 def map_to_armbian_params(map_params):
 	ret = []
 	for param in map_params:
@@ -61,59 +41,42 @@ def map_to_armbian_params(map_params):
 
 
 def run_armbian_compile_and_parse(path_to_compile_sh, armbian_src_path, compile_params):
-	exec_cmd = ([path_to_compile_sh] + ["config-dump"] + map_to_armbian_params(compile_params))
+	exec_cmd = ([path_to_compile_sh] + ["config-dump-json"] + map_to_armbian_params(compile_params))
 	# eprint("Running command: '{}' ", exec_cmd)
 	result = None
 	logs = ["Not available"]
 	try:
 		result = subprocess.run(
 			exec_cmd,
-			stdout=subprocess.PIPE, check=True, universal_newlines=True,
+			stdout=subprocess.PIPE,
+			check=True,
+			universal_newlines=False,  # universal_newlines messes up bash encoding, don't use, instead decode utf8 manually;
+			bufsize=-1,  # full buffering
+			# Early (pre-param-parsing) optimizations for those in Armbian bash code, so use an ENV (not PARAM)
 			env={
 				"CONFIG_DEFS_ONLY": "yes",  # Dont do anything. Just output vars.
-				"ANSI_COLOR": "none",  # Do not use ANSI colors in logging output
+				"ANSI_COLOR": "none",  # Do not use ANSI colors in logging output, don't write to log files
 				"WRITE_EXTENSIONS_METADATA": "no"  # Not interested in ext meta here
 			},
 			stderr=subprocess.PIPE
 		)
 	except subprocess.CalledProcessError as e:
-		lines_stderr = e.stderr.split("\n")
+		# decode utf8 manually, universal_newlines messes up bash encoding
-		eprint(
+		lines_stderr = e.stderr.decode("utf8").split("\n")
-			"Error calling Armbian: params: {}, return code: {}, stderr: {}".format(
+		eprint("Error calling Armbian: params: {}, return code: {}, stderr: {}".format(compile_params, e.returncode, "; ".join(lines_stderr[-5:])))
-				compile_params, e.returncode,
-				# the last 5 elements of lines_stderr, joined
-				"; ".join(lines_stderr[-5:])
-			)
-		)
 		return {"in": compile_params, "out": {}, "logs": lines_stderr, "config_ok": False}
 
 	if result is not None:
 		if result.stderr:
-			# parse list, split by newline, remove armbian_src_path
+			# parse list, split by newline
-			logs = armbian_value_parse_list(result.stderr, "\n", armbian_src_path)
+			lines = result.stderr.decode("utf8").split("\n")
+			# trim lines, remove empty ones
+			logs = [line.strip() for line in lines if line.strip()]
 
-	# Now parse it with regex-power!
+	# parse the result.stdout as json
-	# regex = r"^declare (..) (.*?)=\"(.*?)\"$" # old multiline version
+	parsed = json.loads(result.stdout.decode("utf8"))
-	regex = r"declare (..) (.*?)=\"(.*?)\""
-	test_str = result.stdout
-	matches = re.finditer(regex, test_str, re.DOTALL | re.MULTILINE)
-	all_keys = {}
 
-	for matchNum, match in enumerate(matches, start=1):
+	info = {"in": compile_params, "out": parsed, "config_ok": True}
-		flags = match.group(1)
-		key = match.group(2)
-		value = match.group(3)
-
-		if ("_LIST" in key) or ("_DIRS" in key):
-			value = armbian_value_parse_list(value, " ", armbian_src_path)
-		elif "_TARGET_MAP" in key:
-			value = armbian_value_parse_newline_map(value, armbian_src_path)
-		else:
-			value = armbian_value_parse_simple(value, armbian_src_path)
-
-		all_keys[key] = value
-
-	info = {"in": compile_params, "out": all_keys, "config_ok": True}
 	# info["logs"] = logs
 	return info
 
@@ -133,20 +96,9 @@ if not os.path.exists(compile_sh_full_path):
 	raise Exception("Can't find compile.sh")
 
 common_compile_params = {
-	"BUILD_MINIMAL": "no",
-	# "DEB_COMPRESS": "none",
-	# "CLOUD_IMAGE": "yes",
-	# "CLEAN_LEVEL": "debs",
-	# "SHOW_LOG": "yes",
-	# "SKIP_EXTERNAL_TOOLCHAINS": "yes",
-	# "CONFIG_DEFS_ONLY": "yes",
-	"KERNEL_CONFIGURE": "no",
-	# "EXPERT": "yes"
 }
 
 board_compile_params = {
-	"RELEASE": "jammy",
-	"BUILD_DESKTOP": "no"
 }
 
 
@@ -184,9 +136,7 @@ def get_info_for_one_board(board_file, board_name, common_params, board_info, br
 
 	# eprint("Running Armbian bash for board '{}'".format(board_name))
 	try:
-		parsed = run_armbian_compile_and_parse(compile_sh_full_path, armbian_src_path,
+		parsed = run_armbian_compile_and_parse(compile_sh_full_path, armbian_src_path, common_params | {"BOARD": board_name})
-											   common_params | {"BOARD": board_name})
-		# print(json.dumps(parsed, indent=4, sort_keys=True))
 		return parsed | board_info
 	except BaseException as e:
 		eprint("Failed get info for board '{}': '{}'".format(board_name, e))
@@ -209,15 +159,17 @@ if True:
 			raise e
 	# now loop over gathered infos
 	every_info = []
-	with concurrent.futures.ProcessPoolExecutor() as executor:  # max_workers=32
+	# get the number of processor cores on this machine
+	max_workers = multiprocessing.cpu_count() * 2  # use double the number of cpu cores, that's the sweet spot
+	eprint(f"Using {max_workers} workers for parallel processing.")
+	with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
 		every_future = []
 		for board in all_boards.keys():
 			board_info = info_for_board[board]
 			for possible_branch in board_info["BOARD_POSSIBLE_BRANCHES"]:
 				all_params = common_compile_params | board_compile_params | {"BRANCH": possible_branch}
 				# eprint("Submitting future for board {} with BRANCH={}".format(board, possible_branch))
-				future = executor.submit(get_info_for_one_board, all_boards[board], board, all_params,
+				future = executor.submit(get_info_for_one_board, all_boards[board], board, all_params, board_info, possible_branch)
-										 board_info, possible_branch)
 				every_future.append(future)
 
 		eprint(f"Waiting for all {len(every_future)} configurations to be computed... this might take a long time.")