armbian-next: lib/tools - python - config definitions capture and massaging

lib/tools/index-opensearch.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import json
+import sys
+
+from opensearchpy import OpenSearch  # pip install opensearch-py
+
+
+def eprint(*args, **kwargs):
+	print(*args, file=sys.stderr, **kwargs)
+
+
+# info = get_info_for_one_board(board, all_params)
+print(json.dumps({}, indent=4, sort_keys=True))
+
+eprint("Hello")
+
+# Read JSON from stdin
+# - should be array of objects
+# - loop over array and index each obj into OS in to the passed index
+# read_from_stdin = sys.stdin.read()
+
+json_object = json.load(sys.stdin)
+
+eprint("Loaded {} objects from stdin...".format(len(json_object)))
+
+host = '192.168.66.55'
+port = 31920
+
+# Create the OpenSearch client.
+client = OpenSearch(hosts=[{'host': host, 'port': port}], http_compress=False, use_ssl=False)
+
+# Create an index with non-default settings.
+index_name = 'board-vars-build'
+index_body = {'settings': {'index': {'number_of_shards': 1, 'number_of_replicas': 0}}}
+
+# Delete the index; remove old data.
+try:
+	response = client.indices.delete(index=index_name)
+	print('\nDeleting index:')
+	print(response)
+except:
+	eprint("Failed to delete index {}".format(index_name))
+
+response = client.indices.create(index_name, body=index_body)
+print('\nCreating index:')
+print(response)
+
+for obj in json_object:
+	# print(obj)
+	response = client.index(index=index_name, body=obj, refresh=True)
+	print('\nAdding document:')
+	print(response)

lib/tools/info.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+import concurrent.futures
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def eprint(*args, **kwargs):
+	print(*args, file=sys.stderr, **kwargs)
+
+
+def armbian_value_parse_list(item_value):
+	return item_value.split()
+
+
+def get_all_boards_list_from_armbian(src_path):
+	ret = {}
+	for file in glob.glob(src_path + "/config/boards/*.*"):
+		stem = Path(file).stem
+		if stem != "README":
+			ret[stem] = file
+	# return ret
+	return ret
+
+
+def armbian_value_parse_newline_map(item_value):
+	lines = item_value.split("\n")
+	ret = []
+	for line in lines:
+		ret.append(line.split(";"))
+	return ret
+
+
+def map_to_armbian_params(map_params):
+	ret = []
+	for param in map_params:
+		ret.append(param + "=" + map_params[param])
+	return ret
+
+
+def run_armbian_compile_and_parse(path_to_compile_sh, compile_params):
+	exec_cmd = ([path_to_compile_sh] + map_to_armbian_params(compile_params))
+	result = None
+	logs = ["Not available"]
+	try:
+		result = subprocess.run(
+			exec_cmd,
+			stdout=subprocess.PIPE, check=True, universal_newlines=True,
+			env={
+				"CONFIG_DEFS_ONLY": "yes",  # Dont do anything. Just output vars.
+				"ANSI_COLOR": "none",  # Do not use ANSI colors in logging output
+				"WRITE_EXTENSIONS_METADATA": "no"  # Not interested in ext meta here
+			},
+			stderr=subprocess.PIPE
+		)
+	except subprocess.CalledProcessError as e:
+		eprint(
+			"Error calling Armbian: params: {}, return code: {}, stderr: {}".format(
+				compile_params, e.returncode, e.stderr
+			)
+		)
+		return {"in": compile_params, "out": {}, "logs": e.stderr.split("\n"), "config_ok": False}
+
+	if result is not None:
+		if result.stderr:
+			logs = result.stderr.split("\n")
+
+	# Now parse it with regex-power!
+	# regex = r"^declare (..) (.*?)=\"(.*?)\"$" # old multiline version
+	regex = r"declare (..) (.*?)=\"(.*?)\""
+	test_str = result.stdout
+	matches = re.finditer(regex, test_str, re.DOTALL | re.MULTILINE)
+	all_keys = {}
+
+	for matchNum, match in enumerate(matches, start=1):
+		flags = match.group(1)
+		key = match.group(2)
+		value = match.group(3)
+
+		if ("_LIST" in key) or ("_DIRS" in key):
+			value = armbian_value_parse_list(value)
+		elif "_TARGET_MAP" in key:
+			value = armbian_value_parse_newline_map(value)
+
+		all_keys[key] = value
+
+	return {"in": compile_params, "out": all_keys, "logs": logs, "config_ok": True}
+
+
+# Find the location of compile.sh, relative to this Python script.
+this_script_full_path = os.path.realpath(__file__)
+eprint("Real path to this script", this_script_full_path)
+
+armbian_src_path = os.path.realpath(os.path.join(os.path.dirname(this_script_full_path), "..", ".."))
+eprint("Real path to Armbian SRC", armbian_src_path)
+
+compile_sh_full_path = os.path.realpath(os.path.join(armbian_src_path, "compile.sh"))
+eprint("Real path to compile.sh", compile_sh_full_path)
+
+# Make sure it exists
+if not os.path.exists(compile_sh_full_path):
+	raise Exception("Can't find compile.sh")
+
+common_compile_params = {
+	"KERNEL_ONLY": "no",
+	"BUILD_MINIMAL": "no",
+	"DEB_COMPRESS": "none",
+	"CLOUD_IMAGE": "yes",
+	"CLEAN_LEVEL": "debs",
+	"SHOW_LOG": "yes",
+	"SKIP_EXTERNAL_TOOLCHAINS": "yes",
+	"CONFIG_DEFS_ONLY": "yes",
+	"KERNEL_CONFIGURE": "no",
+	"EXPERT": "yes"
+}
+
+board_compile_params = {
+	"RELEASE": "jammy",
+	"BUILD_DESKTOP": "no"
+}
+
+
+# I've to read the first line from the board file, that's the hardware description in a pound comment.
+# Also, 'KERNEL_TARGET="legacy,current,edge"' which we need to parse.
+def parse_board_file_for_static_info(board_file, board_id):
+	file_handle = open(board_file, 'r')
+	file_lines = file_handle.readlines()
+	file_handle.close()
+
+	file_lines.reverse()
+	hw_desc_line = file_lines.pop()
+	hw_desc_clean = hw_desc_line.strip("# ").strip("\n")
+
+	# Parse KERNEL_TARGET line.
+	kernel_target_matches = re.findall(r"^(export )?KERNEL_TARGET=\"(.*)\"", "\n".join(file_lines), re.MULTILINE)
+	kernel_targets = kernel_target_matches[0][1].split(",")
+	eprint("Possible kernel branches for board: ", board_id, " : ", kernel_targets)
+
+	return {
+		"BOARD_FILE_HARDWARE_DESC": hw_desc_clean,
+		"BOARD_POSSIBLE_BRANCHES": kernel_targets,
+		"BOARD_DESC_ID": board_id
+	}
+
+
+def get_info_for_one_board(board_file, board_name, common_params, board_info):
+	eprint(
+		"Getting info for board '{}' branch '{}' in file '{}'".format(
+			board_name, common_params["BRANCH"], board_file
+		)
+	)
+
+	# eprint("Running Armbian bash for board '{}'".format(board_name))
+	try:
+		parsed = run_armbian_compile_and_parse(compile_sh_full_path, common_params | {"BOARD": board_name})
+		# print(json.dumps(parsed, indent=4, sort_keys=True))
+		return parsed | board_info
+	except:
+		eprint("Failed get info for board '{}'".format(board_name))
+		return board_info | {"ARMBIAN_CONFIG_OK": False}
+
+
+if True:
+	all_boards = get_all_boards_list_from_armbian(armbian_src_path)
+	# eprint(json.dumps(all_boards, indent=4, sort_keys=True))
+
+	# first, gather the board_info for every board. if any fail, stop.
+	info_for_board = {}
+	for board in all_boards.keys():
+		try:
+			board_info = parse_board_file_for_static_info(all_boards[board], board)
+			info_for_board[board] = board_info
+		except BaseException as e:
+			eprint("** Failed to parse board file {} static: {}".format(board, e))
+			raise e
+	# now loop over gathered infos
+	every_info = []
+	with concurrent.futures.ProcessPoolExecutor(max_workers=32) as executor:
+		every_future = []
+		for board in all_boards.keys():
+			board_info = info_for_board[board]
+			for possible_branch in board_info["BOARD_POSSIBLE_BRANCHES"]:
+				all_params = common_compile_params | board_compile_params | {"BRANCH": possible_branch}
+				eprint("Submitting future for board {} with BRANCH={}".format(board, possible_branch))
+				future = executor.submit(get_info_for_one_board, all_boards[board], board, all_params,
+							 board_info)
+				every_future.append(future)
+
+		eprint("Waiting for all futures...")
+		executor.shutdown(wait=True)
+		eprint("Done, all futures awaited")
+
+		for future in every_future:
+			info = future.result()
+			if info is not None:
+				every_info.append(info)
+
+# info = get_info_for_one_board(board, all_params)
+print(json.dumps(every_info, indent=4, sort_keys=True))

lib/tools/json2csv.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import collections.abc
+import json
+import sys
+
+
+def eprint(*args, **kwargs):
+	print(*args, file=sys.stderr, **kwargs)
+
+
+def flatten(d, parent_key='', sep='_'):
+	items = []
+	for k, v in d.items():
+		new_key = parent_key + sep + k if parent_key else k
+		if isinstance(v, collections.abc.MutableMapping):
+			items.extend(flatten(v, new_key, sep=sep).items())
+		else:
+			items.append((new_key, v))
+	return dict(items)
+
+
+json_object = json.load(sys.stdin)
+eprint("Loaded {} objects from stdin...".format(len(json_object)))
+
+flat = []
+for obj in json_object:
+	flat.append(flatten(obj, '', '.'))
+
+columns_map = {}
+for obj in flat:
+	# get the string keys
+	for key in obj.keys():
+		value = obj[key]
+		if type(value) == str:
+			columns_map[key] = True
+		if type(value) == bool:
+			columns_map[key] = True
+
+columns = columns_map.keys()
+
+eprint("columns: {}".format(columns_map))
+
+eprint("columns: {}".format(columns))
+
+import csv
+
+with open('boards_vs_branches.csv', 'w', newline='') as csvfile:
+	fieldnames = columns
+	writer = csv.DictWriter(csvfile, fieldnames=fieldnames, extrasaction='ignore')
+
+	writer.writeheader()
+	for obj in flat:
+		writer.writerow(obj)