pipeline: add output-gha-workflow-template.py utility, for rendering GHA workflow YAML templates with chunks

- python-tools: add Jinja2. Incredible how we made it this far without it.
- output-gha-workflow-template: a double-templater, first runs jinja with a custom syntax, then "moar magic" to be useful for GHA

lib/functions/cli/cli-jsoninfo.sh

@@ -154,6 +154,19 @@ function cli_json_info_run() {
 				run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/output-gha-matrix.py images "${OUTDATED_ARTIFACTS_IMAGES_FILE}" "${MATRIX_IMAGE_CHUNKS}" ">" "${GHA_ALL_IMAGES_JSON_MATRIX_FILE}"
 			fi
 			github_actions_add_output "image-matrix" "$(cat "${GHA_ALL_IMAGES_JSON_MATRIX_FILE}")"
+
+			# If we have userpatches/gha/chunks, run the workflow template utility
+			declare user_gha_dir="${USERPATCHES_PATH}/gha"
+			declare wf_template_dir="${user_gha_dir}/chunks"
+			if [[ -d "${wf_template_dir}" ]]; then
+				display_alert "Generating GHA workflow template" "output-gha-workflow-template :: ${wf_template_dir}" "info"
+				declare GHA_WORKFLOW_TEMPLATE_OUT_FILE="${BASE_INFO_OUTPUT_DIR}/artifact-image-complete-matrix.yml"
+				declare GHA_CONFIG_YAML_FILE="${user_gha_dir}/gha_config.yaml"
+				run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/output-gha-workflow-template.py "${GHA_WORKFLOW_TEMPLATE_OUT_FILE}" "${GHA_CONFIG_YAML_FILE}" "${wf_template_dir}" "${MATRIX_ARTIFACT_CHUNKS:-"10"}" "${MATRIX_IMAGE_CHUNKS:-"10"}"
+			else
+				display_alert "Skipping GHA workflow template" "output-gha-workflow-template :: no ${wf_template_dir}" "info"
+			fi
+
 		fi
 
 		### a secondary stage, which only makes sense to be run inside GHA, and as such should be split in a different CLI or under a flag.

lib/functions/general/python-tools.sh

@@ -20,6 +20,7 @@ function early_prepare_pip3_dependencies_for_python_tools() {
 		"coloredlogs==15.0.1" # for colored logging
 		"PyYAML==6.0"         # for parsing/writing YAML
 		"oras==0.1.17"        # for OCI stuff in mapper-oci-update
+		"Jinja2==3.1.2"       # for templating
 	)
 	return 0
 }

lib/tools/info/output-gha-workflow-template.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
+#  SPDX-License-Identifier: GPL-2.0
+#  Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
+#  This file is a part of the Armbian Build Framework https://github.com/armbian/build/
+# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
+import logging
+import os
+import yaml
+
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from common import armbian_utils
+from jinja2 import Environment
+from jinja2 import StrictUndefined
+
+# Prepare logging
+armbian_utils.setup_logging()
+log: logging.Logger = logging.getLogger("output-gha-workflow-template")
+
+# Parse cmdline
+
+output_file = sys.argv[1]
+config_yaml_file = sys.argv[2]
+template_dir = sys.argv[3]
+num_chunks_artifacts = int(sys.argv[4])
+num_chunks_images = int(sys.argv[5])
+
+log.info(f"output_file: {output_file}")
+log.info(f"template_dir: {template_dir}")
+log.info(f"num_chunks_artifacts: {num_chunks_artifacts}")
+log.info(f"num_chunks_images: {num_chunks_images}")
+
+# load the yaml config (context for all entries)
+config = {}
+with open(config_yaml_file, "r") as f:
+	config_yaml = f.read()
+	config = yaml.load(config_yaml, Loader=yaml.FullLoader)
+
+# get a list of the .yaml files in the template dir
+template_files = [f for f in os.listdir(template_dir) if f.endswith(".yaml") or f.endswith(".yml")]
+# sort it
+template_files.sort()
+log.info(f"template_files: {template_files}")
+
+out: str = ""
+
+
+# here is a list of the filenames we expect:
+# 050.single_header.yaml
+# 150.per-chunk-artifacts_prep-outputs.yaml
+# 151.per-chunk-images_prep-outputs.yaml
+# 250.single_aggr.jobs.yaml
+# 550.per-chunk-artifacts_job.yaml
+# 650.per-chunk-images_job.yaml
+def handle_template(template_content: str, context: dict) -> str:
+	env = Environment(block_start_string='[%', block_end_string='%]',
+					  variable_start_string='[[', variable_end_string=']]', comment_start_string='[#', comment_end_string='#]',
+					  undefined=StrictUndefined)
+	jinja_template = env.from_string(template_content)
+
+	rendered = jinja_template.render(context)
+
+	# Now, strip all the lines that contain the string "<TEMPLATE-IGNORE>"
+	rendered = "\n".join([line for line in rendered.split("\n") if "<TEMPLATE-IGNORE>" not in line])
+
+	# More crazy. For the string '"TEMPLATE-JOB-NAME": # <TEMPLATE-JOB-NAME>' we will replace it with the actual job name
+	rendered = rendered.replace('"TEMPLATE-JOB-NAME": # <TEMPLATE-JOB-NAME>', f'"{context["job_name"]}": # templated "{context["job_name"]}"')
+
+	# ensure it ends with a newline
+	if not rendered.endswith("\n"):
+		rendered += "\n"
+
+	return rendered
+
+
+# loop over the template files
+for template_file in template_files:
+	# parse the filename according to the above list
+	template_order, rest = template_file.split(".", 1)
+	template_order = int(template_order)
+	# parse the type of template, separated by "_"
+	template_type, rest = rest.split("_", 1)
+	# parse the name of the template and the extension. the extension is anything after the first "."
+	template_name, template_ext = rest.split(".", 1)
+	# read the full contents of the template file as UTF-8
+	with open(os.path.join(template_dir, template_file), "r") as f:
+		template_content = f.read()
+
+	log.info(
+		f"Processing template file: {template_file} (order: {template_order}, type: {template_type}, name: {template_name}, ext:{template_ext}, len:{len(template_content)} bytes)")
+
+	# prepare quoted comma lists of the chunks, remove the first and last quotes
+	quoted_comma_list_artifact_chunk_jobs = ",".join([f"\"build-artifacts-chunk-{chunk + 1}\"" for chunk in range(num_chunks_artifacts)])[1:-1]
+
+	# same, but for images, remove the first and last quotes
+	quoted_comma_list_image_chunk_jobs = ",".join([f"\"build-images-chunk-{chunk + 1}\"" for chunk in range(num_chunks_images)])[1:-1]
+
+	context = {
+		"num_chunks_artifacts": num_chunks_artifacts,
+		"num_chunks_images": num_chunks_images,
+		"quoted_comma_list_artifact_chunk_jobs": quoted_comma_list_artifact_chunk_jobs,
+		"quoted_comma_list_image_chunk_jobs": quoted_comma_list_image_chunk_jobs
+	}
+
+	# all 'config' dict on top, for common things re-used everywhere
+	context.update(config)
+
+	out += f"\n# template file: {template_file}\n\n"
+
+	if template_type == "single":
+		context["job_name"] = "ERROR_IN_TEMPLATE!!!"
+		out += handle_template(template_content, context)
+	elif template_type == "per-chunk-artifacts":
+		for chunk in range(num_chunks_artifacts):
+			context["chunk"] = chunk + 1
+			context["num_chunks"] = num_chunks_artifacts
+			context["job_name"] = f"build-artifacts-chunk-{chunk + 1}"
+			out += handle_template(template_content, context)
+	elif template_type == "per-chunk-images":
+		for chunk in range(num_chunks_images):
+			context["chunk"] = chunk + 1
+			context["num_chunks"] = num_chunks_images
+			context["job_name"] = f"build-images-chunk-{chunk + 1}"
+			out += handle_template(template_content, context)
+	else:
+		raise Exception(f"Unknown template type: {template_type}")
+
+# write the out str to the output file
+with open(output_file, "w") as f:
+	f.write(out)
+
+log.info(f"Done. Wrote {len(out)} bytes to {output_file}")