build/lib/tools/info/outdated-artifact-image-reducer.py

#!/usr/bin/env python3

# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
#  SPDX-License-Identifier: GPL-2.0
#  Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
#  This file is a part of the Armbian Build Framework https://github.com/armbian/build/
# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
import json
import logging
import os

import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from common import armbian_utils

# Prepare logging
armbian_utils.setup_logging()
log: logging.Logger = logging.getLogger("outdated-artifact-image-reducer")

# read the outdated-artifacts json file passed as first argument as a json object
with open(sys.argv[1]) as f:
	artifacts = json.load(f)

# read the full images info json file passed as second argument as a json object
with open(sys.argv[2]) as f:
	images = json.load(f)

# give an id to each artifact, store in dict (which is also an output)
artifacts_by_id = {}
counter = 1
for artifact in artifacts:
	# id is the counter left-padded with zeros to 10 digits
	artifact["id"] = str(counter).zfill(10)
	counter = counter + 1
	artifacts_by_id[artifact["id"]] = artifact

# lets find artifacts that have the same name, but different versions
tags_by_oci_name: dict[str, list] = {}
for artifact in artifacts:
	artifact_full_oci_target = artifact["out"]["artifact_full_oci_target"]
	# split at the colon (:)
	oci_split = artifact_full_oci_target.split(":")
	oci_name = oci_split[0]
	oci_tag = oci_split[1]
	log.debug(f"OCI name '{oci_name}' has tag '{oci_tag}'")

	# add it to the dict
	if oci_name not in tags_by_oci_name:
		tags_by_oci_name[oci_name] = []
	tags_by_oci_name[oci_name].append(oci_tag)

# loop over the dict, and warn if any have more than one instance
for oci_name in tags_by_oci_name:
	tags = tags_by_oci_name[oci_name]
	if len(tags) > 1:
		list_tags_escaped_quoted = ', '.join([f"'{tag}'" for tag in tags])
		log.warning(
			f"Artifact '{oci_name}' has {len(tags)} different tags: {list_tags_escaped_quoted} - this is certainly a problem, go fix the artifact.")

# map images to in.target_id
images_by_target_id = {}
for image in images:
	if "config_ok" not in image or not image["config_ok"]:
		log.warning(f"Image {image['in']['target_id']} did not config OK, skipping")
		continue

	if "out" not in image:
		log.warning(f"Image {image['in']['target_id']} has no out field, skipping")
		continue

	if "IMAGE_FILE_ID" not in image["out"]:
		log.warning(f"Image {image['in']['target_id']} has no IMAGE_FILE_ID field, skipping")
		continue

	image["image_file_id"] = image["out"]["IMAGE_FILE_ID"]
	images_by_target_id[image["in"]["target_id"]] = image

# map artifacts to in.wanted_by_targets array
artifacts_by_target_id = {}
for artifact in artifacts:
	# optional: if the artifact is up-to-date, skip?
	artifact_wanted_targets = artifact["in"]["wanted_by_targets"]
	for linked_to_target in artifact_wanted_targets:
		if linked_to_target not in artifacts_by_target_id:
			artifacts_by_target_id[linked_to_target] = []
		artifacts_by_target_id[linked_to_target].append(artifact)

artifacts_by_artifact_name = {}
for artifact in artifacts:
	if artifact["in"]["artifact_name"] not in artifacts_by_artifact_name:
		artifacts_by_artifact_name[artifact["in"]["artifact_name"]] = []
	artifacts_by_artifact_name[artifact["in"]["artifact_name"]].append(artifact["id"])

outdated_artifacts_by_artifact_name = {}
for artifact in artifacts:
	if artifact["oci"]["up-to-date"]:
		continue
	if artifact["in"]["artifact_name"] not in outdated_artifacts_by_artifact_name:
		outdated_artifacts_by_artifact_name[artifact["in"]["artifact_name"]] = []
	outdated_artifacts_by_artifact_name[artifact["in"]["artifact_name"]].append(artifact["id"])

images_with_artifacts = {}
for target_id, image in images_by_target_id.items():
	# skip the images with such pipeline config. only their artifacts are relevant.
	if "pipeline" in image["in"]:
		if "build-image" in image["in"]["pipeline"]:
			if not image["in"]["pipeline"]["build-image"]:
				log.warning(f"Image {image['in']['target_id']} has a pipeline build-image false, skipping")
				continue
			else:
				log.warning(f"Image {image['in']['target_id']} has a pipeline build-image true, processing")

	if target_id not in artifacts_by_target_id:
		continue
	image_artifacts = artifacts_by_target_id[target_id]

	image["artifact_ids"] = []
	for artifact in artifacts_by_target_id[target_id]:
		image["artifact_ids"].append(artifact["id"])

	image["outdated_artifacts_count"] = 0
	image["outdated_artifact_ids"] = []
	for artifact in image_artifacts:
		if not artifact["oci"]["up-to-date"]:
			image["outdated_artifact_ids"].append(artifact["id"])
			image["outdated_artifacts_count"] = image["outdated_artifacts_count"] + 1
	images_with_artifacts[target_id] = image

# images with outdated artifacts
images_with_outdated_artifacts = []
for target_id, image in images_with_artifacts.items():
	if image["outdated_artifacts_count"] > 0:
		images_with_outdated_artifacts.append(target_id)

result = {"images": images_with_artifacts, "artifacts": artifacts_by_id,
		  "artifacts_by_artifact_name": artifacts_by_artifact_name,
		  "outdated_artifacts_by_artifact_name": outdated_artifacts_by_artifact_name,
		  "images_with_outdated_artifacts": images_with_outdated_artifacts}

print(json.dumps(result, indent=4, sort_keys=False))