JDM170/LibreELEC.tv
1
0
Fork 0
mirror of https://github.com/LibreELEC/LibreELEC.tv synced 2025-09-24 19:46:01 +07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10411 from antonlacon/pkgbuilder

Browse Source 
pkgbuilder: convert to fstrings
This commit is contained in:
Christian Hewitt
2025-09-19 17:08:03 +04:00
committed by GitHub
parent a0088c1518 b074980c1d
commit caeeda51a4
1 changed files with 54 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
scripts/pkgbuilder.py
View File

@@ -128,7 +128,7 @@ class Generator:
tokens = "" tokens = ""
tokens += "[v]" if self.sections[pkg_name] == "virtual" else "" tokens += "[v]" if self.sections[pkg_name] == "virtual" else ""
tokens += "[r]" if pkg_name in self.removedPackages else "" tokens += "[r]" if pkg_name in self.removedPackages else ""
yield("%s:%d%s" % (pkg_name, self.refcount[pkg_name], tokens)) yield(f"{pkg_name}:{self.refcount[pkg_name]}{tokens}")
def getFirstFailedJob(self, job): def getFirstFailedJob(self, job):
for dep in job["wants"]: for dep in job["wants"]:
@@ -301,16 +301,16 @@ class BuildProcess(threading.Thread):
if job["failedjobs"]: if job["failedjobs"]:
flist = [] flist = []
for fjob in job["failedjobs"]: for fjob in job["failedjobs"]:
failedinfo = "%s,%s" % (fjob["task"], fjob["name"]) failedinfo = f"{fjob['task']},{fjob['name']}"
if fjob["logfile"]: if fjob["logfile"]:
failedinfo = "%s,%s" % (failedinfo, fjob["seq"]) failedinfo = f"{failedinfo},{fjob['seq']}"
flist.append(failedinfo) flist.append(failedinfo)
failedinfo = ";".join(flist) failedinfo = ";".join(flist)
else: else:
failedinfo = "" failedinfo = ""
job["args"] = ["%s/%s/pkgbuild" % (ROOT, SCRIPTS), job["args"] = [f"{ROOT}/{SCRIPTS}/pkgbuild",
"%d" % self.slot, "%d" % job["seq"], "%d" % self.jobtotal, "%d" % self.maxslot, f"{self.slot}", f"{job['seq']}", f"{self.jobtotal}", f"{self.maxslot}",
job["task"], job["name"], failedinfo] job["task"], job["name"], failedinfo]
job["start"] = time.time() job["start"] = time.time()
@@ -332,9 +332,9 @@ class BuildProcess(threading.Thread):
returncode = cmd.returncode returncode = cmd.returncode
job["cmdproc"] = cmd job["cmdproc"] = cmd
except UnicodeDecodeError: except UnicodeDecodeError:
print('\nPKGBUILDER ERROR: UnicodeDecodeError while reading cmd.stdout from "%s %s"\n' % (job["task"], job["name"]), file=sys.stderr, flush=True) print(f'\nPKGBUILDER ERROR: UnicodeDecodeError while reading cmd.stdout from "{job["task"]} {job["name"]}"\n', file=sys.stderr, flush=True)
except Exception as e: except Exception as e:
print("\nPKGBUILDER ERROR: %s exception while executing: %s\n" % (str(e), job["args"]), file=sys.stderr, flush=True) print(f"\nPKGBUILDER ERROR: {str(e)} exception while executing: {job['args']}\n", file=sys.stderr, flush=True)
job["end"] = time.time() job["end"] = time.time()
job["elapsed"] = job["end"] - job["start"] job["elapsed"] = job["end"] - job["start"]
@@ -365,7 +365,7 @@ class Builder:
self.generator = Generator(plan) self.generator = Generator(plan)
self.jobtotal = self.generator.totalJobCount() self.jobtotal = self.generator.totalJobCount()
self.twidth = len("%d" % self.jobtotal) self.twidth = len(f"{self.jobtotal}")
# parse threadcount # parse threadcount
if maxthreadcount.endswith("%"): if maxthreadcount.endswith("%"):
@@ -380,7 +380,7 @@ class Builder:
self.threadcount = max(1, self.threadcount) self.threadcount = max(1, self.threadcount)
if args.debug: if args.debug:
DEBUG("THREADCOUNT#: input arg: %s, computed: %d" % (maxthreadcount, self.threadcount)) DEBUG(f"THREADCOUNT#: input arg: {maxthreadcount}, computed: {self.threadcount}")
self.joblog = jobglog self.joblog = jobglog
self.loadstats = loadstats self.loadstats = loadstats
@@ -474,7 +474,7 @@ class Builder:
if failed != []: if failed != []:
self.oprint("\nThe following log(s) for this failure are available:") self.oprint("\nThe following log(s) for this failure are available:")
for job in failed: for job in failed:
self.oprint(" %s => %s" % (job["name"], job["logfile"])) self.oprint(f" {job['name']} => {job['logfile']}")
self.oprint("", flush=True) self.oprint("", flush=True)
return False return False
@@ -492,7 +492,7 @@ class Builder:
if not self.failimmediately and self.generator.activeJobCount() != 0: if not self.failimmediately and self.generator.activeJobCount() != 0:
freeslots = self.threadcount - self.generator.activeJobCount() freeslots = self.threadcount - self.generator.activeJobCount()
self.show_status("WAIT", "waiting", ", ".join(self.generator.activeJobNames())) self.show_status("WAIT", "waiting", ", ".join(self.generator.activeJobNames()))
DEBUG("Waiting for : %d active, %d idle [%s]" % (self.generator.activeJobCount(), freeslots, ", ".join(self.generator.activeJobNames()))) DEBUG(f"Waiting for : {self.generator.activeJobCount()} active, {freeslots} idle [{', '}.join.self.generator.activeJobNames()]")
return True return True
else: else:
return False return False
@@ -505,12 +505,12 @@ class Builder:
self.show_status("INIT", "submit", job["name"]) self.show_status("INIT", "submit", job["name"])
if self.debug: if self.debug:
DEBUG("Queueing Job: %s %s" % (job["task"], job["name"])) DEBUG(f"Queueing Job: {job['task']} {job['name']}")
self.wseq += 1 self.wseq += 1
job["seq"] = self.wseq job["seq"] = self.wseq
if self.log_burst: if self.log_burst:
job["logfile"] = "%s/logs/%d.log" % (THREAD_CONTROL, job["seq"]) job["logfile"] = f"{THREAD_CONTROL}/logs/{job['seq']}.log"
self.work.put(job) self.work.put(job)
@@ -519,26 +519,26 @@ class Builder:
if self.debug: if self.debug:
freeslots = self.threadcount - self.generator.activeJobCount() freeslots = self.threadcount - self.generator.activeJobCount()
DEBUG("Building Now: %d active, %d idle [%s]" % (self.generator.activeJobCount(), freeslots, ", ".join(self.generator.activeJobNames()))) DEBUG(f"Building Now: {self.generator.activeJobCount()} active, {freeslots} idle [{', '.join(self.generator.activeJobNames())}]")
except GeneratorStalled: except GeneratorStalled:
if self.verbose: if self.verbose:
freeslots = self.threadcount - self.generator.activeJobCount() freeslots = self.threadcount - self.generator.activeJobCount()
pending = [] pending = []
for (i, (package, wants)) in enumerate(self.generator.getStallInfo()): for (i, (package, wants)) in enumerate(self.generator.getStallInfo()):
pending.append("%s (wants: %s)" % (package, ", ".join(wants))) pending.append(f"{package} (wants: {', '.join(wants)})")
self.show_status("ACTV", "active", ", ".join(self.generator.activeJobNames())) self.show_status("ACTV", "active", ", ".join(self.generator.activeJobNames()))
self.show_status("IDLE", "stalled", "; ".join(pending), p1=len(pending)) self.show_status("IDLE", "stalled", "; ".join(pending), p1=len(pending))
if self.debug: if self.debug:
freeslots = self.threadcount - self.generator.activeJobCount() freeslots = self.threadcount - self.generator.activeJobCount()
DEBUG("Building Now: %d active, %d idle [%s]" % (self.generator.activeJobCount(), freeslots, ", ".join(self.generator.activeJobNames()))) DEBUG(f"Building Now: {self.generator.activeJobCount()} active, {freeslots} idle [{', '.join(self.generator.activeJobNames())}]")
for (i, (package, wants)) in enumerate(self.generator.getStallInfo()): for (i, (package, wants)) in enumerate(self.generator.getStallInfo()):
item = "%-25s wants: %s" % (package, ", ".join(wants)) item = f"{package:<25} wants: {', '.join(wants)}"
if i == 0: if i == 0:
DEBUG("Stalled Jobs: %s" % item) DEBUG(f"Stalled Jobs: {item}")
else: else:
DEBUG(" %s" % item) DEBUG(f" {item}")
except GeneratorEmpty: except GeneratorEmpty:
if self.generator.activeJobCount() == 0: if self.generator.activeJobCount() == 0:
@@ -548,7 +548,7 @@ class Builder:
else: else:
if self.debug: if self.debug:
n = self.generator.activeJobCount() n = self.generator.activeJobCount()
DEBUG("NO MORE JOBS: Waiting on %d job%s to complete..." % (n, ["s",""][n == 1])) DEBUG(f"NO MORE JOBS: Waiting on {n} job{['s',''][n == 1]} to complete...")
return True return True
@@ -560,7 +560,7 @@ class Builder:
self.generator.completed(job) self.generator.completed(job)
if self.debug: if self.debug:
DEBUG("Finished Job: %s %s [%s] after %0.3f seconds" % (job["task"], job["name"], job["status"], job["elapsed"])) DEBUG(f"Finished Job: {job['task']} {job['name']} [{job['status']}] after {job['elapsed']:0.3f} seconds")
return job return job
@@ -588,10 +588,12 @@ class Builder:
procs = loadavg[3].split("/") procs = loadavg[3].split("/")
meminfo = self.getMemory() meminfo = self.getMemory()
print("%d %06d %5s %5s %5s %3s %4s %9d %2d %s" % (now, now - self.build_start, \ print(
loadavg[0], loadavg[1], loadavg[2], procs[0], procs[1], meminfo["MemAvailable"], \ (
self.generator.activeJobCount(), ",".join(self.generator.activeJobNames())), \ f"{now} {now - self.build_start:06} {loadavg[0]:5} {loadavg[1]:5} {loadavg[2]:5}"
file=self.loadstatsfile, flush=True) f" {procs[0]:3} {procs[1]:4} {meminfo['MemAvailable']:9}"
f" {self.generator.activeJobCount():2} {','.join(self.generator.activeJobNames())}"
), file=self.loadstatsfile, flush=True)
if self.progress: if self.progress:
return min((self.nextstats - now), int(now + 1) - now) return min((self.nextstats - now), int(now + 1) - now)
@@ -602,7 +604,7 @@ class Builder:
if self.progress: if self.progress:
freeslots = self.threadcount - self.generator.activeJobCount() freeslots = self.threadcount - self.generator.activeJobCount()
if self.jobtotal != self.generator.completedJobCount(): if self.jobtotal != self.generator.completedJobCount():
percent = "%0.2f" % (100 / self.jobtotal * self.generator.completedJobCount()) percent = f"{100 / self.jobtotal * self.generator.completedJobCount():0.2f}"
else: else:
percent = "100" percent = "100"
loadavg = self.getLoad() loadavg = self.getLoad()
@@ -610,25 +612,24 @@ class Builder:
available = int(meminfo["MemAvailable"]) / 1024 available = int(meminfo["MemAvailable"]) / 1024
lines = [ "", lines = [ "",
"%s: %5s%% | load: %s mem: %d MB | failed: %d idle: %d active: %d" % \ (f"{self.secs2hms(time.time() - self.build_start)}: {percent:5}% | load: {loadavg[0]} "
(self.secs2hms(time.time() - self.build_start), percent, \ f"mem: {available:.0f} MB | failed: {self.generator.failedJobCount()} idle: {freeslots} "
loadavg[0], available, \ f"active: {self.generator.activeJobCount()}"),
self.generator.failedJobCount(), freeslots, self.generator.activeJobCount()), f"Building: {', '.join(self.generator.activeJobNames())}"
"Building: %s" % ", ".join(self.generator.activeJobNames())
] ]
columns = self.columns # in theory could change mid-loop columns = self.columns # in theory could change mid-loop
output = [] output = []
for line in lines: for line in lines:
output.append(line if len(line) < columns else "%s+" % line[0:columns - 2]) output.append(line if len(line) < columns else f"{line[0:columns - 2]}+")
if not self.progress_glitch_fix: if not self.progress_glitch_fix:
self.progress_glitch_fix = "%s\033[%dA" % ("\n" * len(output), len(output)) self.progress_glitch_fix = f"{chr(10) * len(output)}\033[{len(output)}A"
# \033[?7l: disable linewrap # \033[?7l: disable linewrap
# \033[0K: clear cursor to end of line (every line but last) # \033[0K: clear cursor to end of line (every line but last)
# \033[0J: clear cursor to end of screen (last line) # \033[0J: clear cursor to end of screen (last line)
# \033%dA: move cursor up %d lines (move back to "home" position) # \033{len(...)}A: move cursor up len(...) lines (move back to "home" position)
# \033[?7h: re-enable linewrap # \033[?7h: re-enable linewrap
# #
# When the console is resized to a narrower width, lines wider than the # When the console is resized to a narrower width, lines wider than the
@@ -651,7 +652,7 @@ class Builder:
if job["failed"]: if job["failed"]:
if job["logfile"]: if job["logfile"]:
self.eprint("\nThe following log for this failure is available:\n %s\n" % job["logfile"]) self.eprint(f"\nThe following log for this failure is available:\n {job['logfile']}\n")
if job["failedjobs"] and job["failedjobs"][0]["logfile"]: if job["failedjobs"] and job["failedjobs"][0]["logfile"]:
if len(job["failedjobs"]) == 1: if len(job["failedjobs"]) == 1:
@@ -659,7 +660,7 @@ class Builder:
else: else:
self.eprint("The following logs from the failed dependencies may be relevant:") self.eprint("The following logs from the failed dependencies may be relevant:")
for fjob in job["failedjobs"]: for fjob in job["failedjobs"]:
self.eprint(" %-7s %s => %s" % (fjob["task"], fjob["name"], fjob["logfile"])) self.eprint(f" {fjob['task']:<7} {fjob['name']} => {fjob['logfile']}")
self.eprint("") self.eprint("")
# If configured, send output for a job (either a logfile, or captured stdout) to stdout # If configured, send output for a job (either a logfile, or captured stdout) to stdout
@@ -671,7 +672,7 @@ class Builder:
if job["logfile"]: if job["logfile"]:
if self.log_combine == "always" or (job["failed"] and self.log_combine == "fail"): if self.log_combine == "always" or (job["failed"] and self.log_combine == "fail"):
if self.bookends: if self.bookends:
self.oprint("<<< %s seq %s <<<" % (job["name"], job["seq"])) self.oprint(f"<<< {job['name']} seq {job['seq']} <<<")
try: try:
with open(job["logfile"], "r", encoding="utf-8", errors="replace") as logfile: with open(job["logfile"], "r", encoding="utf-8", errors="replace") as logfile:
@@ -679,20 +680,20 @@ class Builder:
self.oprint(line, end="") self.oprint(line, end="")
log_size += len(line) log_size += len(line)
except UnicodeDecodeError: except UnicodeDecodeError:
self.eprint("\nPKGBUILDER ERROR: UnicodeDecodeError while reading log file %s\n" % job["logfile"]) self.eprint(f"\nPKGBUILDER ERROR: UnicodeDecodeError while reading log file {job['logfile']}\n")
if job["failed"]: if job["failed"]:
self.oprint("\nThe following log for this failure is available:\n %s\n" % job["logfile"]) self.oprint(f"\nThe following log for this failure is available:\n {job['logfile']}\n")
if self.bookends: if self.bookends:
self.oprint(">>> %s seq %s >>>" % (job["name"], job["seq"])) self.oprint(f">>> {job['name']} seq {job['seq']} >>>")
log_processed = True log_processed = True
elif job["cmdproc"]: elif job["cmdproc"]:
if self.log_combine == "always" or (job["failed"] and self.log_combine == "fail"): if self.log_combine == "always" or (job["failed"] and self.log_combine == "fail"):
if self.bookends: if self.bookends:
self.oprint("<<< %s" % job["name"]) self.oprint(f"<<< {job['name']}")
for line in job["cmdproc"].stdout: for line in job["cmdproc"].stdout:
self.oprint(line, end="") self.oprint(line, end="")
@@ -705,7 +706,7 @@ class Builder:
job["autoremove"] = None job["autoremove"] = None
if self.bookends: if self.bookends:
self.oprint(">>> %s" % job["name"]) self.oprint(f">>> {job['name']}")
log_processed = True log_processed = True
@@ -715,7 +716,7 @@ class Builder:
if self.debug: if self.debug:
log_elapsed = time.time() - log_start log_elapsed = time.time() - log_start
log_rate = int(log_size / log_elapsed) if log_elapsed != 0 else 0 log_rate = int(log_size / log_elapsed) if log_elapsed != 0 else 0
log_data = ", %s" % "/".join(job["logfile"].split("/")[-2:]) if job["logfile"] else "" log_data = ", {'/'.join(job['logfile'].split('/')[-2:])}" if job["logfile"] else ""
DEBUG("WRITING LOG : {0:,} bytes in {1:0.3f} seconds ({2:,d} bytes/sec{3:})".format(log_size, log_elapsed, log_rate, log_data)) DEBUG("WRITING LOG : {0:,} bytes in {1:0.3f} seconds ({2:,d} bytes/sec{3:})".format(log_size, log_elapsed, log_rate, log_data))
# Log completion stats for job # Log completion stats for job
@@ -734,11 +735,11 @@ class Builder:
def autoRemovePackages(self, job): def autoRemovePackages(self, job):
if self.autoremove: if self.autoremove:
if self.debug: if self.debug:
DEBUG("Cleaning Pkg: %s (%s)" % (job["name"], ", ".join(self.generator.getPackageReferenceCounts(job)))) DEBUG(f"Cleaning Pkg: {job['name']} ({', '.join(self.generator.getPackageReferenceCounts(job))})")
for pkg_name in self.generator.getPackagesToRemove(job): for pkg_name in self.generator.getPackagesToRemove(job):
DEBUG("Removing Pkg: %s" % pkg_name) DEBUG(f"Removing Pkg: {pkg_name}")
args = ["%s/%s/autoremove" % (ROOT, SCRIPTS), pkg_name] args = [f"{ROOT}/{SCRIPTS}/autoremove", pkg_name]
if job["logfile"]: if job["logfile"]:
with open(job["logfile"], "a") as logfile: with open(job["logfile"], "a") as logfile:
cmd = subprocess.run(args, cwd=ROOT, cmd = subprocess.run(args, cwd=ROOT,
@@ -755,7 +756,7 @@ class Builder:
p1 = (self.threadcount - self.generator.activeJobCount()) if p1 == None else p1 p1 = (self.threadcount - self.generator.activeJobCount()) if p1 == None else p1
p2 = self.generator.activeJobCount() if p2 == None else p2 p2 = self.generator.activeJobCount() if p2 == None else p2
colored_status = "%s%-4s\033[0m" % (self.color_code[status], status) if self.colors else "%-4s" % status colored_status = f"{self.color_code[status]}{status:<4}\033[0m" if self.colors else f"{status:<4}"
self.eprint("%s[%0*d/%0*d] [%s] %-7s %s" % \ self.eprint("%s[%0*d/%0*d] [%s] %-7s %s" % \
(self.progress_glitch_fix, self.twidth, p1, self.twidth, p2, colored_status, task, data)) (self.progress_glitch_fix, self.twidth, p1, self.twidth, p2, colored_status, task, data))
@@ -820,11 +821,11 @@ class Builder:
def secs2hms(self, seconds): def secs2hms(self, seconds):
min, sec = divmod(seconds, 60) min, sec = divmod(seconds, 60)
hour, min = divmod(min, 60) hour, min = divmod(min, 60)
return "%02d:%02d:%02d" % (hour, min, sec) return f"{hour:02.0f}:{min:02.0f}:{sec:02.0f}"
def DEBUG(msg): def DEBUG(msg):
if DEBUG_LOG: if DEBUG_LOG:
print("%s: %s" % (datetime.datetime.now(), msg), file=DEBUG_LOG, flush=True) print(f"{datetime.datetime.now()}: {msg}", file=DEBUG_LOG, flush=True)
parser = argparse.ArgumentParser(description="Run processes to build the specified JSON plan", \ parser = argparse.ArgumentParser(description="Run processes to build the specified JSON plan", \
formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=25,width=90)) formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=25,width=90))
@@ -899,14 +900,14 @@ SCRIPTS = os.environ.get("SCRIPTS", "scripts")
THREAD_CONTROL = os.environ["THREAD_CONTROL"] THREAD_CONTROL = os.environ["THREAD_CONTROL"]
if args.debug: if args.debug:
debug_log = "%s/debug.log" % THREAD_CONTROL debug_log = f"{THREAD_CONTROL}/debug.log"
DEBUG_LOG = open(debug_log, "w") DEBUG_LOG = open(debug_log, "w")
print("Debug information is being written to: %s\n" % debug_log, file=sys.stderr, flush=True) print(f"Debug information is being written to: {debug_log}\n", file=sys.stderr, flush=True)
else: else:
DEBUG_LOG = None DEBUG_LOG = None
with open("%s/parallel.pid" % THREAD_CONTROL, "w") as pid: with open(f"{THREAD_CONTROL}/parallel.pid", "w") as pid:
print("%d" % os.getpid(), file=pid) print(f"{os.getpid()}", file=pid)
try: try:
builder = Builder(args.max_procs, args.plan, args.joblog, args.loadstats, args.stats_interval, \ builder = Builder(args.max_procs, args.plan, args.joblog, args.loadstats, args.stats_interval, \