diff --git a/Tools/scripts/update-plist.py b/Tools/scripts/update-plist.py new file mode 100644 index 000000000000..b689997be25f --- /dev/null +++ b/Tools/scripts/update-plist.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python + +# update-plist.py - Adds and removes lines in pkg-plist(s) based on +# poudriere error logs. + +# Inspired by: https://codeberg.org/tcberner/poudlist, by tcberner@, +# but this just requires base Python and doesn't use a config file. + +# ---------------------------------------------------------------------------- +# "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp): +# Jason E. Hale wrote this file. As long as you retain +# this notice you can do whatever you want with this stuff. If we meet some +# day, and you think this stuff is worth it, you can buy me a beer in return. +# +# Jason E. Hale +# ---------------------------------------------------------------------------- + +# MAINTAINER= jhale@FreeBSD.org + +# SYNOPSIS +# update-plist.py [-h] [-E] [-p ] + +# DESCRIPTION +# Adds and removes lines in pkg-plist(s) based on poudriere error logs. + +# It is recommend to run this tool after building a port with all options +# enabled with either `poudriere bulk -t` or `poudriere testport` as it +# will prune all stale plist entires, including those prefixed by '%%', +# but it does recognize where it has already been run and won't try to +# duplicate the previous effort. + +# Use ports-mgmt/hs-panopticum for a more in-depth and CPU-intensive +# analysis than this little script could ever provide, but with some +# human brains behind it, using this takes a fraction of the time for +# most ports. + +# You will still need to manually verify that all of the PLIST_SUBs +# are correct. This tool follows the poudriere logs to a tee, but +# sometimes the PLIST_SUBs for a port are more aggressive than they +# really should be as generated by 'make makeplist'. + +# OPTIONS +# -h, --help print helps and then exits +# +# -p, --portsdir path to the ports tree to operate upon +# Overrides the PORTSDIR environment variable +# if defined, required otherwise. +# +# -E, --extended (experimental) try to work with ports with no pkg-plist +# or more complicated pkg-plist structures (PLIST+=foo). +# Writes pkg-plist..add~ and pkg-plist..rm~ +# files to the port directory manual intervention instead of +# skipping. The appended '~' is meant to prevent accidental +# commital of these temporary files. + +# ENVIRONMENT +# PORTSDIR path to the ports tree to operate upon + +# EXAMPLES +# update-plist.py /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors +# update-plist.py -p ~/src/area51/ports /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors/qt6-webengine-6.11.0.log + +# BUGS +# - Assumes that the existing pkg-plist is already somewhat properly sorted. + +import sys +import re +import os +import argparse +import subprocess + +def extract_plist_sub(log_content): + sub_map = {} + match = re.search(r"--PLIST_SUB--\n(.*?)\n--End PLIST_SUB--", log_content, re.DOTALL) + if match: + raw_subs = match.group(1) + pairs = re.findall(r'(\w+)=("(?:[^"\\]|\\.)*"|[^\s]+)', raw_subs) + for key, val in pairs: + clean_val = val.strip('"') + sub_map[f"%%{key}%%"] = "" if "@comment" in clean_val else clean_val + return sub_map + +def get_expanded_key(line, sub_map): + expanded = line.strip() + for macro in sorted(sub_map.keys(), key=len, reverse=True): + if macro in expanded: + expanded = expanded.replace(macro, sub_map[macro]).replace("//", "/") + clean = re.sub(r"%%.*?%%", "", expanded) + clean = re.sub(r"@\w+\s+", "", clean) + return clean.strip().lstrip('/') + +def get_plist_paths(ports_tree_root, origin): + """Uses make to resolve all PLIST paths, handling multiple space-separated plists.""" + port_dir = os.path.join(ports_tree_root, origin) + try: + result = subprocess.check_output( + ['make', '-C', port_dir, '-V', 'PLIST'], + text=True, + stderr=subprocess.DEVNULL + ).strip() + + if result: + paths = result.split() + return [p if os.path.isabs(p) else os.path.join(port_dir, p) for p in paths] + except Exception: + pass + + return [os.path.join(port_dir, "pkg-plist")] + +def process_log(log_path, ports_tree_root, extended=False): + try: + with open(log_path, 'r', encoding='utf-8', errors='ignore') as f: + first_lines = [] + for _ in range(15): + line = f.readline() + if not line: break + first_lines.append(line) + f.seek(0) + log_content = f.read() + except Exception as e: + sys.stderr.write(f"![Error] Could not read {log_path}: {e}\n") + return + + if "Error: Plist issues found" not in log_content: + return + + origin = None + for line in first_lines: + if "port directory:" in line: + parts = line.split(":", 1)[1].strip().rstrip('/').split('/') + if len(parts) >= 2: + origin = os.path.join(parts[-2], parts[-1]) + break + + if not origin: return + + plist_paths = get_plist_paths(ports_tree_root, origin) + existing_plists = [p for p in plist_paths if os.path.exists(p)] + + is_fallback_mode = False + fallback_rm_path = None + + # Handle the case where no valid plists exist on disk + if not existing_plists: + if not extended: + sys.stderr.write(f"![Skip] {origin}: No valid plist files found (port likely relies on PLIST_FILES). Try using with the '-E' flag.\n") + return + else: + is_fallback_mode = True + log_basename = os.path.basename(log_path) + if log_basename.endswith(".log"): + log_basename = log_basename[:-4] + + fallback_add_name = f"pkg-plist.{log_basename}.add~" + fallback_rm_name = f"pkg-plist.{log_basename}.rm~" + fallback_add_path = os.path.join(ports_tree_root, origin, fallback_add_name) + fallback_rm_path = os.path.join(ports_tree_root, origin, fallback_rm_name) + + sys.stderr.write(f"-> Warning: {origin} has no existing plist. Manual intervention required.\n") + existing_plists = [fallback_add_path] + + plist_data = {} + all_existing_lines = set() + + for path in existing_plists: + if os.path.exists(path): + with open(path, 'r', encoding='utf-8') as f: + lines = [l.strip() for l in f.readlines() if l.strip()] + plist_data[path] = lines + all_existing_lines.update(lines) + else: + # For the newly created pkg-plist.*.add~ + plist_data[path] = [] + + sub_map = extract_plist_sub(log_content) + orphans = re.findall(r"Error: Orphaned: (.*)", log_content) + missing_from_log = set(re.findall(r"Error: Missing: (.*)", log_content)) + + removed_count = 0 + added_count = 0 + + if is_fallback_mode: + # We don't have a source plist to remove from, so dump the missing lines to the rm file + if missing_from_log: + plist_data[fallback_rm_path] = sorted(list(missing_from_log)) + removed_count = len(missing_from_log) + else: + for path, lines in plist_data.items(): + new_lines = [] + for line in lines: + clean_line = re.sub(r"%%.*?%%", "", line).strip().lstrip('/') + if line in missing_from_log or clean_line in missing_from_log: + removed_count += 1 + continue + new_lines.append(line) + plist_data[path] = new_lines + + # Target the FIRST file in our list for orphans (either the base plist or the unique .add~ file) + base_plist = existing_plists[0] + + for orphan in sorted(orphans, key=lambda x: get_expanded_key(x, sub_map)): + if orphan in all_existing_lines: + continue + + orphan_key = get_expanded_key(orphan, sub_map) + inserted = False + + for i, line in enumerate(plist_data[base_plist]): + if get_expanded_key(line, sub_map) > orphan_key: + plist_data[base_plist].insert(i, orphan) + inserted = True + added_count += 1 + break + + if not inserted: + plist_data[base_plist].append(orphan) + added_count += 1 + + all_existing_lines.add(orphan) + + if added_count == 0 and removed_count == 0: + sys.stderr.write(f"-> {origin}: Already up to date.\n") + return + + # Suppress the multi-plist warning if we are writing to a fallback intervention file + if len(existing_plists) > 1 and added_count > 0 and "pkg-plist.add" not in base_plist: + base_name = os.path.basename(base_plist) + sys.stderr.write(f"-> Warning: {origin} uses multiple plists. Orphans appended to {base_name}. Please verify manually.\n") + + sys.stderr.write(f"-> Updating {origin}... Done (+{added_count}/-{removed_count})\n") + + for path, lines in plist_data.items(): + with open(path, 'w', encoding='utf-8') as f: + f.write("\n".join(lines) + "\n") + +if __name__ == "__main__": + env_portsdir = os.environ.get("PORTSDIR") + is_required = not bool(env_portsdir) + + parser = argparse.ArgumentParser(description="Updates pkg-plist based on poudriere error logs.") + parser.add_argument( + "logpath", + help="Poudriere log file or directory of logs" + ) + parser.add_argument( + "-p", "--portsdir", + required=is_required, + default=env_portsdir, + help="Path to ports tree to operate upon (Overrides PORTSDIR env var if set)" + ) + parser.add_argument( + "-E", "--extended", + action="store_true", + help="(experimental) If a port has no pkg-plist or a more complicated pkg-plist structure, write orphaned/missing files to pkg-plist..add~ and pkg-plist..rm~ for manual review instead of skipping" + ) + args = parser.parse_args() + + if os.path.isdir(args.logpath): + for filename in sorted(os.listdir(args.logpath)): + if filename.endswith(".log"): + process_log(os.path.join(args.logpath, filename), args.portsdir, args.extended) + else: + process_log(args.logpath, args.portsdir, args.extended)