diff --git a/Tools/scripts/README b/Tools/scripts/README index aa9b32f612e4..e3712d07d345 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,133 +1,136 @@ NOTE: These scripts need work and are *NOT* safe to use unless you know what they do. Use at your own risk. Patches would be great, but it is preferred they pass through the maintainer of each particular script. MOVEDlint.awk - checks MOVED for common errors ardiff - compare two archives easily addport - replacement for easy-import bad-pkgdescrs.sh - locate identical pkg descriptions bump_revision.pl - Small script to bump the PORTREVISION variable of ports which are depending on a port with a changed shared lib version. checkcats.py - verify that master categories in all ports are correct and report any problems. Beware that the full check takes quite some time. checknewvers - checks for availability for a newest version of distfiles on MASTER_SITES (ftp only). checksum - allows checking of ports to see if their checksums match, and if they don't, give a diff against the older version to help discover why the checksum didn't match. chkorigin.sh - checks all ports in the tree for a wrong PKGORIGIN. Run this tool after every repocopy. doportlint - run portlint on every port and return the results distclean - compare md5 sums of distfiles in ports/distfiles with currently installed ports collection in ports/* and prompt to remove unmatched entries getpatch - downloads patch attachments from a Bug Tracking Systems getpatch.sh - downloads patch attachments from a Bug Tracking Systems (plain shell script) gnomedepends - Analyse pkg/PLIST and give an advice as to which GNOME ports should be listes in {RUN,LIB}_DEPENDS for this port mark_safe.pl - utility to set subsets of ports to MAKE_JOBS_(UN)SAFE=yes neededlibs.sh - Extract direct library dependencies from binaries. +port_conflicts_check.lua - Verify that files installed by more than 1 port are covered + in CONFLICTS or CONFLICTS_INSTALL entries (and generate portedit commands + to fix those issues)x portsearch - A utility for searching the ports tree. It allows more detailed search criteria than ``make search key='' and accepts all perl(1) regular expressions. search_lib_depends_and_bump.sh - Give it a port that has changed and it will bump all ports having a LIB_DEPENDS on this port splitpatch.pl - A small script to convert multi-file patches to several appropriately named single-file patches. tindex - script used to build INDEXes for supported FreeBSD branches, which are the source of the 'make fetchindex' INDEXes, and the build failure reports on ports@FreeBSD.org update-patches - generates updated patches. update_crates - script used to generate an updated Makefile using make cargo-crates output ---------------------------------------------------------------------- gnomedepends is a script, which analyses pkg/PLIST and gives an advice as to which GNOME ports should be listes in {RUN,LIB}_DEPENDS for the port to ensure correct removal of GNOME shared directories. Usage is simple: % cd /usr/ports/CATEGORY/PORT % gnomedepends.py According to the contents of PLIST the port depends on the following GNOME port(s): /usr/ports/audio/gnomeaudio, for directories: share/gnome/sounds /usr/ports/sysutils/gnomecontrolcenter, for directories: share/gnome/apps /usr/ports/x11/gnomecore, for directories: share/gnome/apps/Games /usr/ports/x11/gnomelibs, for directories: etc/sound/events etc/sound share/gnome/games share/gnome/pixmaps share/gnome The example above means that you need to have ${PORTSDIR}/audio/gnomeaudio, ${PORTSDIR}/sysutils/gnomecontrolcenter, ${PORTSDIR}/x11/gnomecore and ${PORTSDIR}/x11/gnomelibs listed in {RUN,LIB}_DEPENDS for this port. Please be warned, that the this only means that the ports listed by the script required for correct removal of GNOME shared directories, not for the port functionality, so actual {RUN,LIB}_DEPENDS may have more entries. ---------------------------------------------------------------------- portsearch - A utility for searching the ports tree. portsearch is a utility to for searching of the ports tree. It permits much more detailed searches to be performed than ``make search key='' by allowing you to specify which field(s) to search. It also supports all valid perl(1) regular expressions for pattern matching. portsearch displays matching ports in the same format as ``make search'' and also displays the number of matching ports found. The following command line options are supported: -h Prints a multi-line help message and exits -n name Search for name in the name field -p path Search for path in the path field -i info Search for info in the comments field -m maint Search for maint in the Maintainer field -x index Search for index in the category field -b b_deps Search for b_deps in the build-depends field -r r_deps Search for r_deps in the run-depends field -d deps Search for deps in the both the build and run dependency fields. This option behaves differently to the other op- tions, see the EXAMPLES section -f file Use the index file instead of /usr/ports/INDEX All searches are case-insensitive See the file README.portsearch for further information. ---------------------------------------------------------------------- The update-patches script looks for files in $WRKSRC (if unset, this defaults to the work/ subdirectory of the current directory) which have a matching .orig file. It also looks in $PATCHDIR (if unset, this defaults to the files/ subdirectory of the current directory) for patches that correspond to the first set. If the changes in an existing patch do not reflect the changes in the files in $WRKSRC, the script renames the existing patch by adding the suffix .orig and generates a new patch in its place. If no patch existed, the new one is created with a name that contains the path and filename of the file being patched, except that "/" separators and "." characters are replaced by underscores: for example, a new patch to $WRKSRC/foo/bar.c would be created as $PATCHDIR/patch-foo_bar_c. If you save a .orig backup of a file, but don't change the file, update-patches will generate an empty patch. diff --git a/Tools/scripts/port_conflicts_check.lua b/Tools/scripts/port_conflicts_check.lua new file mode 100755 index 000000000000..49d8579e58be --- /dev/null +++ b/Tools/scripts/port_conflicts_check.lua @@ -0,0 +1,346 @@ +#!/usr/libexec/flua + +--[[ +SPDX-License-Identifier: BSD-2-Clause-FreeBSD + +Copyright (c) 2022 Stefan Esser + +Generate a list of existing and required CONFLICTS_INSTALL lines +for all ports (limited to ports for which official packages are +provided). + +This script depends on the ports-mgmt/pkg-provides port for the list +of files installed by all pre-built packages for the architecture +the script is run on. + +The script generates a list of ports by running "pkg provides ." and +a mapping from package base name to origin via "pkg rquery '%n %o'". + +The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched +by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is +only representative for the options configured for each port (i.e. +if non-default options have been selected and registered, these may +lead to a non-default list of conflicts). + +The script detects files used by more than one port, than lists by +origin the existing definition and the list of package base names +that have been detected to cause install conflicts followed by the +list of duplicate files separated by a hash character "#". + +This script uses the "hidden" LUA interpreter in the FreeBSD base +systems and does not need any port except "pkg-provides" to be run. + +The run-time on my system checking the ~32000 packages available +for -CURRENT on amd64 is less than 250 seconds. + +Example output: + +# Port: games/sol +# Files: bin/sol +# < aisleriot gnome-games +# > aisleriot +portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol + +The output is per port (for all flavors of the port, if applicable), +gives examples of conflicting files (mostly to understand whether +different versions of a port could co-exist), the current CONFLICTS +and CONFLICTS_INSTALL entries merged, and a suggested new entry. +This information is followed by a portedit command line that should +do the right thing for simple cases, but the result should always +be checked before the resulting Makefile is committed. +--]] + +require "lfs" + +------------------------------------------------------------------- +local function table_sorted_keys(t) + local result = {} + for k, _ in pairs(t) do + result[#result + 1] = k + end + table.sort(result) + return result +end + +local function table_sort_uniq(t) + local result = {} + if t then + local last + table.sort(t) + for _, entry in ipairs(t) do + if entry ~= last then + last = entry + result[#result + 1] = entry + end + end + end + return result +end + +local function fnmatch(name, pattern) + local function fnsubst(s) + s = string.gsub(s, "%%", "%%%%") + s = string.gsub(s, "%+", "%%+") + s = string.gsub(s, "%-", "%%-") + s = string.gsub(s, "%.", "%%.") + s = string.gsub(s, "%?", ".") + s = string.gsub(s, "%*", ".*") + return s + end + local rexpr = "" + local left, middle, right + while true do + left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)") + if not left then + break + end + rexpr = rexpr .. fnsubst(left) .. middle + pattern = right + end + rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$" + return string.find(name, rexpr) +end + +------------------------------------------------------------------- +local function fetch_pkgs_origin() + local pkgs = {} + local pipe = io.popen("pkg rquery '%n %o'") + for line in pipe:lines() do + local pkgbase, origin = string.match(line, "(%S+) (%S+)") + pkgs[origin] = pkgbase + end + pipe:close() + pipe = io.popen("pkg rquery '%n %o %At %Av'") + for line in pipe:lines() do + local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") + if tag == "flavor" then + pkgs[origin] = nil + pkgs[origin .. "@" .. value] = pkgbase + end + end + pipe:close() + return pkgs +end + +------------------------------------------------------------------- +local function read_files(pattern) + local files_table = {} + local pkgbase + local pipe = io.popen("pkg provides " .. pattern) + for line in pipe:lines() do + local label = string.sub(line, 1, 10) + if label == "Name : " then + local name = string.sub(line, 11) + pkgbase = string.match(name, "(.*)-[^-]*") + elseif label == " " or label == "Filename: " then + local file = string.sub(line, 11) + if file:sub(1, 10) == "usr/local/" then + file = file:sub(11) + else + file = "/" .. file + end + local t = files_table[file] or {} + t[#t + 1] = pkgbase + files_table[file] = t + end + end + pipe:close() + return files_table +end + +------------------------------------------------------------------- + +local function fetch_pkg_pairs(pattern) + local pkg_pairs = {} + for file, pkgbases in pairs(read_files(pattern)) do + if #pkgbases >= 2 then + for i = 1, #pkgbases -1 do + local pkg_i = pkgbases[i] + for j = i + 1, #pkgbases do + local pkg_j = pkgbases[j] + if pkg_i ~= pkg_j then + local p1 = pkg_pairs[pkg_i] or {} + local p2 = p1[pkg_j] or {} + p2[#p2 + 1] = file + p1[pkg_j] = p2 + pkg_pairs[pkg_i] = p1 + end + end + end + end + end + return pkg_pairs +end + +------------------------------------------------------------------- +local function conflicts_delta(old, new) + local old_seen = {} + local changed + for i = 1, #new do + local matched + for j = 1, #old do + if fnmatch(new[i], old[j]) then + new[i] = old[j] + old_seen[j] = true + matched = true + break + end + end + changed = changed or not matched + end + if not changed then + for j = 1, #old do + if not old_seen[j] then + changed = true + break + end + end + end + if changed then + return table_sort_uniq(new) + end +end + +------------------------------------------------------------------- +local function fetch_port_conflicts(origin) + local dir, flavor = origin:match("([^@]+)@?(.*)") + if flavor ~= "" then + flavor = " FLAVOR=" .. flavor + end + local seen = {} + local pipe = io.popen("make -C /usr/ports/" .. dir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null") + for line in pipe:lines() do + for word in line:gmatch("(%S+)%s?") do + seen[word] = true + end + end + pipe:close() + return table_sorted_keys(seen) +end + +------------------------------------------------------------------- +local function conflicting_pkgs(conflicting) + local pkgs = {} + for origin, pkgbase in pairs(fetch_pkgs_origin()) do + if conflicting[pkgbase] then + pkgs[origin] = pkgbase + end + end + return pkgs +end + +------------------------------------------------------------------- +local function collect_conflicts(pkg_pairs) + local pkgs = {} + local files = {} + for pkg_i, p1 in pairs(pkg_pairs) do + for pkg_j, p2 in pairs(p1) do + pkgs[pkg_i] = pkgs[pkg_i] or {} + pkgs[pkg_j] = pkgs[pkg_j] or {} + table.insert(pkgs[pkg_i], pkg_j) + table.insert(pkgs[pkg_j], pkg_i) + files[pkg_i] = files[pkg_i] or {} + files[pkg_j] = files[pkg_j] or {} + for _, file in ipairs(p2) do + table.insert(files[pkg_i], file) + table.insert(files[pkg_j], file) + end + end + end + return pkgs, files +end + +------------------------------------------------------------------- +local function split_origins(origin_list) + local port_list = {} + local flavors = {} + local last_port + for _, origin in ipairs(origin_list) do + local port, flavor = string.match(origin, "([^@]+)@?(.*)") + if port ~= last_port then + port_list[#port_list + 1] = port + if flavor ~= "" then + flavors[port] = {flavor} + end + else + table.insert(flavors[port], flavor) + end + last_port = port + end + return port_list, flavors +end + +------------------------------------------------------------------- +-- TODO: Collect FLAVORs and report for port directory + +local function merge_table(t1, t2) + table.move(t2, 1, #t2, #t1 + 1, t1) +end + +local PKG_PAIR_FILES = fetch_pkg_pairs(".") +local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) +local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) +local ORIGIN_LIST = table_sorted_keys(PKGBASE) +local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) + +local function conflicting_files(pkg_i, pkgs) + local files = {} + local f + local p1 = PKG_PAIR_FILES[pkg_i] + if p1 then + for _, pkg_j in ipairs(pkgs) do + f = p1[pkg_j] + if f then + table.sort(f) + files[#files + 1] = f[1] + end + end + end + for _, pkg_j in ipairs(pkgs) do + p1 = PKG_PAIR_FILES[pkg_j] + f = p1 and p1[pkg_i] + if f then + table.sort(f) + files[#files + 1] = f[1] + end + end + return files +end + +for _, port in ipairs(PORT_LIST) do + local port_conflicts = {} + local files = {} + local conflict_pkgs = {} + local function merge_data(origin) + local pkgbase = PKGBASE[origin] + merge_table(files, conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase])) + merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase]) + merge_table(port_conflicts, fetch_port_conflicts(origin)) + end + local flavors = FLAVORS[port] + if flavors then + for _, flavor in ipairs(flavors) do + merge_data(port .. "@" .. flavor) + end + else + merge_data(port) + end + local conflicts_new = table_sort_uniq(conflict_pkgs) + if #port_conflicts then + port_conflicts = table_sort_uniq(port_conflicts) + conflicts_new = conflicts_delta(port_conflicts, conflicts_new) + end + if conflicts_new then + local conflicts_string = table.concat(port_conflicts, " ") + local conflicts_string_new = table.concat(conflicts_new, " ") + local file_list = table.concat(table_sort_uniq(files), " ") + print("# Port: " .. port) + print("# Files: " .. file_list) + if conflicts_string ~= "" then + print("# < " .. conflicts_string) + end + print("# > " .. conflicts_string_new) + print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. " # " .. file_list .. "' /usr/ports/" .. port) + print() + end +end