diff --git a/Tools/scripts/port_conflicts_check.lua b/Tools/scripts/port_conflicts_check.lua index c5265af19a5a..5a7ea9652ad8 100755 --- a/Tools/scripts/port_conflicts_check.lua +++ b/Tools/scripts/port_conflicts_check.lua @@ -1,507 +1,584 @@ #!/usr/libexec/flua ---[[ -SPDX-License-Identifier: BSD-2-Clause-FreeBSD - -Copyright (c) 2022 Stefan Esser - -Generate a list of existing and required CONFLICTS_INSTALL lines -for all ports (limited to ports for which official packages are -provided). - -This script depends on the ports-mgmt/pkg-provides port for the list -of files installed by all pre-built packages for the architecture -the script is run on. - -The script generates a list of ports by running "pkg provides ." and -a mapping from package base name to origin via "pkg rquery '%n %o'". - -The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched -by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is -only representative for the options configured for each port (i.e. -if non-default options have been selected and registered, these may -lead to a non-default list of conflicts). - -The script detects files used by more than one port, than lists by -origin the existing definition and the list of package base names -that have been detected to cause install conflicts followed by the -list of duplicate files separated by a hash character "#". - -This script uses the "hidden" LUA interpreter in the FreeBSD base -systems and does not need any port except "pkg-provides" to be run. - -The run-time on my system checking the ~32000 packages available -for -CURRENT on amd64 is less than 250 seconds. - -Example output: - -# Port: games/sol -# Files: bin/sol -# < aisleriot gnome-games -# > aisleriot -portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol - -The output is per port (for all flavors of the port, if applicable), -gives examples of conflicting files (mostly to understand whether -different versions of a port could co-exist), the current CONFLICTS -and CONFLICTS_INSTALL entries merged, and a suggested new entry. -This information is followed by a portedit command line that should -do the right thing for simple cases, but the result should always -be checked before the resulting Makefile is committed. ---]] - -require "lfs" +-- SPDX-License-Identifier: BSD-2-Clause-FreeBSD +-- +-- Copyright (c) 2022 Stefan Esser +-- +-- Generate a list of existing and required CONFLICTS_INSTALL lines +-- for all ports (limited to ports for which official packages are +-- provided). +-- +-- This script depends on the ports-mgmt/pkg-provides port for the list +-- of files installed by all pre-built packages for the architecture +-- the script is run on. +-- +-- The script generates a list of ports by running "pkg provides ." and +-- a mapping from package base name to origin via "pkg rquery '%n %o'". +-- +-- The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched +-- by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is +-- only representative for the options configured for each port (i.e. +-- if non-default options have been selected and registered, these may +-- lead to a non-default list of conflicts). +-- +-- The script detects files used by more than one port, than lists by +-- origin the existing definition and the list of package base names +-- that have been detected to cause install conflicts followed by the +-- list of duplicate files separated by a hash character "#". +-- +-- This script uses the "hidden" LUA interpreter in the FreeBSD base +-- systems and does not need any port except "pkg-provides" to be run. +-- +-- The run-time on my system checking the ~32000 packages available +-- for -CURRENT on amd64 is less than 250 seconds. +-- +-- Example output: +-- +-- # Port: games/sol +-- # Files: bin/sol +-- # < aisleriot gnome-games +-- # > aisleriot +-- portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol +-- +-- The output is per port (for all flavors of the port, if applicable), +-- gives examples of conflicting files (mostly to understand whether +-- different versions of a port could co-exist), the current CONFLICTS +-- and CONFLICTS_INSTALL entries merged, and a suggested new entry. +-- This information is followed by a portedit command line that should +-- do the right thing for simple cases, but the result should always +-- be checked before the resulting Makefile is committed. + +local lfs = require "lfs" ------------------------------------------------------------------- -local file_pattern = "." +local file_pattern = "" -- empty pattern matches all filenames local database = "/var/db/pkg/provides/provides.db" -local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds +local max_age = 30 * 24 * 3600 -- maximum age of index or database files in seconds ------------------------------------------------------------------- local function table_sorted_keys(t) local result = {} for k, _ in pairs(t) do result[#result + 1] = k end table.sort(result) return result end ------------------------------------------------------------------- local function table_sort_uniq(t) local result = {} + if t then local last table.sort(t) for _, entry in ipairs(t) do if entry ~= last then last = entry result[#result + 1] = entry end end end return result end ------------------------------------------------------------------- -local function fnmatch(name, pattern) +local function fnmatch(pattern, name) local function fnsubst(s) s = string.gsub(s, "%%", "%%%%") s = string.gsub(s, "%+", "%%+") s = string.gsub(s, "%-", "%%-") s = string.gsub(s, "%.", "%%.") s = string.gsub(s, "%?", ".") s = string.gsub(s, "%*", ".*") return s end + local rexpr = "" local left, middle, right + while true do left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)") if not left then break end rexpr = rexpr .. fnsubst(left) .. middle pattern = right end rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$" - return string.find(name, rexpr) + return string.find(name, rexpr) and 0 or 1 +end + +local fnmatch_module = require("posix.fnmatch") +if fnmatch_module then + fnmatch = fnmatch_module.fnmatch end ------------------------------------------------------------------- local function fetch_pkgs_origins() local pkgs = {} local pipe = io.popen("pkg rquery '%n %o'") - for line in pipe:lines() do - local pkgbase, origin = string.match(line, "(%S+) (%S+)") - pkgs[origin] = pkgbase - end - pipe:close() - pipe = io.popen("pkg rquery '%n %o %At %Av'") - for line in pipe:lines() do - local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") - if tag == "flavor" then - pkgs[origin] = nil - pkgs[origin .. "@" .. value] = pkgbase + + if pipe then + for line in pipe:lines() do + local pkgbase, origin = string.match(line, "(%S+) (%S+)") + pkgs[origin] = pkgbase + end + pipe:close() + pipe = io.popen("pkg rquery '%n %o %At %Av'") + if pipe then + for line in pipe:lines() do + local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") + if tag == "flavor" then + pkgs[origin] = nil + pkgs[origin .. "@" .. value] = pkgbase + end + end + pipe:close() + return pkgs end end - pipe:close() - return pkgs end ------------------------------------------------------------------- local BAD_FILE_PATTERN = { "^[^/]+$", "^lib/python[%d%.]+/site%-packages/examples/[^/]+$", "^lib/python[%d%.]+/site%-packages/samples/[^/]+$", "^lib/python[%d%.]+/site%-packages/test/[^/]+$", "^lib/python[%d%.]+/site%-packages/test_app/[^/]+$", "^lib/python[%d%.]+/site%-packages/tests/[^/]+$", "^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$", } local BAD_FILE_PKGS = {} local function check_bad_file(pkgbase, file) for _, pattern in ipairs(BAD_FILE_PATTERN) do if string.match(file, pattern) then BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {} table.insert(BAD_FILE_PKGS[pkgbase], file) break end end end ------------------------------------------------------------------- -local function read_files(pattern) +-- generate table of list of pkgbases indexed by files contained in those packages +local function fetch_provides(pattern) local files_table = {} local now = os.time() local modification_time = lfs.attributes(database, "modification") + if not modification_time then print("# Aborting: package file database " .. database .. " does not exist.") print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.") print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.") os.exit(1) end - if now - modification_time > max_age then + + local age = now - modification_time + if age > max_age then print("# Aborting: package file database " .. database) - print("# is older than " .. max_age .. " seconds.") + print("# is older (" .. age .. " seconds) than " .. max_age .. " seconds.") print("# Use 'pkg provides -u' to update the database.") os.exit(2) end - local pipe = io.popen("locate -d " .. database .. " " .. pattern) + + local pipe = io.popen("locate -d '" .. database .. "' '" .. pattern .. "'") + if pipe then for line in pipe:lines() do local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)") if file:sub(1, 11) == "/usr/local/" then file = file:sub(12) end check_bad_file(pkgbase, file) local t = files_table[file] or {} t[#t + 1] = pkgbase files_table[file] = t end pipe:close() + return files_table end - return files_table end ------------------------------------------------------------------- local DUPLICATE_FILE = {} -local function fetch_pkg_pairs(pattern) +local function generate_pkg_pairs(files_table) local pkg_pairs = {} - for file, pkgbases in pairs(read_files(pattern)) do + + for file, pkgbases in pairs(files_table) do if #pkgbases >= 2 then DUPLICATE_FILE[file] = true for i = 1, #pkgbases -1 do local pkg_i = pkgbases[i] for j = i + 1, #pkgbases do local pkg_j = pkgbases[j] if pkg_i ~= pkg_j then local p1 = pkg_pairs[pkg_i] or {} local p2 = p1[pkg_j] or {} p2[#p2 + 1] = file p1[pkg_j] = p2 pkg_pairs[pkg_i] = p1 end end end end end return pkg_pairs end ------------------------------------------------------------------- local function conflicts_delta(old, new) local old_seen = {} local changed + for i = 1, #new do local matched for j = 1, #old do - if new[i] == old[j] or fnmatch(new[i], old[j]) then + if new[i] == old[j] or fnmatch(old[j], new[i]) == 0 then new[i] = old[j] old_seen[j] = true matched = true break end end changed = changed or not matched end if not changed then for j = 1, #old do if not old_seen[j] then changed = true break end end end if changed then return table_sort_uniq(new) end end ------------------------------------------------------------------- local function fetch_port_conflicts(origin) local dir, flavor = origin:match("([^@]+)@?(.*)") + if flavor ~= "" then flavor = " FLAVOR=" .. flavor end + local seen = {} local portdir = "/usr/ports/" .. dir local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null") - for line in pipe:lines() do - for word in line:gmatch("(%S+)%s?") do - seen[word] = true + + if pipe then + for line in pipe:lines() do + for word in line:gmatch("(%S+)%s?") do + seen[word] = true + end end + pipe:close() + return table_sorted_keys(seen) end - pipe:close() - return table_sorted_keys(seen) end ------------------------------------------------------------------- local function conflicting_pkgs(conflicting) local pkgs = {} + for origin, pkgbase in pairs(fetch_pkgs_origins()) do if conflicting[pkgbase] then pkgs[origin] = pkgbase end end return pkgs end ------------------------------------------------------------------- local function collect_conflicts(pkg_pairs) local pkgs = {} + for pkg_i, p1 in pairs(pkg_pairs) do for pkg_j, _ in pairs(p1) do pkgs[pkg_i] = pkgs[pkg_i] or {} table.insert(pkgs[pkg_i], pkg_j) pkgs[pkg_j] = pkgs[pkg_j] or {} table.insert(pkgs[pkg_j], pkg_i) end end return pkgs end ------------------------------------------------------------------- local function split_origins(origin_list) local port_list = {} local flavors = {} local last_port + for _, origin in ipairs(origin_list) do local port, flavor = string.match(origin, "([^@]+)@?(.*)") if port ~= last_port then port_list[#port_list + 1] = port if flavor ~= "" then flavors[port] = {flavor} end else table.insert(flavors[port], flavor) end last_port = port end return port_list, flavors end ------------------------------------------------------------------- -local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern) -local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) -local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) -local ORIGIN_LIST = table_sorted_keys(PKGBASE) -local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) - +local FILES_TABLE = fetch_provides(file_pattern) -- +local PKG_PAIR_FILES = generate_pkg_pairs(FILES_TABLE) -- table of (table of list of filenames indexed by pkgbase) indexed by pkgbase +local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) -- table of lists of pkgbases indexed by conflicting pkgbase +local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) -- table of pkgbases indexed by origin from provides database +local ORIGIN_LIST = table_sorted_keys(PKGBASE) -- sorted list of origins from provides database +local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) -- list of port directories, table of list of flavors indexed by port directory + +-- return list of first conflicting file per conflict, list of all conflicting files per conflict local function conflicting_files(pkg_i, pkgs) local files = {} local all_files = {} local f local p1 = PKG_PAIR_FILES[pkg_i] + if p1 then for _, pkg_j in ipairs(pkgs) do f = p1[pkg_j] if f then table.sort(f) files[#files + 1] = f[1] table.move(f, 1, #f, #all_files + 1, all_files) end end end for _, pkg_j in ipairs(pkgs) do p1 = PKG_PAIR_FILES[pkg_j] f = p1 and p1[pkg_i] if f then table.sort(f) files[#files + 1] = f[1] table.move(f, 1, #f, #all_files + 1, all_files) end end return table_sort_uniq(files), table_sort_uniq(all_files) end --------------------------------------------------------------------- local version_pattern = { "^lib/python%d%.%d/", - "^share/py3%d%d%?-", - "^share/%a+/py3%d%d%?-", + "^share/py3%d%d%?%-", + "^share/%S+/py3%d%d%?%-", "^lib/lua/%d%.%d/", "^share/lua/%d%.%d/", "^lib/perl5/[%d%.]+/", "^lib/perl5/site_perl/mach/[%d%.]+/", "^lib/ruby/gems/%d%.%d/", "^lib/ruby/site_ruby/%d%.%d/", } local function generalize_patterns(pkgs, files) local function match_any(str, pattern_array) for _, pattern in ipairs(pattern_array) do if string.match(str, pattern) then return true end end return false end + local function unversioned_files() for i = 1, #files do if not match_any(files[i], version_pattern) then return true end end return false end + local function pkg_wildcards(from, ...) local to_list = {...} local result = {} for i = 1, #pkgs do local orig_pkg = pkgs[i] for _, to in ipairs(to_list) do + --[[ + local after = string.gsub(orig_pkg, from, to) + if after ~= orig_pkg then + print ("# %", from, orig_pkg, "->", after) + end + --]] result[string.gsub(orig_pkg, from, to)] = true end end pkgs = table_sorted_keys(result) end + local pkg_pfx_php = "php[0-9][0-9]-" local pkg_sfx_php = "-php[0-9][0-9]" local pkg_pfx_python2 local pkg_sfx_python2 local pkg_pfx_python3 local pkg_sfx_python3 local pkg_pfx_lua local pkg_pfx_ruby + pkgs = table_sort_uniq(pkgs) if unversioned_files() then pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39- pkg_sfx_python2 = "-py3[0-9]" pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311- pkg_sfx_python3 = "-py3[0-9][0-9]" pkg_pfx_lua = "lua[0-9][0-9]-" pkg_pfx_ruby = "ruby[0-9][0-9]-" else pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}" pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}" pkg_pfx_python3 = nil pkg_sfx_python3 = nil pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}" pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}" end + pkg_wildcards("%-dj[0-9][0-9]%-(.*django)", "-dj[0-9][0-9]-%1") + pkg_wildcards("%-django[0-9][0-9]$", "-django[0-9][0-9]") pkg_wildcards("^php%d%d%-", pkg_pfx_php) - pkg_wildcards("-php%d%d$", pkg_sfx_php) + pkg_wildcards("%-php%d%d$", pkg_sfx_php) + pkg_wildcards("php%d%d$", "php[0-9][0-9]") -- only for php%d%d and mod_php%d%d pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-") pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3) - pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3) + pkg_wildcards("%-py3%d%$", pkg_sfx_python2, pkg_sfx_python3) pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3) - pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3) + pkg_wildcards("%-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3) pkg_wildcards("^lua%d%d%-", pkg_pfx_lua) - pkg_wildcards("-emacs_[%a_]*", "-emacs_*") + pkg_wildcards("%-emacs_[%a_]*", "-emacs_*") pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-") - pkg_wildcards("^bacula%d%-", "bacula[0-9]-") - pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-") - pkg_wildcards("^bareos%d%-", "bareos[0-9]-") - pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-") pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-") - pkg_wildcards("^ruby%d+-", pkg_pfx_ruby) + pkg_wildcards("^ruby%d+%-", pkg_pfx_ruby) + pkg_wildcards("^ldb%d%d$", "ldb[0-9][0-9]") + pkg_wildcards("^samba4([%d%*]+)$", "samba4[0-9][0-9]") + pkg_wildcards("^(rubygem%-.*%-rails)([0-9]+)", "%1[0-9]", "%1[0-9][0-9]") + pkg_wildcards("^(moodle)[%d]+%-", "%1[0-9][0-9]-", "%1[0-9][0-9][0-9]-") + pkg_wildcards("^(mediawiki)[%d]+%-", "%1[0-9][0-9][0-9]-") + pkg_wildcards("^cfengine-masterfiles.*", "cfengine-masterfiles*") + pkg_wildcards("%-pysaml%d%d$", "-pysaml[0-9][0-9]") + pkg_wildcards("^openssl.*", "openssl*") + pkg_wildcards("^libressl.*", "libressl*") + pkg_wildcards("^openldap%d%d$", "openldap[0-9][0-9]") + pkg_wildcards("^haproxy.*", "haproxy*") + pkg_wildcards("^postfix.*", "postfix*") + pkg_wildcards("^exim.*", "exim*") + pkg_wildcards("^e2fsprogs.*", "e2fsprogs*") + pkg_wildcards("^virtualbox-ose-additions.*", "virtualbox-ose-additions*") + pkg_wildcards("prompt-toolkit%d*$", "prompt-toolkit*") + pkg_wildcards("sqlalchemy%d+", "sqlalchemy[0-9][0-9]") + pkg_wildcards("postgresql%d+$", "postgresql[0-9][0-9]") + pkg_wildcards("^postgis%d%d$", "postgis[0-9][0-9]") + pkg_wildcards("^puppetdb%d$", "puppetdb[0-9]") + pkg_wildcards("%-II%-?%d%d$", "-II[0-9][0-9]", "-II-[0-9][0-9]") +-- pkg_wildcards("^bacula%d%-", "bacula[0-9]-") +-- pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-") +-- pkg_wildcards("^bareos%d%-", "bareos[0-9]-") +-- pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-") +-- pkg_wildcards("^mariadb%d+", "mariadb*") +-- pkg_wildcards("^mysql%d+", "mysql*") +-- pkg_wildcards("^percona%d%d%-", "percona[0-9][0-9]-") return table_sort_uniq(pkgs) end ------------------------------------------------------------------- for _, port in ipairs(PORT_LIST) do local function merge_table(t1, t2) table.move(t2, 1, #t2, #t1 + 1, t1) end + local port_conflicts = {} local files = {} local msg_files = {} local conflict_pkgs = {} + local function merge_data(origin) local pkgbase = PKGBASE[origin] if not BAD_FILE_PKGS[pkgbase] then local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase]) merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor merge_table(files, pkg_confl_files) -- all conflicting files merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase]) merge_table(port_conflicts, fetch_port_conflicts(origin)) end end + local flavors = FLAVORS[port] + if flavors then for _, flavor in ipairs(flavors) do merge_data(port .. "@" .. flavor) end else merge_data(port) end + files = table_sort_uniq(files) msg_files = table_sort_uniq(msg_files) conflict_pkgs = generalize_patterns(conflict_pkgs, files) if #port_conflicts then port_conflicts = table_sort_uniq(port_conflicts) conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs) end + if conflict_pkgs then local conflicts_string_cur = table.concat(port_conflicts, " ") local conflicts_string_new = table.concat(conflict_pkgs, " ") local file_list = table.concat(msg_files, " ") print("# Port: " .. port) - print("# Files: " .. file_list) if conflicts_string_cur ~= "" then print("# < " .. conflicts_string_cur) end + local portdir = "/usr/ports/" .. port + local makefile = portdir .."/Makefile" print("# > " .. conflicts_string_new) - print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. - " # " .. file_list .. "' /usr/ports/" .. port) + print("# Files: " .. file_list) + --print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. + print("portedit merge -D9 -u -e 'CONFLICTS_INSTALL=" .. conflicts_string_new .. + " # " .. file_list .. "' " .. makefile) print() end end ------------------------------------------------------------------- local BAD_FILES_ORIGINS = {} for _, origin in ipairs(ORIGIN_LIST) do local pkgbase = PKGBASE[origin] local files = BAD_FILE_PKGS[pkgbase] + if files then for _, file in ipairs(files) do if DUPLICATE_FILE[file] then local port = string.match(origin, "([^@]+)@?") BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {} table.insert(BAD_FILES_ORIGINS[port], file) end end end end ------------------------------------------------------------------- local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS) if #bad_origins > 0 then print ("# Ports with badly named files:") print () for _, port in ipairs(bad_origins) do print ("# " .. port) local files = BAD_FILES_ORIGINS[port] table.sort(files) for _, file in ipairs(files) do print ("#", file) end print() end end