diff --git a/usr.sbin/cluck/Makefile b/usr.sbin/cluck/Makefile new file mode 100644 --- /dev/null +++ b/usr.sbin/cluck/Makefile @@ -0,0 +1,5 @@ +# $FreeBSD$ + +SCRIPTS= cluck.sh + +.include diff --git a/usr.sbin/cluck/cluck.sh b/usr.sbin/cluck/cluck.sh new file mode 100644 --- /dev/null +++ b/usr.sbin/cluck/cluck.sh @@ -0,0 +1,575 @@ +#!/bin/sh +# +# Copyright (c) 2023 Jonathan Reynolds +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +# Base on poudriere's parallel.sh +# Copyright (c) 2012-2013 Baptiste Daroussin +# Copyright (c) 2012-2014 Bryan Drewery + +_spawn_wrapper() { + trap - INT INFO + "$@" +} + +spawn() { + _spawn_wrapper "$@" & +} + +##################### + +usage() { + cat >&2 << EOF +Usage: cluck [-aCcdFgrtv] [-b branch] [-d destdir] [-g git-flavor] + cluck [-Fv] [-b branch] -f file ... + +Description: + Bootstrap the ports tree with git. Download, build and install the required + files to build ports-mgmt/pkg(-devel) and devel/git(@FLAVOR) then + clone the ports tree to destdir. + +Options: + -C Do not run \`make clean' (implies -c). + + -F Force fetch/install if already exist. + + -a When the the build is done, do not run \`pkg autoremove'. + + -b branch When fetching/cloning use branch as default (default: main). + + -c Upon successful termination, do not clean the working directory. + + -d destdir When cloning, use destdir as destination (default: /usr/ports). + + -f file Manually fetch a missing file (-v implied). + Please report bug to ... + + -g flavor Use flavor to build devel/git (default: none). + + -m make_env Add argument to make command. Can be set multiple times + + -r Do not clone repository. + + -t If set mount tmpfs on the working directory. + + -v Verbose output. + + +Examples: + + Build devel/git@tiny with tmpfs + + OPTIONS_UNSET=NLS cluck -t -g tiny + + Clone to /usr/myports using quaterly branch: + + cluck -b 2023Q2 -d /usr/myports + + If you plan to build poudriere with QEMU option on + + cluck -a + + +Exit status: + Return non zero status on error. +EOF + exit 1 +} + +WORKDIR="${TMPDIR:-/tmp}/cluck" +PORTSDIR="${WORKDIR}/ports" +LOGDIR="${WORKDIR}/logs" +DEPENDS_FILE="${WORKDIR}/depends" +export PORTSDIR + +# List of items already run by make_depends() +# Some ports may or may not have pkg as dependency +# so we list it as done already +#DEPENDS_DONE=" ports-mgmt/pkg " +DEPENDS_DONE="${WORKDIR}/.depends.done" +DEPENDS_LOCK="${WORKDIR}/.depends.lock" + +CGIT_URL="https://cgit.freebsd.org/ports/plain/" +BRANCH="main" + +# Works badly without pre-fetching, bsd.java.mk is missing and fetched +# on the next port build, install does succeed with missing security-check.awk +# Almost all of these are required by ports-mgmt/pkg. There is no detection +# for files located at the root of the ports tree (UIDs GIDs) +PREFETCH_FILES=" + COPYRIGHT + GIDs + UIDs + Mk/bsd.port.mk + Mk/bsd.commands.mk + Mk/bsd.default-versions.mk + Mk/bsd.java.mk + Mk/bsd.licenses.mk + Mk/bsd.licenses.db.mk + Mk/bsd.options.mk + Mk/bsd.options.desc.mk + Mk/bsd.sanity.mk + Mk/bsd.sites.mk + Mk/Scripts/actual-package-depends.sh + Mk/Scripts/checksum.sh + Mk/Scripts/check-vulnerable.sh + Mk/Scripts/create-manifest.sh + Mk/Scripts/depends-list.sh + Mk/Scripts/do-depends.sh + Mk/Scripts/do-fetch.sh + Mk/Scripts/do-patch.sh + Mk/Scripts/find-lib.sh + Mk/Scripts/functions.sh + Mk/Scripts/security-check.awk + Templates/BSD.local.dist + Templates/config.guess + Templates/config.sub +" # END QUOTE + +err() { + echo "${0##*/}: Error: $@" >&2 + pkill -P ${PID} + exit 1 +} + +_wait() { + local pid ret=0 + + while :; do + pid= + for pid in $(pgrep -P ${PID}); do + wait ${pid} + case $? in + 127|0) ;; + *) ret=1 ;; + esac + done + + if [ -z "${pid}" ]; then + return ${ret} + fi + done +} + +verbose() { + echo "$@" >&2 +} + +echogrep() { + echo "${output}" | grep -Eo "${1}" | sort -u +} + +echosed() { + echo "${output}" | sed -E -e "/${1}/!d" -e "s,${2},\2," | sort -u +} + +fetch_file() { + local -; set +x + local file="${1##*/}" + local dest="${1%/*}" + + if [ "${file}" != "${dest}" -a ! -d "${PORTSDIR}/${dest}" ]; then + mkdir -p "${PORTSDIR}/${dest}" + fi + # In case of previous failure do not re-fetch unless -F is set + if [ ${FORCE:-0} -eq 1 -o ! -f "${PORTSDIR}/${1}" ]; then + verbose "Fetching file ${1}" + fetch -Frqo "${PORTSDIR}/${1}" "${CGIT_URL}${1}?h=${BRANCH}" 2>/dev/null || + err "${1}: unable to fetch (branch: ${BRANCH})" + else + verbose "Skipping file ${1}" + fi +} + +fetch_dir() { + local dir="$1" + local output file= + + if output=$(fetch -Frqo- "${CGIT_URL}${dir}?h=${BRANCH}" 2>/dev/null); then + for file in $(echogrep "${dir}/[[:alnum:]._-]+/?"); do + case ${file} in + */) fetch_dir ${file%/} ;; + *) spawn fetch_file ${file} ;; + esac + done + if [ -z "${file}" ]; then + err "${dir}: invalid port origin" + fi + else + err "${dir}: no such port origin" + fi + + wait # fetch_file +} + +make_sane() { + local output temp file make_args= + local tries=1 maxtries=5 + + while ! output=$(make -C "${PORTSDIR}/${origin}" ${make_args} \ + check-sanity 2>&1) + do + if [ ${tries} -eq ${maxtries} ]; then + echo "${output}" + # XXX Shows wrong message if check-sanity fail + # eg: check-vulnerable fails + err "make_sane(): unable to fetch missing files " \ + "(tries exceeded). Try with: cluck -f file ..." + fi + tries=$((tries + 1)) + + # Match any 'cannot open' line and the last $PORTSDIR/$file + temp=$(echosed "[Cc]annot [Oo]pen" "(.*${PORTSDIR}/)(.*)") + for file in ${temp}; do + fetch_file ${file} + done + [ -z "${temp}" ] || continue + + # Match any USES=$file + temp=$(echosed "USES=" "(.*=)(.*)") + for file in ${temp}; do + fetch_file "Mk/Uses/${file}.mk" + done + [ -z "${temp}" ] || continue + + # Match any ${$file} (make variable) + # Reset $make_args - the fetched files should set + # the missing vars to the appropriate value + make_args= + temp=$(echosed "Malformed conditional" \ + "(.*\\$\\{)([[:alnum:]_]+)(.*)") + for file in ${temp}; do + make_args="${file}=" # var= + done + + # Exit early, nothing changed + if [ -z "${temp}" ]; then + echo "${output}" + err "make_sane(): unable to fetch missing files. " \ + "Try with: cluck -f file ..." + fi + done + + # Grab any missing file that may not be in $origin/* + for file in $(make -C "${PORTSDIR}/${origin}" \ + -VDESCR -VDISTINFO_FILE -VEXTRA_PATCHES -VPKGMESSAGE) + do + case ${file} in + */${origin}/../*) ;; + /nonexistent) continue ;; + */${origin}/*) continue ;; + esac + fetch_file "${file#${PORTSDIR}/}" + done +} + +make_unsafe() { + local target="$1" + local output temp file + local tries=1 maxtries=3 + + verbose "make_unsafe(${target}): ${fullname}" + + while ! output=$(make -C "${PORTSDIR}/${origin}" ${make_args} \ + ${target} 2>&1 | tee -a "${logfile}") + do + if [ ${tries} -eq ${maxtries} ]; then + err "make_unsafe(${target}): ${fullname}: target failed, " \ + "see ${logfile} for more information (tries exceeded)." + fi + tries=$((tries + 1)) + + # Match files in Mk/* Keywords/* Templates/* + temp=$(echogrep "(Mk|Keywords|Templates)/[[:alnum:]/._-]+") + for file in ${temp}; do + fetch_file ${file} + done + + # Exit early, nothing changed + if [ -z "${temp}" ]; then + err "make_unsafe(${target}): ${fullname}: target failed, " \ + "see ${logfile} for more information." + fi + done +} + +make_safe() { + local target="$1" + + verbose "make_safe(${target}): ${fullname}" + make -C "${PORTSDIR}/${origin}" ${make_args} ${target} >>"${logfile}" 2>&1 || + err "make_safe(${target}): ${fullname}: target failed, " \ + "see ${logfile} for more information." +} + +make_depends() { + local -; set +x + local fullname="$1" + local origin flavor dep + + origin="${fullname%%@*}" + flavor="${fullname#*@}" + + echo "Adding ${fullname}" + + fetch_dir ${origin} + + unset FLAVOR + if [ "${origin}" != "${flavor}" ]; then + export FLAVOR="${flavor}" + fi + + make_sane + + # Run through the dependency list + for dep in $(make -C "${PORTSDIR}/${origin}" \ + -VBUILD_DEPENDS -VLIB_DEPENDS -VRUN_DEPENDS | \ + xargs -n1 | sort -u) + do + dep="${dep##*:}" + if [ ${FORCE:-0} -ne 1 ] && pkg_exists ${dep}; then + verbose "Skipping ${dep}" + continue + fi + + # Prepare for tsort + #echo "${fullname} ${dep}" >>"${DEPENDS_FILE}" + lockf -k "${DEPENDS_LOCK}" echo "${fullname} ${dep}" \ + >>"${DEPENDS_FILE}" + + #case "${DEPENDS_DONE}" in + #*" ${dep} "*) continue ;; + #esac + #DEPENDS_DONE="${DEPENDS_DONE}${dep} " + if ! lockf -k "${DEPENDS_LOCK}" grep -q "^${dep}\$" "${DEPENDS_DONE}" + then + lockf -k "${DEPENDS_LOCK}" echo "${dep}" \ + >>"${DEPENDS_DONE}" + spawn make_depends ${dep} + fi + done + + wait # make_depends + + lockf -k "${DEPENDS_LOCK}" echo "${fullname} ${pkg}" >>"${DEPENDS_FILE}" +} + +run_make() { + local -; set +x + local fullname="$1" make_args="$2" + local origin flavor logfile + + origin="${fullname%%@*}" + flavor="${fullname#*@}" + + unset FLAVOR + if [ "${origin}" != "${flavor}" ]; then + export FLAVOR="${flavor}" + fi + + logfile="${LOGDIR}/$(make -C "${PORTSDIR}/${origin}" -VPKGNAME).log" + : >"${logfile}" + + # make_unsafe() is going to grab any missing files in + # $PORTSDIR/Mk/* - $PORTSDIR/Templates/* - $PORTSDIR/Keywords/* + make_unsafe fetch + make_unsafe extract + make_unsafe patch + make_unsafe configure + make_unsafe build + make_unsafe stage + make_unsafe ${INSTALL_TARGET} + #if [ "${MAKE_PACKAGE} = "yes" ]; then + # make_unsafe package + #fi + if [ "${MAKE_CLEAN}" != "no" ]; then + make_unsafe clean + fi +} + +pkg_exists() { + if [ -x "/usr/local/sbin/pkg" ]; then + if pkg info -e "${1}"; then + return 0 + fi + fi + return 1 +} + +mounted() { + mount | grep -Eq "^tmpfs on ${1} \([^)]+\)\$" +} + +################# MAIN + +PID=$$ +set -o pipefail + +pkg="ports-mgmt/pkg" +git="devel/git" + +while getopts "ab:cCd:Ffg:m:rtv" opt; do + case "${opt}" in + a) autoremove="no" ;; + b) BRANCH="${OPTARG}" ;; + C) MAKE_CLEAN="no" cleandir="no" ;; + c) cleandir="no" ;; + d) destdir="${OPTARG}" ;; + F) FORCE=1 ;; + f) FETCH_ONLY=1 VERBOSE=1 break ;; + g) git="${git}@${OPTARG}" ;; + m) make_env="${make_env:+${make_env} }${OPTARG}" ;; + r) CLONE_REPO="no" ;; + t) tmpfs="yes" ;; + v) VERBOSE=1 ;; + *) usage ;; + esac +done +shift $(($OPTIND - 1)) + +if [ ${VERBOSE:-0} -eq 0 ]; then + verbose() { :; } + quiet="-q" + +fi + +if [ ${FETCH_ONLY:-0} -eq 1 ]; then + if [ $# -eq 0 -o \ + -n "${autoremove}" -o \ + -n "${MAKE_CLEAN}" -o \ + -n "${cleandir}" -o \ + -n "${destdir}" -o \ + "${git#*@}" != "${git}" -o \ + -n "${CLONE_REPO}" -o \ + -n "${make_env}" -o \ + -n "${tmpfs}" ] + then + usage + fi + for file in $@; do + fetch_file "${file}" + done + exit 0 +fi + +if [ $# -ne 0 ]; then + usage +fi + +mkdir -p "${WORKDIR}" +if [ "${tmpfs}" = "yes" ] && ! mounted "${WORKDIR}"; then + mount -t tmpfs tmpfs "${WORKDIR}" || + err "${WORKDIR}: unable to mount tmpfs" +fi + +mkdir -p "${LOGDIR}" "${PORTSDIR}" +: >"${DEPENDS_FILE}" +echo "${pkg}" >"${DEPENDS_DONE}" + +echo "<=== Pre-fetching files ===>" +for file in ${PREFETCH_FILES}; do + spawn fetch_file ${file} +done +_wait || exit 1 # fetch_file + +echo "<=== Finding dependency ===>" +if ! pkg_exists "${pkg}"; then + make_depends "${pkg}" +fi +if ! pkg_exists "${git}"; then + make_depends "${git}" +fi + +sort -u <"${DEPENDS_FILE}" | tsort >"${DEPENDS_FILE}.tsort" +tobuild=$(wc -l <"${DEPENDS_FILE}.tsort") +make_env="${make_env:+${make_env} }-DNO_DIALOG" +i=1 + +echo "<=== Starting build - ${tobuild##*[$IFS]} ports ===>" +while [ ${i} -le ${tobuild} ]; do + fullname=$(tail -r -${i} <"${DEPENDS_FILE}.tsort" | tail -1) + i=$((i + 1)) + + if pkg_exists ${fullname}; then + if [ ${FORCE:-0} -eq 0 ]; then + echo "Skipping ${fullname} (installed)" + continue + else + echo "Building ${fullname} (reinstall)" + INSTALL_TARGET="reinstall" + fi + else + echo "Building ${fullname}" + INSTALL_TARGET="install" + fi + + # No real need for this pkg and git will be insatlled + # as non automatic even with INSTALLS_DEPENDS + if ! [ "${fullname}" = "${pkg}" -o "${fullname}" = "${git}" ]; then + run_make ${fullname} "${make_env} -DINSTALLS_DEPENDS" + else + run_make ${fullname} "${make_env}" + fi +done + +if [ "${autoremove}" != "no" ]; then + echo "<=== Deleting build dependency ===>" + pkg autoremove -y${quiet} +fi + +if [ "${CLONE_REPO}" != "no" ]; then + ( # run in subshell + : ${destdir:=/usr/ports} + echo "<=== Cloning to ${destdir} ===>" + if [ ! -d "${destdir}" ]; then + mkdir -p "${destdir}" || err "${destdir}: cannot create directory" + fi + cd "${destdir}" + + # Following ShelLuser's Howto from forums.freebsd.org + git init ${quiet} && + git remote add origin https://git.freebsd.org/ports.git && + git fetch ${quiet:+-q --progress} origin && + git merge origin/main && + if [ "${BRANCH}" = "main" ]; then + git branch -u origin/main + else + git switch ${BRANCH} + fi + ) || err "${destdir}: cannot clone directory" +fi + +if [ "${cleandir}" != "no" ]; then + echo "<=== Cleaning up ${WORKDIR} ===>" + find "${WORKDIR}" -depth 1 -delete + if [ "${tmpfs}" = "yes" ] && mounted "${WORKDIR}"; then + umount "${WORKDIR}" + fi + rm -r "${WORKDIR}" || exit 1 +fi + +exit 0