#!/bin/bash # Some PKGBUILDs need CARCH to be set CARCH="x86_64" # Useful functions UMASK="" set_umask () { [ "$UMASK" == "" ] && UMASK="$(umask)" export UMASK umask 002 } restore_umask () { umask $UMASK >/dev/null } # just like mv -f, but we touch the file and then copy the content so # default ACLs in the target dir will be applied mv_acl() { rm -f "$2" touch "$2" cat "$1" >"$2" || return 1 rm -f "$1" } if [[ $USER != $DBSCRIPTS_USER ]]; then exec sudo -H -u $DBSCRIPTS_USER "$0" "$@" # TODO: uncomment this again and make sure the testsuite doesn't break because of it #else #cd "$TMPDIR" fi # set up general environment WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") LOCKS=() REPO_MODIFIED=0 declare -a HISTORY_add_files=() declare -a HISTORY_remove_files=() # check if messages are to be printed using color unset ALL_OFF BOLD BLUE GREEN RED YELLOW if [[ -t 2 ]]; then ALL_OFF="$(tput sgr0)" BOLD="$(tput bold)" BLUE="${BOLD}$(tput setaf 4)" GREEN="${BOLD}$(tput setaf 2)" RED="${BOLD}$(tput setaf 1)" YELLOW="${BOLD}$(tput setaf 3)" fi readonly ALL_OFF BOLD BLUE GREEN RED YELLOW plain() { local mesg=$1; shift printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" } msg() { local mesg=$1; shift printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" } msg2() { local mesg=$1; shift printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" } warning() { local mesg=$1; shift printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 } error() { local mesg=$1; shift printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 } ## # usage : in_array( $needle, $haystack ) # return : 0 - found # 1 - not found ## in_array() { local needle=$1; shift [[ -z $1 ]] && return 1 # Not Found local item for item in "$@"; do [[ $item = $needle ]] && return 0 # Found done return 1 # Not Found } ## # usage : get_full_version( $epoch, $pkgver, $pkgrel ) # return : full version spec, including epoch (if necessary), pkgver, pkgrel ## get_full_version() { if [[ $1 -eq 0 ]]; then # zero epoch case, don't include it in version echo $2-$3 else echo $1:$2-$3 fi } script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" error "Script ${0##*/} is already locked by $_owner." exit 1 else set_umask return 0 fi } script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then warning "Script ${0##*/} was not locked!" restore_umask return 1 else rmdir "$LOCKDIR" restore_umask return 0 fi } cleanup() { local l local repo local arch trap - EXIT INT QUIT TERM for l in ${LOCKS[@]}; do repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then msg "Removing left over lock from [${repo}] (${arch})" repo_unlock $repo $arch fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then msg "Removing left over lock from ${0##*/}" script_unlock fi rm -rf "$WORKDIR" if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" fi [ "$1" ] && exit $1 } abort() { msg 'Aborting...' cleanup 0 } die() { error "$*" cleanup 1 } trap abort INT QUIT TERM HUP trap cleanup EXIT #repo_lock [timeout] repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" local _count local _trial local _timeout local _lockblock local _owner # This is the lock file used by repo-add and repo-remove if [ -f "${DBLOCKFILE}" ]; then error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" return 1 fi if [ -f "${FILESLOCKFILE}" ]; then error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})" return 1 fi if [ $# -eq 2 ]; then _lockblock=true _trial=0 elif [ $# -eq 3 ]; then _lockblock=false _timeout=$3 let _trial=$_timeout/$LOCK_DELAY fi _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else LOCKS[${#LOCKS[*]}]="$1.$2" set_umask return 0 fi sleep $LOCK_DELAY let _count=$_count+1 done error "Repo [${1}] (${2}) is already locked by $_owner. Giving up!" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then warning "Repo lock [${1}] (${2}) was not locked!" restore_umask return 1 else rmdir "$LOCKDIR" restore_umask return 0 fi } # usage: _grep_pkginfo pkgfile pattern _grep_pkginfo() { local _ret _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } # Get the package base or name as fallback getpkgbase() { local _base _base="$(_grep_pkginfo "$1" "pkgbase")" if [ -z "$_base" ]; then getpkgname "$1" else echo "$_base" fi } issplitpkg() { local _base _base="$(_grep_pkginfo "$1" "pkgbase")" if [ -z "$_base" ]; then return 1 else return 0 fi } # Get the package name getpkgname() { local _name _name="$(_grep_pkginfo "$1" "pkgname")" if [ -z "$_name" ]; then error "Package '$1' has no pkgname in the PKGINFO. Fail!" exit 1 fi echo "$_name" } # Get the pkgver-pkgrel of this package getpkgver() { local _ver _ver="$(_grep_pkginfo "$1" "pkgver")" if [ -z "$_ver" ]; then error "Package '$1' has no pkgver in the PKGINFO. Fail!" exit 1 fi echo "$_ver" } getpkgarch() { local _ver _ver="$(_grep_pkginfo "$1" "arch")" if [ -z "$_ver" ]; then error "Package '$1' has no arch in the PKGINFO. Fail!" exit 1 fi echo "$_ver" } check_packager() { local _packager _packager=$(_grep_pkginfo "$1" "packager") [[ $_packager && $_packager != 'Unknown Packager' ]] } getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' exit 1 elif [ ! -f "${1}" ]; then error "Package ${1} not found!" exit 1 elif ${REQUIRE_SIGNATURE} && [ ! -f "${1}.sig" ]; then error "Package signature ${1}.sig not found!" exit 1 fi echo ${1} } getpkgfiles() { local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' exit 1 fi for f in ${@}; do if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 elif ${REQUIRE_SIGNATURE} && [ ! -f "${f}.sig" ]; then error "Package signature ${f}.sig not found!" exit 1 fi done echo ${@} } check_pkgfile() { local pkgfile=$1 local pkgname="$(getpkgname ${pkgfile})" [ $? -ge 1 ] && return 1 local pkgver="$(getpkgver ${pkgfile})" [ $? -ge 1 ] && return 1 local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 else return 1 fi } check_pkgrepos() { local pkgfile=$1 local pkgname="$(getpkgname ${pkgfile})" [ $? -ge 1 ] && return 1 local pkgver="$(getpkgver ${pkgfile})" [ $? -ge 1 ] && return 1 local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 return 0 } #usage: chk_license ${license[@]}" chk_license() { local l for l in ${@}; do in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 done return 1 } check_repo_permission() { local repo=$1 [ ${#PKGREPOS[@]} -eq 0 ] && return 1 [ -z "${PKGPOOL}" ] && return 1 in_array "${repo}" ${PKGREPOS[@]} || return 1 [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 local arch for arch in ${ARCHES}; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1 done return 0 } set_repo_permission() { local repo=$1 local arch=$2 local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" else error "You don't have permission to change ${dbfile}" fi } arch_repo_add() { local repo=$1 local arch=$2 local pkgs=(${@:3}) # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 } arch_repo_remove() { local repo=$1 local arch=$2 local pkgs=(${@:3}) local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then error "No database found at '${dbfile}'" return 1 fi /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ || error "repo-remove ${dbfile} ${pkgs[@]}" /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 } check_pkgfile_pkgbuild() { local pkgfile="${1}" local _pkgbase="$(getpkgbase ${pkgfile})" [ $? -ge 1 ] && return 1 local _pkgname="$(getpkgname ${pkgfile})" [ $? -ge 1 ] && return 1 local _pkgver="$(getpkgver ${pkgfile})" [ $? -ge 1 ] && return 1 local _pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" get_file_from_pkgrepo "$_pkgbase" "$_pkgver" PKGBUILD \ > "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" [ $? -ge 1 ] && return 1 fi local pkgbuild_ver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" [ "${pkgbuild_ver}" == "${_pkgver}" ] || return 1 local pkgbuild_names=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) in_array "${_pkgname}" ${pkgbuild_names[@]} || return 1 return 0 } arch_add_to_pool() { local pkgfile="$1" if [ -f "${pkgfile}" ]; then if ! check_pkgfile_pkgbuild "$pkgfile"; then warning "package $pkgfile failed check against PKGBUILD. skipping" return 0 fi mv "${pkgfile}" "$FTP_BASE/${PKGPOOL}" fi # also move signatures if [ -f "${pkgfile}.sig" ]; then mv "${pkgfile}.sig" "$FTP_BASE/${PKGPOOL}" fi } # add a package file (from the pool) to the database # call arch_add_to_pool first arch_db_add() { local repo="$1" local tarch="$2" local pkgfiles=("${@:3}") pkgfiles=("${pkgfiles[@]##*/}") local dstdir="$FTP_BASE/$repo/os/$tarch" for pkgfile in "${pkgfiles[@]}"; do local srcfile="$FTP_BASE/${PKGPOOL}/${pkgfile}" if [[ ! -f "${srcfile}" ]]; then warning "Package file ${pkgfile} not found in ${FTP_BASE}/${PKGPOOL}. skipping" return 0 #else # TODO: echo message in db-add executable so this can be used by db-move without unwanted output #msg2 "Adding $pkgfile to [$repo]-$tarch..." fi ln -sr "$srcfile" "$dstdir/" if [ -f "${srcfile}.sig" ]; then ln -sr "${srcfile}.sig" "$dstdir/" fi done arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" for pkgfile in "${pkgfiles[@]}"; do arch_history_add "$repo" "$dstdir/$pkgfile" "$tarch" done } arch_db_remove() { local repo=$1 local arch=$2 local pkgs=(${@:3}) arch_repo_remove "$@" for pkgname in "${pkgs[@]}"; do arch_history_remove "$repo" "$pkgname" "$arch" done } arch_db_move() { local repo_from=$1 local repo_to=$2 local arch=$3 local pkgnames=(${@:4}) local ftppath_from="${FTP_BASE}/${repo_from}/os/" # TODO: detect if a pkgname has been removed from a split package and clean up the repo for pkgname in "${pkgnames[@]}"; do pkgentry=$(pkgentry_from_db "$repo_from" "$arch" "$pkgname") pkgs=($(getpkgfiles "$ftppath_from$arch/$pkgentry"*${PKGEXT})) if [[ -z $pkgentry ]]; then warning "Failed to detect pkgentry for $pkgname" return 1 fi for pkg in "${pkgs[@]}"; do pkgname=$(getpkgname "$pkg") arch_db_add "${repo_to}" "$arch" "${pkg##*/}" arch_db_remove "${repo_from}" "$arch" "$pkgname" done done } arch_history_add() { local repo="$1" local pkgfile="$2" local tarch="$3" local pkgname=$(getpkgname "$pkgfile") local pkgbase=$(getpkgbase "$pkgfile") local pkgver=$(getpkgver "$pkgfile") local history_file="$HISTORYREPO/$repo/$tarch/$pkgname" mkdir -p "${history_file%/*}" echo "$pkgbase $pkgver" > "$history_file" HISTORY_add_files=("${HISTORY_add_files[@]}" "$history_file") } arch_history_remove() { local repo="$1" local pkgname="$2" local tarch="$3" local history_file="$HISTORYREPO/$repo/$tarch/$pkgname" [[ -f "$history_file" ]] || return 1 HISTORY_remove_files=("${HISTORY_remove_files[@]}" "$history_file") } arch_history_commit() { pushd "$HISTORYREPO" >/dev/null did_work=0 if ((${#HISTORY_add_files[@]} > 0)); then git add -- "${HISTORY_add_files[@]}" did_work=1 fi if ((${#HISTORY_remove_files[@]} > 0)); then git rm -q -- "${HISTORY_remove_files[@]}" did_work=1 fi if ((did_work)); then git commit -q -m "$1" fi popd >/dev/null } pkgentry_from_db() { local repo="$1" local arch="$2" local pkgname="$3" local db="$FTP_BASE/$repo/os/$arch/$repo$DBEXT" local line for line in $(tar tf "$db" | sed -n 's#^\([^/]*\)/$#\1#p'); do local name=${line##*/} if [[ ${line%-*-*} = $pkgname ]]; then echo $name return 0 fi done return 1 } # get_file_from_pkgrepo pkgbase tag filename # # Get a file from the package's git/svn/whatever repo. # tag is the full package version (epoch:pkgver-pkgrel) or depending on how # the VCS is used any other recognized tag. # # Implementation note: This function should be the only one directly # interacting with the given VCS and should automatically detect # where a pkgbase is stored if multiple VCSs are used. get_file_from_pkgrepo() { local pkgbase="$1" local tag="$2" local filename="$3" # TODO: implement svn/git extraction # for git: cd $whereever; git show $tag:$filename #curl -s https://projects.archlinux.org/svntogit/community.git/plain/trunk/$filename?h=packages/$pkgbase # TODO: this requires us to restructure the repos arch_svn cat "${SVNREPO}/${pkgbase}/tags/$tag/$filename" #arch_svn cat "${SVNREPO}/${pkgbase}/trunk/$filename" } get_dir_from_pkgrepo() { local pkgbase="$1" local tag="$2" local src="$3" local dest="$4" # TODO: implement svn/git extraction # for git: use git archive? arch_svn export -q "${SVNREPO}/${pkgbase}/tags/${tag}/${src}" "${dest}" } arch_svn() { if [ -z "${SVNUSER}" ]; then /usr/bin/svn "${@}" else sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${SUDO_USER}" "${@}" fi }