diff options
| -rwxr-xr-x | cron-jobs/sourceballs | 144 | ||||
| -rw-r--r-- | db-functions | 35 | ||||
| -rwxr-xr-x | misc-scripts/make-sourceball | 97 | ||||
| -rwxr-xr-x | misc-scripts/sourceballs-cleanup | 82 |
4 files changed, 101 insertions, 257 deletions
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b708009..3f690e7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,57 +1,111 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +dirname="$(dirname $0)" +. "${dirname}/../db-functions" +. "${dirname}/../config" script_lock -set_umask +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +# Create a readable file for each repo with the following format +# <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>] +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" + continue + fi + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done -dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" +# Create a list of all available source package file names +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +# Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - ftppath="${FTP_BASE}/$repo/os/$arch" - dbfile="${ftppath}/${repo}${DBEXT}" - if [ ! -r "${dbfile}" ]; then - warning "${dbfile} not found, skipping" - continue - fi - - repo_lock ${repo} ${arch} || exit 1 - # Read packages from db file - # Format is: <pkgbase|pkgname>/<pkgver>-<pkgrel> - pkgs=($(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ - | sort -u)) - repo_unlock ${repo} ${arch} - - for pkg in ${pkgs[@]}; do - pkgbase=${pkg%/*} - pkgver=${pkg#*/} - srcpkg="${pkgbase}-${pkgver}${SRCEXT}" - - # Don't do anything for package in this 'blacklist' - if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then - continue - fi - - # Use this file to 'whitelist' or force building some sourceballs, - # skipping the license check - force="" - if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then - force="-f" - fi - - if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then - error "Failed to download sources for $pkgbase" - fi - fi - done - done + msg "Updating source packages for ${repo}..." + while read line; do + pkginfo=(${line}) + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=(${pkginfo[@]:3}) + + # Should this package be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Check if the license or .force file does not enforce creating a source package + if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + msg2 "${pkgbase}" + # Looks like a previous arch faild; skip it + if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then + continue + fi + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + if [ $? -ge 1 ]; then + error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}" + continue + fi + + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + else + error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" done -$dirname/../misc-scripts/sourceballs-cleanup +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + done +fi script_unlock diff --git a/db-functions b/db-functions index d2df1a7..67a52aa 100644 --- a/db-functions +++ b/db-functions @@ -347,44 +347,13 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - local allowed - for l in "$@"; do - l="$(echo $l | tr '[:upper:]' '[:lower:]')" - for allowed in ${ALLOWED_LICENSES[@]}; do - allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" - if [ "$l" = "$allowed" ]; then - return 0 - fi - done + for l in ${@}; do + in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 done return 1 } -pkgname_from_src() { - local tmp - local a - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} -} - -pkgver_from_src() { - local tmp - local a - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' -} - check_repo_permission() { local repo=$1 diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball deleted file mode 100755 index 9a43376..0000000 --- a/misc-scripts/make-sourceball +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -if [ $# -ne 3 -a $# -ne 4 ]; then - msg "usage: $(basename $0) [-f] <packagename> <repo> <arch>" - msg " -f Force building. Skip license checks" - exit 1 -fi - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -FORCE=0 -if [ "$1" = "-f" ]; then - FORCE=1 - shift -fi - -packagename="$1" -reponame="$2" -arch="$3" - -script_lock - -create_srcpackage() { - if [ -f "$1/PKGBUILD" ]; then - pushd "$1" >/dev/null - pkgver=$(. PKGBUILD; echo ${pkgver}) - if [ $? -ne 0 ]; then - error "Failed to read pkgver from $1" - return 1 - fi - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - if [ $? -ne 0 ]; then - error "Failed to read pkgrel from $1" - return 1 - fi - license=($(. PKGBUILD; echo ${license[@]})) - if [ $? -ne 0 ]; then - error "Failed to read license from $1" - return 1 - fi - if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then - # Removed so as not to clutter failed.txt - #warning "$packagename license (${license[@]}) does not require source tarballs" - return 0 - else - msg "Creating source tarball for $packagename-$pkgver-$pkgrel" - fi - - local line - /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1 - if [ $? -ne 0 ]; then - error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then - mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" - else - error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - popd >/dev/null - - return 0 - fi -} - -set_umask -cd "$WORKDIR" - -failed=0 - -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-any" || failed=1 - elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 - else - error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'" - failed=1 - fi -else - error "Package '$packagename' does not exist in repo '$reponame-$arch'" - failed=1 -fi - -script_unlock - -exit $failed
\ No newline at end of file diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup deleted file mode 100755 index 595f501..0000000 --- a/misc-scripts/sourceballs-cleanup +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -script_lock - -${CLEANUP_DRYRUN} && warning 'dry run mode is active' - -remove_old() { - if [ -d "$1" ]; then - pushd "$1" >/dev/null - PKGVERS="" - for repo in *; do - cd "$repo" - if [ -f PKGBUILD ]; then - pkgver=$(. PKGBUILD; echo ${pkgver}) - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - PKGVERS="$PKGVERS $pkgver-$pkgrel" - else - error "PKGBUILD not found in $1/$repo" - fi - cd .. - done - - for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do - [ -f "$srcpkg" ] || continue - if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then - skip=0 - pver="$(pkgver_from_src $srcpkg)" - for v in $PKGVERS; do - if [ "$v" = "$pver" ]; then - skip=1 - break - fi - done - if [ $skip -ne 1 ]; then - ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR - fi - fi - done - - popd >/dev/null - fi -} - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -cd "$WORKDIR" - -for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do - [ -f "$sourceball" ] || continue - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then - msg "$packagename : package found in sourceballs.skip. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then - if [ $? -ne 1 ]; then - error "$packagename : svn died during export. Skipping sourceball." - else - msg "$packagename : no longer in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - elif [ -z "$(ls -A "$packagename/repos")" ]; then - warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then - if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then - msg "$packagename : source hosting no longer required by license. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - else - remove_old "$packagename/repos/" - fi -done - -script_unlock |