Simplify sourceballs creation

* Read package lists directly from DB file
* Make SVNREPO configurable

6 files changed, 32 insertions, 56 deletions

diff --git a/config b/config
index efc4381..e149b18 100644
--- a/config
+++ b/config
@@ -1,8 +1,5 @@
 FTP_BASE="/srv/ftp"
-
-SVNREPO="file:///srv/svn-packages"
-SVNREPOCOMMUNITY="file:///srv/svn-community"
-
+SVNREPO=''
 PKGREPOS=()
 PKGPOOL=''
 SRCPOOL=''
diff --git a/config.local.gerolde b/config.local.gerolde
index 03e8a70..4501a93 100644
--- a/config.local.gerolde
+++ b/config.local.gerolde
@@ -1,3 +1,4 @@
 PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
 PKGPOOL='pool/packages'
 SRCPOOL='sources/packages'
+SVNREPO='file:///srv/svn-packages'
diff --git a/config.local.sigurd b/config.local.sigurd
index ddb5ba4..f62372c 100644
--- a/config.local.sigurd
+++ b/config.local.sigurd
@@ -1,3 +1,4 @@
 PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing')
 PKGPOOL='pool/community'
 SRCPOOL='sources/community'
+SVNREPO='file:///srv/svn-community'
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index 4fc194f..1add36e 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -9,59 +9,47 @@ set_umask
 
 dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
 
-FAILED_PKGS=""
-
 [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
 [ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old"
+[ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old"
 
 for repo in ${PKGREPOS[@]}; do
     for arch in ${ARCHES[@]}; do
         ftppath="${FTP_BASE}/$repo/os/$arch"
-        if [ ! -d "$ftppath" ]; then
-            error "FTP path does not exist: $ftppath"
+        dbfile="${ftppath}/${repo}${DBEXT}"
+        if [ ! -r "${dbfile}" ]; then
+            warning "DB file does not exist: ${dbfile}"
             continue
         fi
-        cd $ftppath
-        for pkg in *$PKGEXT; do
-            [ -f "$pkg" ] || continue
-            pkgbase=$(getpkgbase $pkg)
-            srcpkg="${pkg//$PKGEXT/$SRCEXT}"
-            srcpkg="${srcpkg//-$arch/}"
-            srcpkgname="${srcpkg%-*-*$SRCEXT}"
-            srcpkgbase="${srcpkg/$srcpkgname/$pkgbase}"
-
-            #Don't do anything for package in this 'blacklist'
-            if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then
-                continue
-            fi
-
-            #This pkgbase has already failed. No sense in trying it again this run
-            if echo $FAILED_PKGS | grep "\<$pkgbase\>" >&/dev/null; then
+        for pkg in $(bsdtar -xOf "${dbfile}" \
+            | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \
+            | sort -u); do
+            pkgbase=${pkg%/*}
+            pkgver=${pkg#*/}
+            srcpkg="${pkgbase}-${pkgver}${SRCEXT}"
+
+            # Don't do anything for package in this 'blacklist'
+            if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then
                 continue
             fi
 
-            #Use this file to 'whitelist' or force building some sourceballs,
+            # Use this file to 'whitelist' or force building some sourceballs,
             # skipping the license check
             force=""
-            if grep "^$pkgbase\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then
+            if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then
                 force="-f"
             fi
 
-            if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then
+            if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then
                 if ! $dirname/../misc-scripts/make-sourceball $force \
                     $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then
-                    FAILED_PKGS="$FAILED_PKGS $pkgbase"
+                    echo "$pkgbase" >> "${LOGDIR}/sourceballs/failed.txt"
                 fi
             fi
         done
     done
 done
 
-if [ -n "$FAILED_PKGS" ]; then
-    [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old"
-    echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt"
-fi
-
 $dirname/../misc-scripts/sourceballs-cleanup
 
 script_unlock
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
index 221d0ce..661f481 100755
--- a/misc-scripts/make-sourceball
+++ b/misc-scripts/make-sourceball
@@ -24,28 +24,26 @@ script_lock
 create_srcpackage() {
     if [ -d "$1" ]; then
         pushd "$1" >/dev/null
-	pkgver=$(. PKGBUILD; echo ${pkgver})
-	pkgrel=$(. PKGBUILD; echo ${pkgrel})
-	license=($(. PKGBUILD; echo ${license[@]}))
-	if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
-            #Removed so as not to clutter failed.txt
+        pkgver=$(. PKGBUILD; echo ${pkgver})
+        pkgrel=$(. PKGBUILD; echo ${pkgrel})
+        license=($(. PKGBUILD; echo ${license[@]}))
+        if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
+            # Removed so as not to clutter failed.txt
             #warning "$packagename license (${license[@]}) does not require source tarballs"
             cleanup 0
         else
             msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
         fi
 
-        local logfile="${LOGDIR}/sourceballs/$packagename"
-        if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then
-            popd >/dev/null
-            /bin/gzip -f -9 "$logfile"
+        local logfile="${LOGDIR}/sourceballs/$packagename.gz"
+        if ! /usr/bin/makepkg --allsource --ignorearch 2>&1 | gzip >"$logfile"; then
             die "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)"
         fi
-        /bin/rm -f "$logfile"{,.gz}
+        /bin/rm -f "$logfile"
 
         local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
 
-        cp "$pkg_file" "${FTP_BASE}/${SRCPOOL}"
+        mv "$pkg_file" "${FTP_BASE}/${SRCPOOL}"
 
         popd >/dev/null
 
@@ -57,18 +55,10 @@ set_umask
 [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
 cd "$WORKDIR"
 
-if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then
-    if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then
-	create_srcpackage "$packagename/repos/$reponame-$arch"
-    else
-	die "Package '$packagename' does not exist in repo '$reponame-$arch'"
-    fi
+if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
+  create_srcpackage "$packagename/repos/$reponame-$arch"
 else
-    if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
-	create_srcpackage "$packagename/repos/$reponame-$arch"
-    else
-	die "Package '$packagename' does not exist in repo '$reponame-$arch'"
-    fi
+  die "Package '$packagename' does not exist in repo '$reponame-$arch'"
 fi
 
 script_unlock
diff --git a/test/lib/common.inc b/test/lib/common.inc
index 14dc000..e47ae2d 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -62,7 +62,6 @@ setUp() {
 	cat <<eot > "${curdir}/../config.local"
 	FTP_BASE="${TMP}/ftp"
 	SVNREPO="file://${TMP}/svn-packages-repo"
-	SVNREPOCOMMUNITY="file://${TMP}/svn-community-repo"
 	PKGREPOS=(${PKGREPOS[@]})
 	PKGPOOL="${PKGPOOL}"
 	CLEANUP_DESTDIR="${TMP}/package-cleanup"