sourceballs: Make cleanup more efficient

Moved all cleanup related code in sourceballs-cleanup script and moved,
now common, functions in db-functions.  The cleanup script is now ran
after all the new sourceballs have been fetched.

Signed-off-by: Eric Bélanger <snowmaniscool@gmail.com>

4 files changed, 118 insertions, 54 deletions

diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index b7a4885..ccb00ad 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -89,4 +89,6 @@ if [ -n "$FAILED_PKGS" ]; then
     echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt"
 fi
 
+$dirname/../misc-scripts/sourceballs-cleanup
+
 cleanup
diff --git a/db-functions b/db-functions
index df50f22..d688df6 100644
--- a/db-functions
+++ b/db-functions
@@ -170,4 +170,25 @@ chk_license() {
     return 1
 }
 
+pkgname_from_src() {
+    local tmp
+    tmp=${1##*/}
+    tmp=${tmp%$SRCEXT}
+    for a in ${ARCHES[@]}; do
+        tmp=${tmp%-$a}
+    done
+    tmp=${tmp%-any}
+    echo ${tmp%-*-*}
+}
+
+pkgver_from_src() {
+    tmp=${1##*/}
+    tmp=${tmp%$SRCEXT}
+    for a in ${ARCHES[@]}; do
+        tmp=${tmp%-$a}
+    done
+    tmp=${tmp%-any}
+    echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g'
+}
+
 # vim: set ts=4 sw=4 noet ft=sh:
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
index 8899fca..610d7f1 100755
--- a/misc-scripts/make-sourceball
+++ b/misc-scripts/make-sourceball
@@ -40,27 +40,6 @@ die() {
     cleanup 1
 }
 
-pkgname_from_src() {
-    local tmp
-    tmp=${1##*/}
-    tmp=${tmp%$SRCEXT}
-    for a in ${ARCHES[@]}; do
-        tmp=${tmp%-$a}
-    done
-    tmp=${tmp%-any}
-    echo ${tmp%-*-*}
-}
-
-pkgver_from_src() {
-    tmp=${1##*/}
-    tmp=${tmp%$SRCEXT}
-    for a in ${ARCHES[@]}; do
-        tmp=${tmp%-$a}
-    done
-    tmp=${tmp%-any}
-    echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g'
-}
-
 create_srcpackage() {
     if [ -d "$1" ]; then
         pushd "$1" >/dev/null
@@ -94,38 +73,6 @@ create_srcpackage() {
     fi
 }
 
-remove_old() {
-    if [ -d "$1" ]; then
-        pushd "$1" >/dev/null
-        PKGVERS=""
-        for repo in *; do
-            cd "$repo"
-            . "$BUILDSCRIPT"
-            PKGVERS="$PKGVERS $pkgver-$pkgrel"
-            cd ..
-        done
-
-        for srcpkg in "$srcpath/$packagename-"*; do
-            [ -f "$srcpkg" ] || continue
-            if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
-                skip=0
-                pver="$(pkgver_from_src $srcpkg)"
-                for v in $PKGVERS; do
-                    if [ "$v" = "$pver" ]; then
-                        skip=1
-                        break
-                    fi
-                done
-                if [ $skip -ne 1 ]; then 
-                    mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR
-                fi
-            fi
-        done
-
-        popd >/dev/null
-    fi
-}
-
 trap ctrl_c 2
 trap cleanup 0 1
 
@@ -135,7 +82,6 @@ set_umask
 cd "$WORKDIR"
 
 if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
-    remove_old "$packagename/repos/"
     create_srcpackage "$packagename/repos/$reponame-$_arch"
 else
     die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'"
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
new file mode 100755
index 0000000..0a1ac4d
--- /dev/null
+++ b/misc-scripts/sourceballs-cleanup
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+. "$(dirname $0)/../db-functions"
+. "$(dirname $0)/../config"
+
+srcpath="$FTP_BASE/sources/"
+logfile="$srcpath/cleanup.txt"
+
+LOCKFILE="/tmp/.sourceball-cleanup.lock"
+WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID"
+
+cleanup () {
+    restore_umask
+    rm -rf "$WORKDIR"
+    rm -f "$LOCKFILE"
+    exit 0
+}
+
+ctrl_c() {
+    echo "Interrupted" >&2
+    cleanup 0
+}
+
+remove_old() {
+    if [ -d "$1" ]; then
+        pushd "$1" >/dev/null
+        PKGVERS=""
+        for repo in *; do
+            cd "$repo"
+            . "$BUILDSCRIPT"
+            PKGVERS="$PKGVERS $pkgver-$pkgrel"
+            cd ..
+        done
+
+        for srcpkg in "$srcpath/$packagename-"*; do
+            [ -f "$srcpkg" ] || continue
+            if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
+                skip=0
+                pver="$(pkgver_from_src $srcpkg)"
+                for v in $PKGVERS; do
+                    if [ "$v" = "$pver" ]; then
+                        skip=1
+                        break
+                    fi
+                done
+                if [ $skip -ne 1 ]; then 
+                    mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR
+                fi
+            fi
+        done
+
+        popd >/dev/null
+    fi
+}
+
+if [ -f "$LOCKFILE" ]; then
+    owner="$(/usr/bin/stat -c %U $LOCKFILE)"
+    echo "error: source tarball generation is already in progress (started by $owner)"
+    exit 1
+fi
+
+trap cleanup 0
+trap ctrl_c 2
+
+/bin/touch "$LOCKFILE"
+
+#adjust the nice level to run at a lower priority
+/usr/bin/renice +10 -p $$ > /dev/null
+
+set_umask
+/bin/mkdir -p "$WORKDIR"
+cd "$WORKDIR"
+
+[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old"
+echo  "Orphaned sourceballs:" > "$logfile"
+
+for sourceball in "$srcpath"/*$SRCEXT; do
+    packagename=$(basename $sourceball)
+    packagename=${packagename%-*-*$SRCEXT}
+
+    if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then
+      echo "$packagename : no longer in svn.  Removing sourceball." >> "$logfile"
+      mv $sourceball $SOURCE_CLEANUP_DESTDIR
+    elif [ -z "$(ls -A "$packagename/repos")" ]; then
+      echo "$packagename : no longer in repos but trunk is still in svn.  Removing sourceball." >> "$logfile"
+      mv $sourceball $SOURCE_CLEANUP_DESTDIR
+    elif !  source  "$packagename/trunk/$BUILDSCRIPT" &&  chk_license ${license[@]}; then
+      echo "$packagename : source hosting no longer required by license.  Removing sourceball." >> "$logfile"
+      mv $sourceball $SOURCE_CLEANUP_DESTDIR
+    else
+      remove_old "$packagename/repos/"
+    fi
+done
+
+cleanup 0