Merge branch 'master' of file:///srv/projects/git/dbscripts

Conflicts:
	db-update

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>

9 files changed, 87 insertions, 44 deletions

diff --git a/config b/config
index 35fe446..5c39919 100644
--- a/config
+++ b/config
@@ -1,5 +1,7 @@
 
-FTP_BASE="/srv/ftp/"
+FTP_BASE="/srv/ftp"
 SVN_PATH="file:///srv/svn-packages"
-STAGING="$HOME/staging/"
+STAGING="$HOME/staging"
 TMPDIR="/srv/tmp"
+# To be used if $BUILDSCRIPT is removed from makepkg.conf
+#BUILDSCRIPT="PKGBUILD"
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 69b14c3..f0fd4a5 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -49,11 +49,11 @@ class Depend:
 		self.version = version
 		self.mod = mod
 
-def parse_pkgbuilds(repos):
+def parse_pkgbuilds(repos,arch):
 	oldcwd = os.getcwd()
 	os.chdir(absroot)
 	for repo in repos:
-		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo)
+		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo)
 		parse_data(repo,data)
 	os.chdir(oldcwd)
 
@@ -292,24 +292,27 @@ def print_usage():
 	print "Usage: ./check_packages.py [OPTION]"
 	print ""
 	print "Options:"
-	print "  --abs-tree=<path>             Check specified tree (default : /var/abs)"
-	print "  --repos=<r1,r2,...>           Check specified repos (default : core,extra)"
+	print "  --abs-tree=<path>             Check the specified tree (default : /var/abs)"
+	print "  --repos=<r1,r2,...>           Check the specified repos (default : core,extra)"
+	print "  --arch=<i686|x86_64>          Check the specified arch (default : i686)"
 	print "  -h, --help                    Show this help and exit"
 	print ""
 	print "Examples:"
 	print "\n  Check core and extra in existing abs tree:"
-	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686"
 	print "\n  Check community:"
-	print "    ./check_packages.py --abs-tree=/var/abs --repos=community"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686"
 	print ""
 
 ## Default path to the abs root directory
 absroot = "/var/abs"
 ## Default list of repos to check
 repos = ['core', 'extra']
+## Default arch
+arch = "i686"
 
 try:
-	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="])
+	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="])
 except getopt.GetoptError:
 	print_usage()
 	sys.exit()
@@ -319,6 +322,8 @@ if opts != []:
 			absroot = a
 		elif o in ("--repos"):
 			repos = a.split(",")
+		elif o in ("--arch"):
+			arch = a
 		else:
 			print_usage()
 			sys.exit()
@@ -339,11 +344,11 @@ loadrepos = set([])
 for repo in repos:
 	loadrepos = loadrepos | set(get_repo_hierarchy(repo))
 
-print_heading("Integrity Check")
+print_heading("Integrity Check " + arch + " of " + ",".join(repos))
 print "\nPerforming integrity checks..."
 
 print "==> parsing pkgbuilds"
-parse_pkgbuilds(loadrepos)
+parse_pkgbuilds(loadrepos,arch)
 
 repopkgs = {}
 for name,pkg in packages.iteritems():
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index d4205ae..47aec89 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Usage : parse_pkgbuilds.sh arch <pkgbuilds_dir1,dir2,...>
+# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra
+
+exit() { return; }
+
 parse() {
 	unset pkgname pkgver pkgrel
 	unset depends makedepends conflicts provides
@@ -67,10 +72,12 @@ find_pkgbuilds() {
 	fi
 }
 
-if [ -z "$*" ]; then
+if [ -z "$1" -o -z "$2" ]; then
 	exit 1
 fi
 
+CARCH=$1
+shift
 for dir in "$@"; do
 	find_pkgbuilds $dir
 done
diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer
index b24d530..ca2e46b 100755
--- a/cron-jobs/devlist-mailer
+++ b/cron-jobs/devlist-mailer
@@ -6,9 +6,13 @@ LIST="arch-dev-public@archlinux.org"
 #LIST="aaronmgriffin@gmail.com"
 FROM="repomaint@archlinux.org"
 
-SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")"
-if [ $# -eq 1 ]; then
-    SUBJECT="$1"
+SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")"
+if [ $# -ge 1 ]; then
+    SUBJECT="$1 $(date +"%d-%m-%Y")"
+fi
+
+if [ $# -ge 2 ]; then
+    LIST="$2"
 fi
 
 stdin="$(cat)"
diff --git a/db-functions b/db-functions
index b196d57..a2e2f72 100644
--- a/db-functions
+++ b/db-functions
@@ -3,7 +3,10 @@
 # Random integrity things
 [ "$UID" = "" ] && UID=$(uid)
 
-BASEDIR="$(dirname $0)"
+if [ -z "$BASEDIR" ]; then
+	BASEDIR="$(dirname $0)"
+fi
+
 if [ -f "$BASEDIR/config" ]; then
 	. "$BASEDIR/config"
 fi
diff --git a/db-move b/db-move
index 0e0587c..ff9677b 100755
--- a/db-move
+++ b/db-move
@@ -84,23 +84,18 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then
     # copy the db file into our working area
     if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then
         /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" .
-    else
-        touch "$repofrom.db.tar.$DB_COMPRESSION"
+        /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove"
+        #use '*' to move the old DB too
+        mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from
+        echo "    Package files will be cleaned up automatically"
     fi
 
-    /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove"
-    #use '*' to move the old DB too
-    mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from
-    echo "    Package files will be cleaned up automatically"
-
     if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then
         /bin/cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" .
-    else
-        touch "$repoto.db.tar.$DB_COMPRESSION"
     fi
 
     /bin/cp "$ftppath_from/$_pkgfile" .
-    /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add"
+    /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add"
     #use '*' to move the old DB too
     mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to
 else
diff --git a/db-remove b/db-remove
index c5b6819..acc849d 100755
--- a/db-remove
+++ b/db-remove
@@ -74,7 +74,7 @@ fi
 
 echo "    Removing from $reponame DB file"
 cd build/
-/usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename
+/usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename
 
 /bin/cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/"
 
diff --git a/db-update b/db-update
index 42f5430..77af39f 100755
--- a/db-update
+++ b/db-update
@@ -71,6 +71,11 @@ repo_lock $reponame $_arch
 /bin/mkdir -p "$WORKDIR/build"
 cd "$WORKDIR"
 
+# copy the db file into our working area
+if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then
+	/bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/
+fi
+
 echo "Updating DB for $reponame $_arch"
 
 if [ -d "${stagedir}64" ]; then
@@ -90,9 +95,7 @@ if [ -d "${stagedir}/add" ]; then
 	echo "Please delete staging/<reponame>/{add,del}"
 	echo " and ensure you are using the newest devtools"
 	echo "--------------------------------------------------"
-	if [ -e "${stagedir}/add/"* ]; then
-		/bin/mv "${stagedir}/add/"* "$stagedir/"
-	fi
+	/bin/mv "${stagedir}/add/"* "$stagedir/"
 fi
 
 to_add=""
@@ -145,7 +148,7 @@ if [ -n "$ADDPKGS" ]; then
 		pkgs=""
 		for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done
 
-		/usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $pkgs
+		/usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs
 	else
 		rm -f "build/$reponame.db.tar.$DB_COMPRESSION"
 		echo "Errors found when adding packages"
@@ -167,9 +170,9 @@ else
 	echo "Nothing to copy, no work done"
 fi
 
-if [ -n "$ADDPKGS" ]; then
+if [ -n "$to_add" ]; then
 	echo "Cleaning staging dir"
-	/bin/rm $ADDPKGS
+	/bin/rm $to_add
 fi
 
 cleanup
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
index 0006092..91dc122 100755
--- a/misc-scripts/make-sourceball
+++ b/misc-scripts/make-sourceball
@@ -1,11 +1,15 @@
 #!/bin/bash
 
+# Allowed licenses: build only for licenses in this array
+ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2')
+
 if [ $# -ne 3 ]; then
 	echo "usage: $(basename $0) <packagename> <repo> <arch>"
 	exit 1
 fi
 
-. "$(dirname $0)/../db-functions"
+BASEDIR="$(dirname $0)/../"
+. "$BASEDIR/db-functions"
 
 source_makepkg
 
@@ -13,15 +17,12 @@ packagename="$1"
 reponame="$2"
 arch="$3"
 
-##### Arch specific stuff. TODO make this configurable #####
-srcpath="/srv/ftp/sources/"
-svnpath="file:///srv/svn-packages/$packagename/"
-############################################################
+srcpath="$FTP_BASE/sources/"
 
 WORKDIR="/tmp/make-sourceball.$packagename.$UID"
 
 cleanup() {
-	# unlock
+	restore_umask
 	rm -rf "$WORKDIR"
 	[ "$1" ] && exit $1
 }
@@ -36,10 +37,32 @@ die() {
 	cleanup 1
 }
 
+#usage: chk_license ${license[@]}"
+chk_license() {
+	local l
+	for l in "$@"; do
+		l="$(echo $l | tr '[:upper:]' '[:lower:]')"
+        for allowed in ${ALLOWED_LICENSES[@]}; do
+            allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')"
+            if [ "$l" = "$allowed" ]; then
+                return 0
+            fi
+        done
+	done
+
+    return 1
+}
+
 create_srcpackage() {
     if [ -d "$1" ]; then
         pushd "$1" >/dev/null
         . "$BUILDSCRIPT"
+        if ! chk_license ${license[@]}; then
+            echo "Package license does not require source tarballs. Doing nothing"
+            echo "   license => (${license[@]})"
+            cleanup 0
+        fi
+
         if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then
             popd >/dev/null
             return 1
@@ -53,13 +76,13 @@ create_srcpackage() {
             mkdir -p "$srcpath"
         fi
         #Remove old sourceballs
-        for pkg in "$srcpath/$pkgname-*"; do
+        for pkg in "$srcpath/$pkgname-"*; do
             pkg="$(basename $pkg)"
-            if "$(getpkgname $pkg)" == "$pkgname" ]; then
+            if [ "$(getpkgname $pkg)" == "$pkgname" ]; then
                 rm -f "$srcpath/$pkg"
             fi
         done
-        cp $pkg_file "$srcpath"
+        cp "$pkgname/$pkg_file" "$srcpath"
 
         return 0
     fi
@@ -68,12 +91,13 @@ create_srcpackage() {
 trap ctrl_c 2
 trap cleanup 0 1
 
+set_umask
 /bin/mkdir -p "$WORKDIR"
 cd "$WORKDIR"
 
 echo "Creating Source tarball for $packagename ($reponame-$arch)"
 
-if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then
+if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then
     create_srcpackage "$packagename"
     if [ $? -eq 0 ]; then
         exit 0
@@ -81,7 +105,7 @@ if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then
         #trunk sometimes has updated URLs
         echo ":: Failed to download source, attempting trunk build"
         rm -rf "$packagename"
-        if /usr/bin/svn export -q "$svnpath/trunk" "$packagename"; then
+        if /usr/bin/svn export -q "$SVN_PATH/$packagename/trunk" "$packagename"; then
             create_srcpackage "$packagename"
             if [ $? -eq 0 ]; then
                 echo ":: Source package complete: $pkg_file"