diff options
| -rw-r--r-- | config | 6 | ||||
| -rwxr-xr-x | cron-jobs/check_archlinux/check_packages.py | 23 | ||||
| -rwxr-xr-x | cron-jobs/check_archlinux/parse_pkgbuilds.sh | 9 | ||||
| -rwxr-xr-x | cron-jobs/devlist-mailer | 10 | ||||
| -rw-r--r-- | db-functions | 5 | ||||
| -rwxr-xr-x | db-move | 15 | ||||
| -rwxr-xr-x | db-remove | 2 | ||||
| -rwxr-xr-x | db-update | 15 | ||||
| -rwxr-xr-x | misc-scripts/make-sourceball | 46 |
9 files changed, 87 insertions, 44 deletions
@@ -1,5 +1,7 @@ -FTP_BASE="/srv/ftp/" +FTP_BASE="/srv/ftp" SVN_PATH="file:///srv/svn-packages" -STAGING="$HOME/staging/" +STAGING="$HOME/staging" TMPDIR="/srv/tmp" +# To be used if $BUILDSCRIPT is removed from makepkg.conf +#BUILDSCRIPT="PKGBUILD" diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 69b14c3..f0fd4a5 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -49,11 +49,11 @@ class Depend: self.version = version self.mod = mod -def parse_pkgbuilds(repos): +def parse_pkgbuilds(repos,arch): oldcwd = os.getcwd() os.chdir(absroot) for repo in repos: - data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo) + data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo) parse_data(repo,data) os.chdir(oldcwd) @@ -292,24 +292,27 @@ def print_usage(): print "Usage: ./check_packages.py [OPTION]" print "" print "Options:" - print " --abs-tree=<path> Check specified tree (default : /var/abs)" - print " --repos=<r1,r2,...> Check specified repos (default : core,extra)" + print " --abs-tree=<path> Check the specified tree (default : /var/abs)" + print " --repos=<r1,r2,...> Check the specified repos (default : core,extra)" + print " --arch=<i686|x86_64> Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" print "\n Check core and extra in existing abs tree:" - print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" print "\n Check community:" - print " ./check_packages.py --abs-tree=/var/abs --repos=community" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" print "" ## Default path to the abs root directory absroot = "/var/abs" ## Default list of repos to check repos = ['core', 'extra'] +## Default arch +arch = "i686" try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="]) + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="]) except getopt.GetoptError: print_usage() sys.exit() @@ -319,6 +322,8 @@ if opts != []: absroot = a elif o in ("--repos"): repos = a.split(",") + elif o in ("--arch"): + arch = a else: print_usage() sys.exit() @@ -339,11 +344,11 @@ loadrepos = set([]) for repo in repos: loadrepos = loadrepos | set(get_repo_hierarchy(repo)) -print_heading("Integrity Check") +print_heading("Integrity Check " + arch + " of " + ",".join(repos)) print "\nPerforming integrity checks..." print "==> parsing pkgbuilds" -parse_pkgbuilds(loadrepos) +parse_pkgbuilds(loadrepos,arch) repopkgs = {} for name,pkg in packages.iteritems(): diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index d4205ae..47aec89 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Usage : parse_pkgbuilds.sh arch <pkgbuilds_dir1,dir2,...> +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + parse() { unset pkgname pkgver pkgrel unset depends makedepends conflicts provides @@ -67,10 +72,12 @@ find_pkgbuilds() { fi } -if [ -z "$*" ]; then +if [ -z "$1" -o -z "$2" ]; then exit 1 fi +CARCH=$1 +shift for dir in "$@"; do find_pkgbuilds $dir done diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index b24d530..ca2e46b 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -6,9 +6,13 @@ LIST="arch-dev-public@archlinux.org" #LIST="aaronmgriffin@gmail.com" FROM="repomaint@archlinux.org" -SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" -if [ $# -eq 1 ]; then - SUBJECT="$1" +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" fi stdin="$(cat)" diff --git a/db-functions b/db-functions index b196d57..a2e2f72 100644 --- a/db-functions +++ b/db-functions @@ -3,7 +3,10 @@ # Random integrity things [ "$UID" = "" ] && UID=$(uid) -BASEDIR="$(dirname $0)" +if [ -z "$BASEDIR" ]; then + BASEDIR="$(dirname $0)" +fi + if [ -f "$BASEDIR/config" ]; then . "$BASEDIR/config" fi @@ -84,23 +84,18 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . - else - touch "$repofrom.db.tar.$DB_COMPRESSION" + /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + #use '*' to move the old DB too + mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from + echo " Package files will be cleaned up automatically" fi - /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" - #use '*' to move the old DB too - mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from - echo " Package files will be cleaned up automatically" - if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . - else - touch "$repoto.db.tar.$DB_COMPRESSION" fi /bin/cp "$ftppath_from/$_pkgfile" . - /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" + /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to else @@ -74,7 +74,7 @@ fi echo " Removing from $reponame DB file" cd build/ -/usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename +/usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename /bin/cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" @@ -71,6 +71,11 @@ repo_lock $reponame $_arch /bin/mkdir -p "$WORKDIR/build" cd "$WORKDIR" +# copy the db file into our working area +if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ +fi + echo "Updating DB for $reponame $_arch" if [ -d "${stagedir}64" ]; then @@ -90,9 +95,7 @@ if [ -d "${stagedir}/add" ]; then echo "Please delete staging/<reponame>/{add,del}" echo " and ensure you are using the newest devtools" echo "--------------------------------------------------" - if [ -e "${stagedir}/add/"* ]; then - /bin/mv "${stagedir}/add/"* "$stagedir/" - fi + /bin/mv "${stagedir}/add/"* "$stagedir/" fi to_add="" @@ -145,7 +148,7 @@ if [ -n "$ADDPKGS" ]; then pkgs="" for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $pkgs + /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs else rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when adding packages" @@ -167,9 +170,9 @@ else echo "Nothing to copy, no work done" fi -if [ -n "$ADDPKGS" ]; then +if [ -n "$to_add" ]; then echo "Cleaning staging dir" - /bin/rm $ADDPKGS + /bin/rm $to_add fi cleanup diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 0006092..91dc122 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -1,11 +1,15 @@ #!/bin/bash +# Allowed licenses: build only for licenses in this array +ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') + if [ $# -ne 3 ]; then echo "usage: $(basename $0) <packagename> <repo> <arch>" exit 1 fi -. "$(dirname $0)/../db-functions" +BASEDIR="$(dirname $0)/../" +. "$BASEDIR/db-functions" source_makepkg @@ -13,15 +17,12 @@ packagename="$1" reponame="$2" arch="$3" -##### Arch specific stuff. TODO make this configurable ##### -srcpath="/srv/ftp/sources/" -svnpath="file:///srv/svn-packages/$packagename/" -############################################################ +srcpath="$FTP_BASE/sources/" WORKDIR="/tmp/make-sourceball.$packagename.$UID" cleanup() { - # unlock + restore_umask rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -36,10 +37,32 @@ die() { cleanup 1 } +#usage: chk_license ${license[@]}" +chk_license() { + local l + for l in "$@"; do + l="$(echo $l | tr '[:upper:]' '[:lower:]')" + for allowed in ${ALLOWED_LICENSES[@]}; do + allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" + if [ "$l" = "$allowed" ]; then + return 0 + fi + done + done + + return 1 +} + create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null . "$BUILDSCRIPT" + if ! chk_license ${license[@]}; then + echo "Package license does not require source tarballs. Doing nothing" + echo " license => (${license[@]})" + cleanup 0 + fi + if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then popd >/dev/null return 1 @@ -53,13 +76,13 @@ create_srcpackage() { mkdir -p "$srcpath" fi #Remove old sourceballs - for pkg in "$srcpath/$pkgname-*"; do + for pkg in "$srcpath/$pkgname-"*; do pkg="$(basename $pkg)" - if "$(getpkgname $pkg)" == "$pkgname" ]; then + if [ "$(getpkgname $pkg)" == "$pkgname" ]; then rm -f "$srcpath/$pkg" fi done - cp $pkg_file "$srcpath" + cp "$pkgname/$pkg_file" "$srcpath" return 0 fi @@ -68,12 +91,13 @@ create_srcpackage() { trap ctrl_c 2 trap cleanup 0 1 +set_umask /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" echo "Creating Source tarball for $packagename ($reponame-$arch)" -if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then +if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then exit 0 @@ -81,7 +105,7 @@ if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then #trunk sometimes has updated URLs echo ":: Failed to download source, attempting trunk build" rm -rf "$packagename" - if /usr/bin/svn export -q "$svnpath/trunk" "$packagename"; then + if /usr/bin/svn export -q "$SVN_PATH/$packagename/trunk" "$packagename"; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then echo ":: Source package complete: $pkg_file" |