From 39298cf09e2e88b1f6aec779170990675baf8992 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 18:49:20 -0400 Subject: Cron cleanup, and only copy DB files once This would break all adds if there were any deletes. We definitely don't want that. Also, cleanup of cron scripts in the same commit because I'm lazy Signed-off-by: Aaron Griffin --- cron-jobs/genpkglist | 86 ++++++++++++------------------------------------- db-update | 50 +++++++++++++++------------- misc-scripts/find-dupes | 77 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 88 deletions(-) create mode 100755 misc-scripts/find-dupes diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist index 31b7c78..0b5d76f 100755 --- a/cron-jobs/genpkglist +++ b/cron-jobs/genpkglist @@ -1,74 +1,28 @@ #!/bin/bash -# -# genpkglist -# -# Generates a text package database for use with the setup script -# (also used to check for missing packages in the download directory) -# -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} +reponame=$1 +arch=$2 -pkgfile="$(pwd)/packages.txt" -ftppath=$1 -repotag=$2 +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ -rm -f $pkgfile -for package in *; do - if [ -d "$package/$repotag/" ]; then - cd "$package/$repotag/" - if [ -f PKGBUILD ]; then - . PKGBUILD - if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile - elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile - else - echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 - fi - fi - cd .. - fi -done +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi -cd $ftppath -unset DUPES DUPEFILES last -for pkg in *.pkg.tar.gz; do - pkgname=$(getpkgname $pkg) - if [ "$last" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - DUPEFILES="$DUPEFILES $pkg" - fi - last=$pkgname -done +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi -showdupes() { - done= - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - ls -l $i | awk '{print $6" "$7" "$8" "$9}' - done=1 - else - [ "$done" = "1" ] && return - fi - done -} +. /etc/makepkg.conf -if [ "$DUPES" ]; then - DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) - echo "Possible Dupes for $ftppath (remove old versions)" - echo "Date Filename" - for dupe in $DUPES; do - showdupes $dupe - done -fi +cd "$ftppath" +ls -1 *$PKGEXT diff --git a/db-update b/db-update index 9d0ecad..93672f5 100755 --- a/db-update +++ b/db-update @@ -83,7 +83,15 @@ trap cleanup 0 /bin/touch "$LOCKFILE" -/bin/mkdir -p "$WORKDIR" +/bin/mkdir -p "$WORKDIR/build" +cd "$WORKDIR" + +# copy the db file into our working area +if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ +else + touch "build/$reponame.db.tar.$DB_COMPRESSION" +fi echo "Updating DB for $reponame $arch" @@ -126,17 +134,15 @@ if [ -n "$ADDPKGS" ]; then done if [ -n "$to_add" ]; then + cd "$WORKDIR/build/" + /bin/cp $to_add . - cd "$WORKDIR" - [ -d build/ ] || mkdir build + pkgs="" + for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - /bin/cp $to_add build/ - - cd build/ - /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add + /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $pkgs else + rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when adding packages" fi else @@ -166,26 +172,22 @@ if [ -n "$REMPKGS" ]; then if [ -d "$_pkgname/repos/$svnrepo" ]; then echo " WARNING: $_pkgname still exists in $svnrepo" else - to_rem="$to_rem $pkg" + to_rem="$to_rem $_pkgname" fi done if [ -n "$to_rem" ]; then - cd "$WORKDIR" - [ -d build/ ] || mkdir build - - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + cd "$WORKDIR/build/" - cd build/ + #NOTE: to_rem consists of package NAMES only /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem - for rem in $to_rem; do - if [ -f "$ftppath/$rem" ]; then - /bin/rm "$ftppath/$rem" - fi + for rem in $REMPKGS; do + rem="$(basename $rem)" + /bin/rm -f "$ftppath/$rem" done else + rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when removing packages" fi else @@ -196,11 +198,15 @@ fi if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" /bin/cp -r "$WORKDIR/build/"* "$ftppath" - echo "Cleaning staging dir" - /bin/rm $to_add $to_rem + else echo "Nothing to copy, no work done" fi +if [ -n "$ADDPKGS" -o -n "$REMPKGS" ]; then + echo "Cleaning staging dir" + /bin/rm $ADDPKGS $REMPKGS +fi + cleanup # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/find-dupes b/misc-scripts/find-dupes new file mode 100755 index 0000000..82cd1a4 --- /dev/null +++ b/misc-scripts/find-dupes @@ -0,0 +1,77 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf + +cd $ftppath + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$CARCH} + echo ${tmp%-*-*} +} + +showdupes() { + done="" + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + if [ "$pkgname" = "$1" ]; then + lastmod="$(stat -c %y $i | cut -d. -f1)" + fname="$(basename $i)" + + echo "$lastmod $fname" + + done=1 + else + if [ "$done" = "1" ]; then + return + fi + fi + done +} + +echo "Scanning for duplicate packages in '$reponame' ($arch)" +DUPES="" +lastpkg="" + +for pkg in *.pkg.tar.gz; do + pkgname="$(getpkgname $pkg)" + if [ "$lastpkg" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + fi + lastpkg=$pkgname +done + +if [ "$DUPES" ]; then + DUPES="$(echo $DUPES | sed 's| |\n|g' | sort -u)" + echo "Date Filename" + for dupe in $DUPES; do + showdupes $dupe + done +fi -- cgit v1.2.3-24-g4f1b