diff options
| -rwxr-xr-x | cron-jobs/cleanup2.sh | 46 | ||||
| -rw-r--r-- | cron-jobs/createFileLists | 79 | ||||
| -rwxr-xr-x | cron-jobs/ftpdir-cleanup | 111 |
3 files changed, 159 insertions, 77 deletions
diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh deleted file mode 100755 index 4a79654..0000000 --- a/cron-jobs/cleanup2.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz -OBSOLETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} - -cd ${FTPDIR} -for pkgfile in *.pkg.tar.gz; do - pkgname="$(getpkgname ${pkgfile})" - for p in ${FTPDIR}/${pkgname}-*; do - if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists new file mode 100644 index 0000000..02af296 --- /dev/null +++ b/cron-jobs/createFileLists @@ -0,0 +1,79 @@ +#!/bin/bash + +reposdir=/home/ftp/ +targetdir=/home/pierre/public_html/test-repo/ +repos="core extra unstable testing community" +arches="i686 x86_64" + +if [ -f "/tmp/createFileList.lock" ]; then + echo "Error: createFileList allready in progress." + exit 1 +fi + +touch "/tmp/createFileList.lock" || exit 1 +TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + tmp=${tmp%-any} + echo $tmp +} + +cd $reposdir +for repo in $repos; do + REPO_DB_FILE=${repo}.files.tar.gz + for arch in $arches; do + repodir=${repo}/os/${arch}/ + cached="no" + + # extract old file archive + if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then + mkdir -p ${CACHEDIR}/${repodir} + bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + cached="yes" + fi + + # create file lists + for pkg in $repodir*.pkg.tar.gz; do + basename=$(basename $pkg) + pkgname=$(getpkgname $basename) + tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ + mkdir -p $tmppkgdir + if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then +# echo "cache: $pkgname" + mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + else +# echo "$repo/$arch: $pkgname" + echo '%FILES%' > ${tmppkgdir}files + bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + cached="no" + fi + done + + # create new file archive + if [ "$cached" == "no" ]; then + # at least one package has changed, so let's rebuild the archive +# echo "creating ${REPO_DB_FILE}/${arch}" + pkgdir=${targetdir}${repodir} + mkdir -p $pkgdir + cd ${TMPDIR}/${repodir} + [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" + [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" + bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} * + fi + + cd $reposdir + done +done + +cd - >/dev/null +rm -rf $TMPDIR || exit 1 +rm -rf $CACHEDIR || exit 1 +rm -f "/tmp/createFileList.lock" || exit 1 +# echo 'done' diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 18e26f2..83c7cf1 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,45 +1,64 @@ #!/bin/bash -usage() { - echo "Usage: $0 repo architecture" -} +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) <reponame> <arch>" + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$arch} + echo ${tmp%-*-*} } -reponame=$1 -arch=$2 +getpkgname_ver() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + echo ${tmp%-$arch} +} -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/$reponame/os/$arch -DBFILE=${FTPDIR}/$reponame.db.tar.gz MISSINGFILES="" DELETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi +EXTRAFILES="" TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} +cd "${TMPDIR}" +/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - if [ ! -f ${FTPDIR}/${filename} ]; then + + if [ ! -f "${ftppath}/${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" else pkgname="$(getpkgname ${filename})" - for otherfile in ${FTPDIR}/${pkgname}-*; do + for otherfile in ${ftppath}/${pkgname}-*; do otherfile="$(basename ${otherfile})" if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then DELETEFILES="${DELETEFILES} ${otherfile}" @@ -48,13 +67,43 @@ for pkg in *; do fi done -cd - >/dev/null +cd "$ftppath" +for pkg in *$PKGEXT; do + pkgname="$(getpkgname_ver ${filename})" + if [ ! -d "$TMPDIR/$pkgname" ]; then + EXTRAFILES="$EXTRAFILES $pkg" + fi +done + + + +cd "$ftppath" rm -rf ${TMPDIR} -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" -echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" +if [ -n "${DELETEFILES}" ]; then + #rm -f ${DELETEFILES} + #mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi -cd ${FTPDIR} -[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ -cd - >/dev/null +echo "Scan complete for $reponame ($arch) at ${ftppath}" +if [ -n "$DELETEFILES" ]; then + echo " The following files have been moved to package-cleanup:" + for f in $DELETEFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo:" + for f in $MISSINGFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db:" + for f in $EXTRAFILES; do + echo " $f" + done +fi |