summaryrefslogtreecommitdiffstats
path: root/cron-jobs
diff options
context:
space:
mode:
authorAaron Griffin <aaronmgriffin@gmail.com>2008-05-15 18:05:36 +0200
committerAaron Griffin <aaronmgriffin@gmail.com>2008-05-15 18:05:36 +0200
commitc544d99f907014a5f40356d79b9fb518b559372d (patch)
treeb85d66e2e6dd45ae4acbf3a6f03ddc3b770bfc0a /cron-jobs
parent39298cf09e2e88b1f6aec779170990675baf8992 (diff)
downloaddbscripts-c544d99f907014a5f40356d79b9fb518b559372d.tar.gz
dbscripts-c544d99f907014a5f40356d79b9fb518b559372d.tar.xz
More cron job cleanup and fixes
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
Diffstat (limited to 'cron-jobs')
-rwxr-xr-xcron-jobs/cleanup2.sh46
-rw-r--r--cron-jobs/createFileLists79
-rwxr-xr-xcron-jobs/ftpdir-cleanup111
3 files changed, 159 insertions, 77 deletions
diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh
deleted file mode 100755
index 4a79654..0000000
--- a/cron-jobs/cleanup2.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-usage() {
- echo "Usage: $0 repo architecture"
-}
-
-getpkgname() {
- local tmp
-
- tmp=${1##*/}
- tmp=${tmp%.pkg.tar.gz}
- tmp=${tmp%-i686}
- tmp=${tmp%-x86_64}
- echo ${tmp%-*-*}
-}
-
-FTPBASEDIR="/home/ftp"
-FTPDIR=${FTPBASEDIR}/${1}/os/${2}
-DBFILE=${FTPDIR}/${1}.db.tar.gz
-OBSOLETEFILES=""
-
-if [ $# -lt 2 -o ! -f ${DBFILE} ]; then
- usage
- exit 1
-fi
-
-TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1
-cd ${TMPDIR}
-tar xzf ${DBFILE}
-
-cd ${FTPDIR}
-for pkgfile in *.pkg.tar.gz; do
- pkgname="$(getpkgname ${pkgfile})"
- for p in ${FTPDIR}/${pkgname}-*; do
- if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then
- continue 2
- fi
- done
- OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}"
-done
-
-cd - >/dev/null
-rm -rf ${TMPDIR}
-
-echo -ne "DIRECTORY:\n${FTPDIR}\n\n"
-echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n"
diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists
new file mode 100644
index 0000000..02af296
--- /dev/null
+++ b/cron-jobs/createFileLists
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+reposdir=/home/ftp/
+targetdir=/home/pierre/public_html/test-repo/
+repos="core extra unstable testing community"
+arches="i686 x86_64"
+
+if [ -f "/tmp/createFileList.lock" ]; then
+ echo "Error: createFileList allready in progress."
+ exit 1
+fi
+
+touch "/tmp/createFileList.lock" || exit 1
+TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+
+getpkgname() {
+ local tmp
+
+ tmp=${1##*/}
+ tmp=${tmp%.pkg.tar.gz}
+ tmp=${tmp%-i686}
+ tmp=${tmp%-x86_64}
+ tmp=${tmp%-any}
+ echo $tmp
+}
+
+cd $reposdir
+for repo in $repos; do
+ REPO_DB_FILE=${repo}.files.tar.gz
+ for arch in $arches; do
+ repodir=${repo}/os/${arch}/
+ cached="no"
+
+ # extract old file archive
+ if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then
+ mkdir -p ${CACHEDIR}/${repodir}
+ bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir}
+ cached="yes"
+ fi
+
+ # create file lists
+ for pkg in $repodir*.pkg.tar.gz; do
+ basename=$(basename $pkg)
+ pkgname=$(getpkgname $basename)
+ tmppkgdir=${TMPDIR}/${repodir}${pkgname}/
+ mkdir -p $tmppkgdir
+ if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then
+# echo "cache: $pkgname"
+ mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files
+ else
+# echo "$repo/$arch: $pkgname"
+ echo '%FILES%' > ${tmppkgdir}files
+ bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files
+ cached="no"
+ fi
+ done
+
+ # create new file archive
+ if [ "$cached" == "no" ]; then
+ # at least one package has changed, so let's rebuild the archive
+# echo "creating ${REPO_DB_FILE}/${arch}"
+ pkgdir=${targetdir}${repodir}
+ mkdir -p $pkgdir
+ cd ${TMPDIR}/${repodir}
+ [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old"
+ [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old"
+ bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} *
+ fi
+
+ cd $reposdir
+ done
+done
+
+cd - >/dev/null
+rm -rf $TMPDIR || exit 1
+rm -rf $CACHEDIR || exit 1
+rm -f "/tmp/createFileList.lock" || exit 1
+# echo 'done'
diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 18e26f2..83c7cf1 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -1,45 +1,64 @@
#!/bin/bash
-usage() {
- echo "Usage: $0 repo architecture"
-}
+if [ $# -ne 2 ]; then
+ echo "usage: $(basename $0) <reponame> <arch>"
+ exit 1
+fi
+
+reponame=$1
+arch=$2
+
+##### Arch specific stuff. TODO make this configurable #####
+ftppath="/home/ftp/$reponame/os/$arch/"
+############################################################
+
+if [ ! -d "$ftppath" ]; then
+ echo "FTP path '$ftppath' does not exist"
+ exit 1
+fi
+
+if [ ! -f /etc/makepkg.conf ]; then
+ echo "/etc/makepkg.conf not found! Aborting"
+ exit 1
+fi
+
+. /etc/makepkg.conf
getpkgname() {
- local tmp
+ local tmp
- tmp=${1##*/}
- tmp=${tmp%.pkg.tar.gz}
- tmp=${tmp%-i686}
- tmp=${tmp%-x86_64}
- echo ${tmp%-*-*}
+ tmp=${1##*/}
+ tmp=${tmp%$PKGEXT}
+ tmp=${tmp%-$arch}
+ echo ${tmp%-*-*}
}
-reponame=$1
-arch=$2
+getpkgname_ver() {
+ local tmp
+
+ tmp=${1##*/}
+ tmp=${tmp%$PKGEXT}
+ echo ${tmp%-$arch}
+}
-FTPBASEDIR="/home/ftp"
-FTPDIR=${FTPBASEDIR}/$reponame/os/$arch
-DBFILE=${FTPDIR}/$reponame.db.tar.gz
MISSINGFILES=""
DELETEFILES=""
-
-if [ $# -lt 2 -o ! -f ${DBFILE} ]; then
- usage
- exit 1
-fi
+EXTRAFILES=""
TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1
-cd ${TMPDIR}
-tar xzf ${DBFILE}
+cd "${TMPDIR}"
+/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION"
+
for pkg in *; do
- filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1)
+ filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1)
[ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz"
- if [ ! -f ${FTPDIR}/${filename} ]; then
+
+ if [ ! -f "${ftppath}/${filename}" ]; then
MISSINGFILES="${MISSINGFILES} ${filename}"
else
pkgname="$(getpkgname ${filename})"
- for otherfile in ${FTPDIR}/${pkgname}-*; do
+ for otherfile in ${ftppath}/${pkgname}-*; do
otherfile="$(basename ${otherfile})"
if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then
DELETEFILES="${DELETEFILES} ${otherfile}"
@@ -48,13 +67,43 @@ for pkg in *; do
fi
done
-cd - >/dev/null
+cd "$ftppath"
+for pkg in *$PKGEXT; do
+ pkgname="$(getpkgname_ver ${filename})"
+ if [ ! -d "$TMPDIR/$pkgname" ]; then
+ EXTRAFILES="$EXTRAFILES $pkg"
+ fi
+done
+
+
+
+cd "$ftppath"
rm -rf ${TMPDIR}
-echo -ne "DIRECTORY:\n${FTPDIR}\n\n"
-echo -ne "DELETEFILES:\n${DELETEFILES}\n\n"
-echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n"
+if [ -n "${DELETEFILES}" ]; then
+ #rm -f ${DELETEFILES}
+ #mv ${DELETEFILES} /home/package-cleanup/
+ echo ""
+fi
-cd ${FTPDIR}
-[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/
-cd - >/dev/null
+echo "Scan complete for $reponame ($arch) at ${ftppath}"
+if [ -n "$DELETEFILES" ]; then
+ echo " The following files have been moved to package-cleanup:"
+ for f in $DELETEFILES; do
+ echo " $f"
+ done
+fi
+echo ""
+if [ -n "$MISSINGFILES" ]; then
+ echo " The following files are missing in the repo:"
+ for f in $MISSINGFILES; do
+ echo " $f"
+ done
+fi
+echo ""
+if [ -n "$EXTRAFILES" ]; then
+ echo " The following files are in the repo but not the db:"
+ for f in $EXTRAFILES; do
+ echo " $f"
+ done
+fi