More cron job cleanup and fixes

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>

3 files changed, 159 insertions, 77 deletions

diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh
deleted file mode 100755
index 4a79654..0000000
--- a/cron-jobs/cleanup2.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-usage() {
-  echo "Usage: $0 repo architecture"
-}
-
-getpkgname() {
-  local tmp
-
-  tmp=${1##*/}
-  tmp=${tmp%.pkg.tar.gz}
-  tmp=${tmp%-i686}
-  tmp=${tmp%-x86_64}
-  echo ${tmp%-*-*}
-}
-
-FTPBASEDIR="/home/ftp"
-FTPDIR=${FTPBASEDIR}/${1}/os/${2}
-DBFILE=${FTPDIR}/${1}.db.tar.gz
-OBSOLETEFILES=""
-
-if [ $# -lt 2 -o ! -f ${DBFILE} ]; then
-  usage
-  exit 1
-fi
-
-TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1
-cd ${TMPDIR}
-tar xzf ${DBFILE}
-
-cd ${FTPDIR}
-for pkgfile in *.pkg.tar.gz; do
-  pkgname="$(getpkgname ${pkgfile})"
-  for p in ${FTPDIR}/${pkgname}-*; do
-    if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then
-      continue 2
-    fi
-  done
-  OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}"
-done
-
-cd - >/dev/null
-rm -rf ${TMPDIR}
-
-echo -ne "DIRECTORY:\n${FTPDIR}\n\n"
-echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n"
diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists
new file mode 100644
index 0000000..02af296
--- /dev/null
+++ b/cron-jobs/createFileLists
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+reposdir=/home/ftp/
+targetdir=/home/pierre/public_html/test-repo/
+repos="core extra unstable testing community"
+arches="i686 x86_64"
+
+if [ -f "/tmp/createFileList.lock" ]; then
+    echo "Error: createFileList allready in progress."
+    exit 1
+fi
+
+touch "/tmp/createFileList.lock" || exit 1
+TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+
+getpkgname() {
+    local tmp
+
+    tmp=${1##*/}
+    tmp=${tmp%.pkg.tar.gz}
+    tmp=${tmp%-i686}
+    tmp=${tmp%-x86_64}
+    tmp=${tmp%-any}
+    echo $tmp
+}
+
+cd $reposdir
+for repo in $repos; do
+    REPO_DB_FILE=${repo}.files.tar.gz
+    for arch in $arches; do
+        repodir=${repo}/os/${arch}/
+        cached="no"
+
+        # extract old file archive
+        if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then
+            mkdir -p ${CACHEDIR}/${repodir}
+            bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir}
+            cached="yes"
+        fi
+
+        # create file lists
+        for pkg in $repodir*.pkg.tar.gz; do
+            basename=$(basename $pkg)
+            pkgname=$(getpkgname $basename)
+            tmppkgdir=${TMPDIR}/${repodir}${pkgname}/
+            mkdir -p $tmppkgdir
+            if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then
+#               echo "cache: $pkgname"
+                mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files
+            else
+#               echo "$repo/$arch: $pkgname"
+                echo '%FILES%' > ${tmppkgdir}files
+                bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files
+                cached="no"
+            fi
+        done
+
+        # create new file archive
+        if [ "$cached" == "no" ]; then
+            # at least one package has changed, so let's rebuild the archive
+#           echo "creating ${REPO_DB_FILE}/${arch}"
+            pkgdir=${targetdir}${repodir}
+            mkdir -p $pkgdir
+            cd ${TMPDIR}/${repodir}
+            [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old"
+            [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old"
+            bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} *
+        fi
+
+        cd $reposdir
+    done
+done
+
+cd - >/dev/null
+rm -rf $TMPDIR || exit 1
+rm -rf $CACHEDIR || exit 1
+rm -f "/tmp/createFileList.lock" || exit 1
+# echo 'done'
diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 18e26f2..83c7cf1 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -1,45 +1,64 @@
 #!/bin/bash
 
-usage() {
-  echo "Usage: $0 repo architecture"
-}
+if [ $# -ne 2 ]; then
+	echo "usage: $(basename $0) <reponame> <arch>"
+	exit 1
+fi
+
+reponame=$1
+arch=$2
+
+##### Arch specific stuff. TODO make this configurable #####
+ftppath="/home/ftp/$reponame/os/$arch/"
+############################################################
+
+if [ ! -d "$ftppath" ]; then
+    echo "FTP path '$ftppath' does not exist"
+    exit 1
+fi
+
+if [ ! -f /etc/makepkg.conf ]; then
+    echo "/etc/makepkg.conf not found! Aborting"
+    exit 1
+fi
+
+. /etc/makepkg.conf
 
 getpkgname() {
-  local tmp
+	local tmp
 
-  tmp=${1##*/}
-  tmp=${tmp%.pkg.tar.gz}
-  tmp=${tmp%-i686}
-  tmp=${tmp%-x86_64}
-  echo ${tmp%-*-*}
+	tmp=${1##*/}
+	tmp=${tmp%$PKGEXT}
+	tmp=${tmp%-$arch}
+	echo ${tmp%-*-*}
 }
 
-reponame=$1
-arch=$2
+getpkgname_ver() {
+	local tmp
+
+	tmp=${1##*/}
+	tmp=${tmp%$PKGEXT}
+	echo ${tmp%-$arch}
+}
 
-FTPBASEDIR="/home/ftp"
-FTPDIR=${FTPBASEDIR}/$reponame/os/$arch
-DBFILE=${FTPDIR}/$reponame.db.tar.gz
 MISSINGFILES=""
 DELETEFILES=""
-
-if [ $# -lt 2 -o ! -f ${DBFILE} ]; then
-  usage
-  exit 1
-fi
+EXTRAFILES=""
 
 TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1
 
-cd ${TMPDIR}
-tar xzf ${DBFILE}
+cd "${TMPDIR}"
+/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION"
+
 for pkg in *; do
-  filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1)
+  filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1)
   [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz"
-  if [ ! -f ${FTPDIR}/${filename} ]; then
+
+  if [ ! -f "${ftppath}/${filename}" ]; then
     MISSINGFILES="${MISSINGFILES} ${filename}"
   else
     pkgname="$(getpkgname ${filename})"
-    for otherfile in ${FTPDIR}/${pkgname}-*; do
+    for otherfile in ${ftppath}/${pkgname}-*; do
       otherfile="$(basename ${otherfile})"
       if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then
         DELETEFILES="${DELETEFILES} ${otherfile}"
@@ -48,13 +67,43 @@ for pkg in *; do
   fi
 done
 
-cd - >/dev/null
+cd "$ftppath"
+for pkg in *$PKGEXT; do
+    pkgname="$(getpkgname_ver ${filename})"
+    if [ ! -d "$TMPDIR/$pkgname" ]; then
+        EXTRAFILES="$EXTRAFILES $pkg"
+    fi
+done 
+
+
+
+cd "$ftppath"
 rm -rf ${TMPDIR}
 
-echo -ne "DIRECTORY:\n${FTPDIR}\n\n"
-echo -ne "DELETEFILES:\n${DELETEFILES}\n\n"
-echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n"
+if [ -n "${DELETEFILES}" ]; then
+    #rm -f ${DELETEFILES}
+    #mv ${DELETEFILES} /home/package-cleanup/
+    echo ""
+fi
 
-cd ${FTPDIR}
-[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/
-cd - >/dev/null
+echo "Scan complete for $reponame ($arch) at ${ftppath}"
+if [ -n "$DELETEFILES" ]; then
+    echo "    The following files have been moved to package-cleanup:"
+    for f in $DELETEFILES; do
+        echo "        $f"
+    done
+fi
+echo ""
+if [ -n "$MISSINGFILES" ]; then
+    echo "    The following files are missing in the repo:"
+    for f in $MISSINGFILES; do
+        echo "        $f"
+    done
+fi
+echo ""
+if [ -n "$EXTRAFILES" ]; then
+    echo "    The following files are in the repo but not the db:"
+    for f in $EXTRAFILES; do
+        echo "        $f"
+    done
+fi