From 1d6421c7a12e9a07d00ec4de999ae95f7dd0874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:47:52 -0600 Subject: create-filelists: general cleanups * Specify lock name once * Use new script name everywhere * Clean up tabs/spaces and add a modeline. This isn't necessarily the one we wanted to standardize on, but I picked the one the entire file is written to at the moment. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index c9d7db9..e90da00 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,27 +3,28 @@ reposdir="/srv/ftp" targetdir="/srv/ftp" repos="core extra testing community community-testing" +lock="/tmp/create-filelists.lock" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -if [ -f "/tmp/createFileList.lock" ]; then - echo "Error: createFileList allready in progress." +if [ -f "$lock" ]; then + echo "Error: create-filelists already in progress." exit 1 fi -touch "/tmp/createFileList.lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 +touch "$lock" || exit 1 +TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac FILESEXT="${DBEXT//db/files}" @@ -77,5 +78,7 @@ done cd - >/dev/null rm -rf "$TMPDIR" || exit 1 rm -rf "$CACHEDIR" || exit 1 -rm -f "/tmp/createFileList.lock" || exit 1 +rm -f "$lock" || exit 1 # echo 'done' + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.3-24-g4f1b From 6587ebadf6b3a0d8e58be1ab7169a4c2f88ed57c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:58:22 -0600 Subject: create-filelists: s/REPO_DB_FILE/FILES_DB_FILE/g This will set up changes soon to come where we actually use the real repos DB file so I don't want variable name confusion. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index e90da00..a0b6a57 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -30,7 +30,7 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do - REPO_DB_FILE="${repo}$FILESEXT" + FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do cd "$reposdir" @@ -38,9 +38,9 @@ for repo in $repos; do cached="no" # extract old file archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}" + bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi @@ -64,13 +64,13 @@ for repo in $repos; do # create new file archive if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive -# echo "creating ${REPO_DB_FILE}/${arch}" +# echo "creating ${FILES_DB_FILE}/${arch}" pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" cd "${TMPDIR}/${repodir}" - [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old" - [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" * + [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old" + [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * fi done done -- cgit v1.2.3-24-g4f1b From ff1530def072daf95f077ec0f8a4d984da4304d6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:27:13 -0600 Subject: create-filelists: rework the package loop completely Instead of wasting time extracting .PKGINFO twice from every single package in the repos, use the package DB to eliminate most of the heavy lifting. This way we only need to worry about looking at the packages that actually have changed since the last time we built the package database. This should give a noticeable performance increase to this job in addition to reducing IO load and unnecessary reading of every package file. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a0b6a57..6091bf4 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -14,8 +14,12 @@ if [ -f "$lock" ]; then fi touch "$lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +# location where the package DB is extracted so we know what to include +DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1 +# location where the old files DB is extracted to save us some work +CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1 +# location where the new files DB is built up and eventually zipped +TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -30,33 +34,45 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do + REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do +# echo "Running for architecture $arch, repo $repo" cd "$reposdir" repodir="${repo}/os/${arch}" cached="no" + # extract package db archive + if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + mkdir -p "${DBDIR}/${repodir}" +# echo "extracting $REPO_DB_FILE" + bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" + else + echo "Fail! Does the repo $repo with arch $arch even exist?" + continue + fi + # extract old file archive if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" +# echo "extracting $FILES_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi # create file lists - for pkg in $repodir/*${PKGEXT}; do - pkgname="$(getpkgname "$pkg")" - pkgver="$(getpkgver "$pkg")" - tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}" + for pkg in $(ls ${DBDIR}/${repodir}); do + tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then -# echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files" + if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then +# echo "cache: $pkg" + mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" else -# echo "$repo/$arch: $pkgname" +# echo "not cache: $repo/$arch: $pkg" + filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files" + bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" fi done @@ -76,8 +92,7 @@ for repo in $repos; do done cd - >/dev/null -rm -rf "$TMPDIR" || exit 1 -rm -rf "$CACHEDIR" || exit 1 +rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR" rm -f "$lock" || exit 1 # echo 'done' -- cgit v1.2.3-24-g4f1b From ffa88b335b2f756925b388ecbd6681bf3aa58579 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:52:56 -0600 Subject: create-filelists: include desc/depends entries Make the files DB include everything the original packages DB includes instead of just being 'files' entries. This will allow tools to do more with these generated files and they can be used as a drop-in replacement for a regular package database. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 6091bf4..84867d8 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -63,14 +63,18 @@ for repo in $repos; do # create file lists for pkg in $(ls ${DBDIR}/${repodir}); do + dbpkgdir="${DBDIR}/${repodir}/${pkg}" + cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then + ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" + ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" + if [ -f "${cachepkgdir}/files" ]; then # echo "cache: $pkg" - mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" + ln "${cachepkgdir}/files" "${tmppkgdir}/files" else # echo "not cache: $repo/$arch: $pkg" - filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" -- cgit v1.2.3-24-g4f1b