From e792abefa6eff4cf54616add0b19d731d486045f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 16:03:44 +0100 Subject: Rewrote create-filelists * use correct locking of the repos * added test case * removes file lists of deleted packages * add compression independent symlink to files db --- cron-jobs/create-filelists | 147 ++++++++++++++++++++++------------------ test/test.d/create-filelists.sh | 110 ++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 66 deletions(-) create mode 100755 test/test.d/create-filelists.sh diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 010194b..c3e7d72 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,20 +3,42 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -reposdir=${FTP_BASE} -targetdir=${FTP_BASE} - script_lock -# location where the package DB is extracted so we know what to include -DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1 -# location where the old files DB is extracted to save us some work -CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1 -# location where the new files DB is built up and eventually zipped -MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done #adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null +renice +10 -p $$ > /dev/null + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${repodb}" ]; then + continue + fi + # get a list of package files defined in the repo db + mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" + bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" + find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}" + + # get a list of package files defined in the files db + mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" + if [ ! -f "${filedb}" ]; then + echo > "${WORKDIR}/files-${repo}-${arch}" + else + bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" + find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}" + fi + done +done case "${FILESEXT}" in *.gz) TAR_OPT="z" ;; @@ -26,62 +48,55 @@ case "${FILESEXT}" in esac for repo in ${PKGREPOS[@]}; do - REPO_DB_FILE="${repo}$DBEXT" - FILES_DB_FILE="${repo}$FILESEXT" - for arch in ${ARCHES[@]}; do - pushd "$reposdir" >/dev/null - - repodir="${repo}/os/${arch}" - cached="no" - - # extract package db archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then - mkdir -p "${DBDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" - else - warning "The repo $repo with arch $arch does not exist" - continue - fi - - # extract old file archive - if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then - mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" - cached="yes" - fi - - # create file lists - for pkg in $(ls ${DBDIR}/${repodir}); do - dbpkgdir="${DBDIR}/${repodir}/${pkg}" - cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" - tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}" - mkdir -p "$tmppkgdir" - ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" - ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" - if [ -f "${cachepkgdir}/files" ]; then - ln "${cachepkgdir}/files" "${tmppkgdir}/files" - else - filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) - echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" - cached="no" - fi - done - - # create new file archive - if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then - # at least one package has changed, so let's rebuild the archive - pkgdir="${targetdir}/${repodir}" - mkdir -p "$pkgdir" - pushd "${MYTMPDIR}/${repodir}" >/dev/null - [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" - bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * - popd >/dev/null - fi - popd >/dev/null - done + for arch in ${ARCHES[@]}; do + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then + # remove any files db that might be in this empty repo + if [ -f "${filedb}" ]; then + rm -f "${filedb}" + fi + continue + fi + + # Check if updating the files db is needed + if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then + mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}" + + # Include all unchanged file lists + # Note: deleted packages are implicitly excluded + for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \ + "${WORKDIR}/files-new-dir-${repo}-${arch}" + done + + # Create file lists for new packages + for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mkdir "${tdir}" + echo '%FILES%' > "${tdir}/files" + bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files" + + # add desc and depends file from db + dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mv "${dbdir}/desc" "${tdir}/desc" + mv "${dbdir}/depends" "${tdir}/depends" + done + + # Create the actual file db + pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null + bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" * + popd >/dev/null + mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}" + ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}" + fi + done +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done done script_unlock - -# vim: set ts=4 sw=4 et ft=sh: diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh new file mode 100755 index 0000000..da76710 --- /dev/null +++ b/test/test.d/create-filelists.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testCreateSimpleFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then + fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz" + fi + done + done +} + +testCreateAnyFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then + fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz" + fi + done + done +} + +testCreateSplitFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local pkgname + local pkgnames + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) + for pkgname in ${pkgnames[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then + fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz" + fi + done + done + done +} + + +testCleanupFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + ../cron-jobs/create-filelists + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + ../cron-jobs/create-filelists + + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then + fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz" + fi + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then + fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz" + fi + done + +} + +. "${curdir}/../lib/shunit2" -- cgit v1.2.3-24-g4f1b