From fc6a6ab07bde03c7f20d5a4ed971f8e699ee9b20 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 26 Mar 2011 14:13:57 +0100 Subject: Use repo-add to create the files database for each repo --- cron-jobs/create-filelists | 101 ---------------------------------------- db-functions | 24 ++++++++-- test/lib/common.inc | 38 +++++++++------ test/test.d/create-filelists.sh | 25 ++++------ 4 files changed, 54 insertions(+), 134 deletions(-) delete mode 100755 cron-jobs/create-filelists diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists deleted file mode 100755 index 8bcfc0b..0000000 --- a/cron-jobs/create-filelists +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -script_lock - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 - done -done - -#adjust the nice level to run at a lower priority -renice +10 -p $$ > /dev/null - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" - - if [ ! -f "${repodb}" ]; then - continue - fi - # get a list of package files defined in the repo db - mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" - bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" - # This should actualy be faster than reading all the just extracted files - bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" - - # get a list of package files defined in the files db - mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" - if [ ! -f "${filedb}" ]; then - touch "${WORKDIR}/files-${repo}-${arch}" - else - bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" - bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}" - fi - done -done - -case "${FILESEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) die "Unknown compression type for FILESEXT=${FILESEXT}" ;; -esac - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" - - if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then - # remove any files db that might be in this empty repo - if [ -f "${filedb}" ]; then - rm -f "${filedb}" - fi - continue - fi - - # Check if updating the files db is needed - if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then - mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}" - - # Include all unchanged file lists - # Note: deleted packages are implicitly excluded - for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do - mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \ - "${WORKDIR}/files-new-dir-${repo}-${arch}" - done - - # Create file lists for new packages - for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do - tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" - mkdir "${tdir}" - echo '%FILES%' > "${tdir}/files" - bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files" - - # add desc and depends file from db - dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" - mv "${dbdir}/desc" "${tdir}/desc" - mv "${dbdir}/depends" "${tdir}/depends" - done - - # Create the actual file db - pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null - bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" * - popd >/dev/null - mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}" - ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}" - fi - done -done - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_unlock ${repo} ${arch} - done -done - -script_unlock diff --git a/db-functions b/db-functions index 676ecf4..b469039 100644 --- a/db-functions +++ b/db-functions @@ -148,7 +148,8 @@ trap cleanup EXIT #repo_lock [timeout] repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" - local LOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" + local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" + local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" local _count local _trial local _timeout @@ -156,8 +157,12 @@ repo_lock () { local _owner # This is the lock file used by repo-add and repo-remove - if [ -f "${LOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $LOCKFILE)" + if [ -f "${DBLOCKFILE}" ]; then + error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" + return 1 + fi + if [ -f "${FILESLOCKFILE}" ]; then + error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})" return 1 fi @@ -445,6 +450,7 @@ check_repo_permission() { local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 + [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1 done return 0 @@ -454,11 +460,14 @@ set_repo_permission() { local repo=$1 local arch=$2 local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" + chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" + chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" else error "You don't have permission to change ${dbfile}" fi @@ -471,8 +480,10 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" } @@ -482,12 +493,15 @@ arch_repo_remove() { local arch=$2 local pkgs=(${@:3}) local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ || error "repo-remove ${dbfile} ${pkgs[@]}" + /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ + || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/test/lib/common.inc b/test/lib/common.inc index 8a53768..2cf2769 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -121,6 +121,7 @@ checkAnyPackage() { local repo=$1 local pkg=$2 local arch + local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" @@ -131,9 +132,11 @@ checkAnyPackage() { done [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" } @@ -142,6 +145,7 @@ checkPackage() { local repo=$1 local pkg=$2 local arch=$3 + local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" @@ -150,9 +154,11 @@ checkPackage() { [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" @@ -164,10 +170,13 @@ checkRemovedPackage() { local repo=$1 local pkgbase=$2 local arch=$3 + local db - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ @@ -178,11 +187,14 @@ checkRemovedAnyPackage() { local repo=$1 local pkgbase=$2 local arch + local db - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + for db in ${DBEXT} ${FILESEXT}; do + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done done svn up -q "${TMP}/svn-packages-copy/${pkgbase}" diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 08b6790..80b0927 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -16,11 +16,10 @@ testCreateSimpleFileLists() { done ../db-update - ../cron-jobs/create-filelists for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then - fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz" + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then + fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi done done @@ -37,11 +36,10 @@ testCreateAnyFileLists() { done ../db-update - ../cron-jobs/create-filelists for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then - fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz" + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then + fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi done done @@ -63,13 +61,12 @@ testCreateSplitFileLists() { done ../db-update - ../cron-jobs/create-filelists for pkgbase in ${pkgs[@]}; do pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) for pkgname in ${pkgnames[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then - fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz" + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then + fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi done done @@ -89,19 +86,17 @@ testCleanupFileLists() { done done ../db-update - ../cron-jobs/create-filelists for arch in ${arches[@]}; do ../db-remove pkg-simple-a extra ${arch} done - ../cron-jobs/create-filelists for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then - fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz" + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then + fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then - fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz" + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then + fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" fi done -- cgit v1.2.3-24-g4f1b