summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2011-03-26 14:13:57 +0100
committerPierre Schmitz <pierre@archlinux.de>2011-03-26 14:13:57 +0100
commitfc6a6ab07bde03c7f20d5a4ed971f8e699ee9b20 (patch)
tree96acb057f856dd5ecbbb719ee8e6d2276d15683b
parent575ba47d34870f8d7e83bc05f274f40a2d52d8fe (diff)
downloaddbscripts-20110326.tar.gz
dbscripts-20110326.tar.xz
Use repo-add to create the files database for each repo20110326
-rwxr-xr-xcron-jobs/create-filelists101
-rw-r--r--db-functions24
-rw-r--r--test/lib/common.inc38
-rwxr-xr-xtest/test.d/create-filelists.sh25
4 files changed, 54 insertions, 134 deletions
diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
deleted file mode 100755
index 8bcfc0b..0000000
--- a/cron-jobs/create-filelists
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-script_lock
-
-for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- repo_lock ${repo} ${arch} || exit 1
- done
-done
-
-#adjust the nice level to run at a lower priority
-renice +10 -p $$ > /dev/null
-
-for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
- filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
-
- if [ ! -f "${repodb}" ]; then
- continue
- fi
- # get a list of package files defined in the repo db
- mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
- bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
- # This should actualy be faster than reading all the just extracted files
- bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
-
- # get a list of package files defined in the files db
- mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
- if [ ! -f "${filedb}" ]; then
- touch "${WORKDIR}/files-${repo}-${arch}"
- else
- bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
- bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}"
- fi
- done
-done
-
-case "${FILESEXT}" in
- *.gz) TAR_OPT="z" ;;
- *.bz2) TAR_OPT="j" ;;
- *.xz) TAR_OPT="J" ;;
- *) die "Unknown compression type for FILESEXT=${FILESEXT}" ;;
-esac
-
-for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
-
- if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then
- # remove any files db that might be in this empty repo
- if [ -f "${filedb}" ]; then
- rm -f "${filedb}"
- fi
- continue
- fi
-
- # Check if updating the files db is needed
- if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then
- mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}"
-
- # Include all unchanged file lists
- # Note: deleted packages are implicitly excluded
- for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
- mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \
- "${WORKDIR}/files-new-dir-${repo}-${arch}"
- done
-
- # Create file lists for new packages
- for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
- tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
- mkdir "${tdir}"
- echo '%FILES%' > "${tdir}/files"
- bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files"
-
- # add desc and depends file from db
- dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
- mv "${dbdir}/desc" "${tdir}/desc"
- mv "${dbdir}/depends" "${tdir}/depends"
- done
-
- # Create the actual file db
- pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null
- bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" *
- popd >/dev/null
- mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}"
- ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}"
- fi
- done
-done
-
-for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- repo_unlock ${repo} ${arch}
- done
-done
-
-script_unlock
diff --git a/db-functions b/db-functions
index 676ecf4..b469039 100644
--- a/db-functions
+++ b/db-functions
@@ -148,7 +148,8 @@ trap cleanup EXIT
#repo_lock <repo-name> <arch> [timeout]
repo_lock () {
local LOCKDIR="$TMPDIR/.repolock.$1.$2"
- local LOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck"
+ local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck"
+ local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck"
local _count
local _trial
local _timeout
@@ -156,8 +157,12 @@ repo_lock () {
local _owner
# This is the lock file used by repo-add and repo-remove
- if [ -f "${LOCKFILE}" ]; then
- error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $LOCKFILE)"
+ if [ -f "${DBLOCKFILE}" ]; then
+ error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)"
+ return 1
+ fi
+ if [ -f "${FILESLOCKFILE}" ]; then
+ error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})"
return 1
fi
@@ -445,6 +450,7 @@ check_repo_permission() {
local dir="${FTP_BASE}/${repo}/os/${arch}/"
[ -w "${dir}" ] || return 1
[ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1
+ [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1
done
return 0
@@ -454,11 +460,14 @@ set_repo_permission() {
local repo=$1
local arch=$2
local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+ local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
if [ -w "${dbfile}" ]; then
local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")")
chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group"
+ chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group"
chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}"
+ chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}"
else
error "You don't have permission to change ${dbfile}"
fi
@@ -471,8 +480,10 @@ arch_repo_add() {
# package files might be relative to repo dir
pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null
- /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \
+ /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \
|| error "repo-add ${repo}${DBEXT} ${pkgs[@]}"
+ /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \
+ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}"
popd >/dev/null
set_repo_permission "${repo}" "${arch}"
}
@@ -482,12 +493,15 @@ arch_repo_remove() {
local arch=$2
local pkgs=(${@:3})
local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+ local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
if [ ! -f "${dbfile}" ]; then
error "No database found at '${dbfile}'"
return 1
fi
- /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \
+ /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \
|| error "repo-remove ${dbfile} ${pkgs[@]}"
+ /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \
+ || error "repo-remove ${filesfile} ${pkgs[@]}"
set_repo_permission "${repo}" "${arch}"
}
diff --git a/test/lib/common.inc b/test/lib/common.inc
index 8a53768..2cf2769 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -121,6 +121,7 @@ checkAnyPackage() {
local repo=$1
local pkg=$2
local arch
+ local db
[ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found"
@@ -131,9 +132,11 @@ checkAnyPackage() {
done
[ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir"
- ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
- && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \
- || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+ for db in ${DBEXT} ${FILESEXT}; do
+ ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \
+ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+ done
[ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist"
}
@@ -142,6 +145,7 @@ checkPackage() {
local repo=$1
local pkg=$2
local arch=$3
+ local db
[ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found"
[ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink"
@@ -150,9 +154,11 @@ checkPackage() {
[ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \
|| fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}"
- ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
- && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \
- || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+ for db in ${DBEXT} ${FILESEXT}; do
+ ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \
+ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+ done
local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}")
svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
@@ -164,10 +170,13 @@ checkRemovedPackage() {
local repo=$1
local pkgbase=$2
local arch=$3
+ local db
- ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
- && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \
- && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+ for db in ${DBEXT} ${FILESEXT}; do
+ ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \
+ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+ done
svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
[ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \
@@ -178,11 +187,14 @@ checkRemovedAnyPackage() {
local repo=$1
local pkgbase=$2
local arch
+ local db
- for arch in i686 x86_64; do
- ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
- && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \
- && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+ for db in ${DBEXT} ${FILESEXT}; do
+ for arch in i686 x86_64; do
+ ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \
+ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+ done
done
svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh
index 08b6790..80b0927 100755
--- a/test/test.d/create-filelists.sh
+++ b/test/test.d/create-filelists.sh
@@ -16,11 +16,10 @@ testCreateSimpleFileLists() {
done
../db-update
- ../cron-jobs/create-filelists
for pkgbase in ${pkgs[@]}; do
for arch in ${arches[@]}; do
- if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then
- fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz"
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then
+ fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}"
fi
done
done
@@ -37,11 +36,10 @@ testCreateAnyFileLists() {
done
../db-update
- ../cron-jobs/create-filelists
for pkgbase in ${pkgs[@]}; do
for arch in ${arches[@]}; do
- if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then
- fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz"
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then
+ fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}"
fi
done
done
@@ -63,13 +61,12 @@ testCreateSplitFileLists() {
done
../db-update
- ../cron-jobs/create-filelists
for pkgbase in ${pkgs[@]}; do
pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]}))
for pkgname in ${pkgnames[@]}; do
for arch in ${arches[@]}; do
- if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then
- fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz"
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then
+ fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}"
fi
done
done
@@ -89,19 +86,17 @@ testCleanupFileLists() {
done
done
../db-update
- ../cron-jobs/create-filelists
for arch in ${arches[@]}; do
../db-remove pkg-simple-a extra ${arch}
done
- ../cron-jobs/create-filelists
for arch in ${arches[@]}; do
- if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then
- fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz"
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then
+ fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}"
fi
- if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then
- fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz"
+ if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then
+ fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}"
fi
done