Use repo-add to create the files database for each repo20110326

4 files changed, 54 insertions, 134 deletions

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
deleted file mode 100755
index 8bcfc0b..0000000
--- a/cron-jobs/create-filelists
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-script_lock
-
-for repo in ${PKGREPOS[@]}; do
-	for arch in ${ARCHES[@]}; do
-		repo_lock ${repo} ${arch} || exit 1
-	done
-done
-
-#adjust the nice level to run at a lower priority
-renice +10 -p $$ > /dev/null
-
-for repo in ${PKGREPOS[@]}; do
-	for arch in ${ARCHES[@]}; do
-		repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
-		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
-
-		if [ ! -f "${repodb}" ]; then
-			continue
-		fi
-		# get a list of package files defined in the repo db
-		mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
-		bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
-		# This should actualy be faster than reading all the just extracted files
-		bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
-
-		# get a list of package files defined in the files db
-		mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
-		if [ ! -f "${filedb}" ]; then
-			touch "${WORKDIR}/files-${repo}-${arch}"
-		else
-			bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
-			bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}"
-		fi
-	done
-done
-
-case "${FILESEXT}" in
-	*.gz)  TAR_OPT="z" ;;
-	*.bz2) TAR_OPT="j" ;;
-	*.xz)  TAR_OPT="J" ;;
-	*) die "Unknown compression type for FILESEXT=${FILESEXT}" ;;
-esac
-
-for repo in ${PKGREPOS[@]}; do
-	for arch in ${ARCHES[@]}; do
-		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
-
-		if [ ! -f  "${WORKDIR}/db-${repo}-${arch}" ]; then
-			# remove any files db that might be in this empty repo
-			if [ -f "${filedb}" ]; then
-				rm -f "${filedb}"
-			fi
-			continue
-		fi
-
-		# Check if updating the files db is needed
-		if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then
-			mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}"
-
-			# Include all unchanged file lists
-			# Note: deleted packages are implicitly excluded
-			for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
-				mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \
-					"${WORKDIR}/files-new-dir-${repo}-${arch}"
-			done
-
-			# Create file lists for new packages
-			for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
-				tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
-				mkdir "${tdir}"
-				echo '%FILES%' > "${tdir}/files"
-				bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files"
-
-				# add desc and depends file from db
-				dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
-				mv "${dbdir}/desc" "${tdir}/desc"
-				mv "${dbdir}/depends" "${tdir}/depends"
-			done
-
-			# Create the actual file db
-			pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null
-			bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" *
-			popd >/dev/null
-			mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}"
-			ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}"
-		fi
-	done
-done
-
-for repo in ${PKGREPOS[@]}; do
-	for arch in ${ARCHES[@]}; do
-		repo_unlock ${repo} ${arch}
-	done
-done
-
-script_unlock
diff --git a/db-functions b/db-functions
index 676ecf4..b469039 100644
--- a/db-functions
+++ b/db-functions
@@ -148,7 +148,8 @@ trap cleanup EXIT
 #repo_lock <repo-name> <arch> [timeout]
 repo_lock () {
 	local LOCKDIR="$TMPDIR/.repolock.$1.$2"
-	local LOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck"
+	local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck"
+	local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck"
 	local _count
 	local _trial
 	local _timeout
@@ -156,8 +157,12 @@ repo_lock () {
 	local _owner
 
 	# This is the lock file used by repo-add and repo-remove
-	if [ -f "${LOCKFILE}" ]; then
-		error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $LOCKFILE)"
+	if [ -f "${DBLOCKFILE}" ]; then
+		error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)"
+		return 1
+	fi
+	if [ -f "${FILESLOCKFILE}" ]; then
+		error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})"
 		return 1
 	fi
 
@@ -445,6 +450,7 @@ check_repo_permission() {
 		local dir="${FTP_BASE}/${repo}/os/${arch}/"
 		[ -w "${dir}" ] || return 1
 		[ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1
+		[ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1
 	done
 
 	return 0
@@ -454,11 +460,14 @@ set_repo_permission() {
 	local repo=$1
 	local arch=$2
 	local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+	local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
 
 	if [ -w "${dbfile}" ]; then
 		local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")")
 		chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group"
+		chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group"
 		chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}"
+		chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}"
 	else
 		error "You don't have permission to change ${dbfile}"
 	fi
@@ -471,8 +480,10 @@ arch_repo_add() {
 
 	# package files might be relative to repo dir
 	pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null
-	/usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \
+	/usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \
 		|| error "repo-add ${repo}${DBEXT} ${pkgs[@]}"
+	/usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \
+		|| error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}"
 	popd >/dev/null
 	set_repo_permission "${repo}" "${arch}"
 }
@@ -482,12 +493,15 @@ arch_repo_remove() {
 	local arch=$2
 	local pkgs=(${@:3})
 	local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+	local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
 
 	if [ ! -f "${dbfile}" ]; then
 		error "No database found at '${dbfile}'"
 		return 1
 	fi
-	/usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \
+	/usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \
 		|| error "repo-remove ${dbfile} ${pkgs[@]}"
+	/usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \
+		|| error "repo-remove ${filesfile} ${pkgs[@]}"
 	set_repo_permission "${repo}" "${arch}"
 }
diff --git a/test/lib/common.inc b/test/lib/common.inc
index 8a53768..2cf2769 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -121,6 +121,7 @@ checkAnyPackage() {
 	local repo=$1
 	local pkg=$2
 	local arch
+	local db
 
 	[ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found"
 
@@ -131,9 +132,11 @@ checkAnyPackage() {
 	done
 	[ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir"
 
-	( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
-		&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \
-		|| fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+	for db in ${DBEXT} ${FILESEXT}; do
+		( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+			&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \
+			|| fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+	done
 
 	[ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist"
 }
@@ -142,6 +145,7 @@ checkPackage() {
 	local repo=$1
 	local pkg=$2
 	local arch=$3
+	local db
 
 	[ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found"
 	[ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink"
@@ -150,9 +154,11 @@ checkPackage() {
 	[ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \
 		|| fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}"
 
-	( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
-		&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \
-		|| fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+	for db in ${DBEXT} ${FILESEXT}; do
+		( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+			&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \
+			|| fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+	done
 
 	local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}")
 	svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
@@ -164,10 +170,13 @@ checkRemovedPackage() {
 	local repo=$1
 	local pkgbase=$2
 	local arch=$3
+	local db
 
-	( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
-		&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \
-		&& fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+	for db in ${DBEXT} ${FILESEXT}; do
+		( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+			&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \
+			&& fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+	done
 
 	svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
 	[ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \
@@ -178,11 +187,14 @@ checkRemovedAnyPackage() {
 	local repo=$1
 	local pkgbase=$2
 	local arch
+	local db
 
-	for arch in i686 x86_64; do
-		( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \
-			&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \
-			&& fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}"
+	for db in ${DBEXT} ${FILESEXT}; do
+		for arch in i686 x86_64; do
+			( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \
+				&& bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \
+				&& fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}"
+		done
 	done
 
 	svn up -q "${TMP}/svn-packages-copy/${pkgbase}"
diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh
index 08b6790..80b0927 100755
--- a/test/test.d/create-filelists.sh
+++ b/test/test.d/create-filelists.sh
@@ -16,11 +16,10 @@ testCreateSimpleFileLists() {
 	done
 	../db-update
 
-	../cron-jobs/create-filelists
 	for pkgbase in ${pkgs[@]}; do
 		for arch in ${arches[@]}; do
-			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then
-				fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz"
+			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then
+				fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}"
 			fi
 		done
 	done
@@ -37,11 +36,10 @@ testCreateAnyFileLists() {
 	done
 	../db-update
 
-	../cron-jobs/create-filelists
 	for pkgbase in ${pkgs[@]}; do
 		for arch in ${arches[@]}; do
-			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then
-				fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz"
+			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then
+				fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}"
 			fi
 		done
 	done
@@ -63,13 +61,12 @@ testCreateSplitFileLists() {
 	done
 	../db-update
 
-	../cron-jobs/create-filelists
 	for pkgbase in ${pkgs[@]}; do
 		pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]}))
 		for pkgname in ${pkgnames[@]}; do
 			for arch in ${arches[@]}; do
-				if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then
-					fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz"
+				if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then
+					fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}"
 				fi
 			done
 		done
@@ -89,19 +86,17 @@ testCleanupFileLists() {
 		done
 	done
 	../db-update
-	../cron-jobs/create-filelists
 
 	for arch in ${arches[@]}; do
 		../db-remove pkg-simple-a extra ${arch}
 	done
-	../cron-jobs/create-filelists
 
 	for arch in ${arches[@]}; do
-		if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then
-			fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz"
+		if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then
+			fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}"
 		fi
-		if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then
-			fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz"
+		if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then
+			fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}"
 		fi
 	done