summaryrefslogtreecommitdiffstats
path: root/misc-scripts
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
committerPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
commit4d16d294f338d5e90a92acb2096bca77e9e924e7 (patch)
tree7c977635edc813dfb1a766d214d56ecf63a49867 /misc-scripts
parent1dca721c5149f69067f38d4a33dabbebef98009b (diff)
downloaddbscripts-4d16d294f338d5e90a92acb2096bca77e9e924e7.tar.gz
dbscripts-4d16d294f338d5e90a92acb2096bca77e9e924e7.tar.xz
Rewrite sourceballs to increase performance and reliability
* Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup
Diffstat (limited to 'misc-scripts')
-rwxr-xr-xmisc-scripts/make-sourceball97
-rwxr-xr-xmisc-scripts/sourceballs-cleanup82
2 files changed, 0 insertions, 179 deletions
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
deleted file mode 100755
index 9a43376..0000000
--- a/misc-scripts/make-sourceball
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 3 -a $# -ne 4 ]; then
- msg "usage: $(basename $0) [-f] <packagename> <repo> <arch>"
- msg " -f Force building. Skip license checks"
- exit 1
-fi
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-FORCE=0
-if [ "$1" = "-f" ]; then
- FORCE=1
- shift
-fi
-
-packagename="$1"
-reponame="$2"
-arch="$3"
-
-script_lock
-
-create_srcpackage() {
- if [ -f "$1/PKGBUILD" ]; then
- pushd "$1" >/dev/null
- pkgver=$(. PKGBUILD; echo ${pkgver})
- if [ $? -ne 0 ]; then
- error "Failed to read pkgver from $1"
- return 1
- fi
- pkgrel=$(. PKGBUILD; echo ${pkgrel})
- if [ $? -ne 0 ]; then
- error "Failed to read pkgrel from $1"
- return 1
- fi
- license=($(. PKGBUILD; echo ${license[@]}))
- if [ $? -ne 0 ]; then
- error "Failed to read license from $1"
- return 1
- fi
- if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
- # Removed so as not to clutter failed.txt
- #warning "$packagename license (${license[@]}) does not require source tarballs"
- return 0
- else
- msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
- fi
-
- local line
- /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1
- if [ $? -ne 0 ]; then
- error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)"
- while read line; do
- msg2 "${line}"
- done < "$WORKDIR/makepkg.log"
- return 1
- fi
-
- if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then
- mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
- else
- error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
- while read line; do
- msg2 "${line}"
- done < "$WORKDIR/makepkg.log"
- return 1
- fi
-
- popd >/dev/null
-
- return 0
- fi
-}
-
-set_umask
-cd "$WORKDIR"
-
-failed=0
-
-if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
- if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then
- create_srcpackage "$packagename/repos/$reponame-any" || failed=1
- elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then
- create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1
- else
- error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'"
- failed=1
- fi
-else
- error "Package '$packagename' does not exist in repo '$reponame-$arch'"
- failed=1
-fi
-
-script_unlock
-
-exit $failed \ No newline at end of file
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
deleted file mode 100755
index 595f501..0000000
--- a/misc-scripts/sourceballs-cleanup
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-script_lock
-
-${CLEANUP_DRYRUN} && warning 'dry run mode is active'
-
-remove_old() {
- if [ -d "$1" ]; then
- pushd "$1" >/dev/null
- PKGVERS=""
- for repo in *; do
- cd "$repo"
- if [ -f PKGBUILD ]; then
- pkgver=$(. PKGBUILD; echo ${pkgver})
- pkgrel=$(. PKGBUILD; echo ${pkgrel})
- PKGVERS="$PKGVERS $pkgver-$pkgrel"
- else
- error "PKGBUILD not found in $1/$repo"
- fi
- cd ..
- done
-
- for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do
- [ -f "$srcpkg" ] || continue
- if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
- skip=0
- pver="$(pkgver_from_src $srcpkg)"
- for v in $PKGVERS; do
- if [ "$v" = "$pver" ]; then
- skip=1
- break
- fi
- done
- if [ $skip -ne 1 ]; then
- ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR
- fi
- fi
- done
-
- popd >/dev/null
- fi
-}
-
-#adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
-
-set_umask
-cd "$WORKDIR"
-
-for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do
- [ -f "$sourceball" ] || continue
- packagename=$(basename $sourceball)
- packagename=${packagename%-*-*$SRCEXT}
-
- if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then
- msg "$packagename : package found in sourceballs.skip. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then
- if [ $? -ne 1 ]; then
- error "$packagename : svn died during export. Skipping sourceball."
- else
- msg "$packagename : no longer in svn. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- elif [ -z "$(ls -A "$packagename/repos")" ]; then
- warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then
- if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then
- msg "$packagename : source hosting no longer required by license. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- else
- remove_old "$packagename/repos/"
- fi
-done
-
-script_unlock