From 4d16d294f338d5e90a92acb2096bca77e9e924e7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 12:04:30 +0100 Subject: Rewrite sourceballs to increase performance and reliability * Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup --- misc-scripts/make-sourceball | 97 ---------------------------------------- misc-scripts/sourceballs-cleanup | 82 --------------------------------- 2 files changed, 179 deletions(-) delete mode 100755 misc-scripts/make-sourceball delete mode 100755 misc-scripts/sourceballs-cleanup (limited to 'misc-scripts') diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball deleted file mode 100755 index 9a43376..0000000 --- a/misc-scripts/make-sourceball +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -if [ $# -ne 3 -a $# -ne 4 ]; then - msg "usage: $(basename $0) [-f] " - msg " -f Force building. Skip license checks" - exit 1 -fi - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -FORCE=0 -if [ "$1" = "-f" ]; then - FORCE=1 - shift -fi - -packagename="$1" -reponame="$2" -arch="$3" - -script_lock - -create_srcpackage() { - if [ -f "$1/PKGBUILD" ]; then - pushd "$1" >/dev/null - pkgver=$(. PKGBUILD; echo ${pkgver}) - if [ $? -ne 0 ]; then - error "Failed to read pkgver from $1" - return 1 - fi - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - if [ $? -ne 0 ]; then - error "Failed to read pkgrel from $1" - return 1 - fi - license=($(. PKGBUILD; echo ${license[@]})) - if [ $? -ne 0 ]; then - error "Failed to read license from $1" - return 1 - fi - if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then - # Removed so as not to clutter failed.txt - #warning "$packagename license (${license[@]}) does not require source tarballs" - return 0 - else - msg "Creating source tarball for $packagename-$pkgver-$pkgrel" - fi - - local line - /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1 - if [ $? -ne 0 ]; then - error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then - mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" - else - error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - popd >/dev/null - - return 0 - fi -} - -set_umask -cd "$WORKDIR" - -failed=0 - -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-any" || failed=1 - elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 - else - error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'" - failed=1 - fi -else - error "Package '$packagename' does not exist in repo '$reponame-$arch'" - failed=1 -fi - -script_unlock - -exit $failed \ No newline at end of file diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup deleted file mode 100755 index 595f501..0000000 --- a/misc-scripts/sourceballs-cleanup +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -script_lock - -${CLEANUP_DRYRUN} && warning 'dry run mode is active' - -remove_old() { - if [ -d "$1" ]; then - pushd "$1" >/dev/null - PKGVERS="" - for repo in *; do - cd "$repo" - if [ -f PKGBUILD ]; then - pkgver=$(. PKGBUILD; echo ${pkgver}) - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - PKGVERS="$PKGVERS $pkgver-$pkgrel" - else - error "PKGBUILD not found in $1/$repo" - fi - cd .. - done - - for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do - [ -f "$srcpkg" ] || continue - if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then - skip=0 - pver="$(pkgver_from_src $srcpkg)" - for v in $PKGVERS; do - if [ "$v" = "$pver" ]; then - skip=1 - break - fi - done - if [ $skip -ne 1 ]; then - ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR - fi - fi - done - - popd >/dev/null - fi -} - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -cd "$WORKDIR" - -for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do - [ -f "$sourceball" ] || continue - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then - msg "$packagename : package found in sourceballs.skip. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then - if [ $? -ne 1 ]; then - error "$packagename : svn died during export. Skipping sourceball." - else - msg "$packagename : no longer in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - elif [ -z "$(ls -A "$packagename/repos")" ]; then - warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then - if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then - msg "$packagename : source hosting no longer required by license. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - else - remove_old "$packagename/repos/" - fi -done - -script_unlock -- cgit v1.2.3-24-g4f1b