t0003: Make tests more strict

Instead of testing the exit code only, also check the error output.

This reveals two bugs. The corresponding tests are marked as known
breakages.

Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org>

1 files changed, 68 insertions, 23 deletions

diff --git a/git-interface/test/t0003-update.sh b/git-interface/test/t0003-update.sh
index aeb223e8..ec830ba5 100755
--- a/git-interface/test/t0003-update.sh
+++ b/git-interface/test/t0003-update.sh
@@ -104,22 +104,34 @@ test_expect_success 'Test restore mode.' '
 '
 
 test_expect_success 'Test restore mode on a non-existent repository.' '
+	cat >expected <<-EOD &&
+	error: restore: repository not found: foobar3
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar3 AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" restore 2>&1
+	test_must_fail "$GIT_UPDATE" restore >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_expect_success 'Pushing to a branch other than master.' '
 	old=0000000000000000000000000000000000000000 &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	error: pushing to a branch other than master is restricted
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/pu "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/pu "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_expect_success 'Performing a non-fast-forward ref update.' '
 	old=$(git -C aur.git rev-parse HEAD) &&
 	new=$(git -C aur.git rev-parse HEAD^) &&
+	cat >expected <<-EOD &&
+	error: denying non-fast-forward (you should pull first)
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_expect_success 'Performing a non-fast-forward ref update as Trusted User.' '
@@ -136,7 +148,8 @@ test_expect_success 'Removing .SRCINFO.' '
 	git -C aur.git commit -q -m "Remove .SRCINFO" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing .SRCINFO$" actual
 '
 
 test_expect_success 'Removing .SRCINFO with a follow-up fix.' '
@@ -147,7 +160,8 @@ test_expect_success 'Removing .SRCINFO with a follow-up fix.' '
 	git -C aur.git revert --no-edit HEAD &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing .SRCINFO$" actual
 '
 
 test_expect_success 'Removing PKGBUILD.' '
@@ -157,7 +171,8 @@ test_expect_success 'Removing PKGBUILD.' '
 	git -C aur.git commit -q -m "Remove PKGBUILD" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing PKGBUILD$" actual
 '
 
 test_expect_success 'Pushing a tree with a subdirectory.' '
@@ -169,7 +184,8 @@ test_expect_success 'Pushing a tree with a subdirectory.' '
 	git -C aur.git commit -q -m "Add subdirectory" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: the repository must not contain subdirectories$" actual
 '
 
 test_expect_success 'Pushing a tree with a large blob.' '
@@ -180,7 +196,8 @@ test_expect_success 'Pushing a tree with a large blob.' '
 	git -C aur.git commit -q -m "Add large blob" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: maximum blob size (250.00KiB) exceeded$" actual
 '
 
 test_expect_success 'Pushing .SRCINFO with a non-matching package base.' '
@@ -194,7 +211,8 @@ test_expect_success 'Pushing .SRCINFO with a non-matching package base.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: invalid pkgbase: foobar2, expected foobar$" actual
 '
 
 test_expect_success 'Pushing .SRCINFO with invalid syntax.' '
@@ -222,7 +240,8 @@ test_expect_success 'Pushing .SRCINFO without pkgver.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing mandatory field: pkgver$" actual
 '
 
 test_expect_success 'Pushing .SRCINFO without pkgrel.' '
@@ -236,7 +255,8 @@ test_expect_success 'Pushing .SRCINFO without pkgrel.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing mandatory field: pkgrel$" actual
 '
 
 test_expect_success 'Pushing .SRCINFO with epoch.' '
@@ -270,7 +290,8 @@ test_expect_success 'Pushing .SRCINFO with invalid pkgname.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: invalid package name: !$" actual
 '
 
 test_expect_success 'Pushing .SRCINFO with invalid epoch.' '
@@ -284,7 +305,8 @@ test_expect_success 'Pushing .SRCINFO with invalid epoch.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: invalid epoch: !$" actual
 '
 
 test_expect_success 'Missing install file.' '
@@ -298,7 +320,8 @@ test_expect_success 'Missing install file.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing install file: install$" actual
 '
 
 test_expect_success 'Missing changelog file.' '
@@ -312,7 +335,8 @@ test_expect_success 'Missing changelog file.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing changelog file: changelog$" actual
 '
 
 test_expect_success 'Missing source file.' '
@@ -326,7 +350,8 @@ test_expect_success 'Missing source file.' '
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	grep -q "^error: missing source file: file$" actual
 '
 
 test_expect_success 'Pushing a blacklisted package.' '
@@ -335,18 +360,26 @@ test_expect_success 'Pushing a blacklisted package.' '
 	echo "pkgname = forbidden" >>aur.git/.SRCINFO &&
 	git -C aur.git commit -q -am "Add blacklisted package" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	error: package is blacklisted: forbidden
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
-test_expect_success 'Pushing a blacklisted package as Trusted User.' '
+test_expect_failure 'Pushing a blacklisted package as Trusted User.' '
 	old=$(git -C aur.git rev-parse HEAD) &&
 	test_when_finished "git -C aur.git reset --hard $old" &&
 	echo "pkgname = forbidden" >>aur.git/.SRCINFO &&
 	git -C aur.git commit -q -am "Add blacklisted package" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	warning: package is blacklisted: forbidden
+	EOD
 	AUR_USER=tu AUR_PKGBASE=foobar AUR_PRIVILEGED=1 \
-	"$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1 | grep ^warning:
+	"$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_expect_success 'Pushing a package already in the official repositories.' '
@@ -355,18 +388,26 @@ test_expect_success 'Pushing a package already in the official repositories.' '
 	echo "pkgname = official" >>aur.git/.SRCINFO &&
 	git -C aur.git commit -q -am "Add official package" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	error: package already provided by [core]: official
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
-test_expect_success 'Pushing a package already in the official repositories as Trusted User.' '
+test_expect_failure 'Pushing a package already in the official repositories as Trusted User.' '
 	old=$(git -C aur.git rev-parse HEAD) &&
 	test_when_finished "git -C aur.git reset --hard $old" &&
 	echo "pkgname = official" >>aur.git/.SRCINFO &&
 	git -C aur.git commit -q -am "Add official package" &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	warning: package already provided by [core]: official
+	EOD
 	AUR_USER=tu AUR_PKGBASE=foobar AUR_PRIVILEGED=1 \
-	"$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1 | grep ^warning:
+	"$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_expect_success 'Trying to hijack a package.' '
@@ -380,8 +421,12 @@ test_expect_success 'Trying to hijack a package.' '
 		git commit -q -am "Change package name"
 	) &&
 	new=$(git -C aur.git rev-parse HEAD) &&
+	cat >expected <<-EOD &&
+	error: cannot overwrite package: foobar
+	EOD
 	AUR_USER=user AUR_PKGBASE=foobar2 AUR_PRIVILEGED=0 \
-	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" 2>&1
+	test_must_fail "$GIT_UPDATE" refs/heads/master "$old" "$new" >actual 2>&1 &&
+	test_cmp expected actual
 '
 
 test_done