pactest: add test.expectfailure option for tests

This will enable us to mark tests we know currently fail to differentiate
them from those that we know should pass. Regressions should be easier to
spot this way.

Signed-off-by: Dan McGee <dan@archlinux.org>

9 files changed, 41 insertions, 5 deletions

diff --git a/pactest/pmenv.py b/pactest/pmenv.py
index f2327f91..b86abe94 100755
--- a/pactest/pmenv.py
+++ b/pactest/pmenv.py
@@ -86,13 +86,25 @@ class pmenv:
         """
         passed = 0
         tpassed = []
+        failed = 0
         tfailed = []
+        expectedfail = 0
+        texpectedfail = []
+        unexpectedpass = 0
+        tunexpectedpass = []
         for test in self.testcases:
             fail = test.result["fail"]
-            if fail == 0:
+            if fail == 0 and not test.expectfailure:
                 passed += 1
                 tpassed.append(test)
+            elif fail != 0 and test.expectfailure:
+                expectedfail += 1
+                texpectedfail.append(test)
+            elif fail == 0: # and not test.expectfail
+                unexpectedpass += 1
+                tunexpectedpass.append(test)
             else:
+                failed += 1
                 tfailed.append(test)
 
         def _printtest(t):
@@ -114,17 +126,26 @@ class pmenv:
         print "=========="*8
         print "Results"
         print "----------"*8
+        print " Passed:"
         for test in tpassed: _printtest(test)
         print "----------"*8
+        print " Expected Failures:"
+        for test in texpectedfail: _printtest(test)
+        print "----------"*8
+        print " Unexpected Passes:"
+        for test in tunexpectedpass: _printtest(test)
+        print "----------"*8
+        print " Failed:"
         for test in tfailed: _printtest(test)
         print "----------"*8
 
         total = len(self.testcases)
-        failed = total - passed
-        print "TOTAL = %3u" % total
+        print "Total            = %3u" % total
         if total:
-            print "PASS  = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
-            print "FAIL  = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
+            print "Pass             = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total)
+            print "Expected Fail    = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total)
+            print "Unexpected Pass  = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total)
+            print "Fail             = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total)
         print ""
 
 if __name__ == "__main__":
diff --git a/pactest/pmtest.py b/pactest/pmtest.py
index e8f6fa8d..f31563b1 100755
--- a/pactest/pmtest.py
+++ b/pactest/pmtest.py
@@ -90,6 +90,7 @@ class pmtest:
         # Test rules
         self.rules = []
         self.files = []
+        self.expectfailure = False
         
         if os.path.isfile(self.name):
             execfile(self.name)
diff --git a/pactest/tests/fileconflict001.py b/pactest/tests/fileconflict001.py
index 4c2069ee..8c13911c 100644
--- a/pactest/tests/fileconflict001.py
+++ b/pactest/tests/fileconflict001.py
@@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict002.py b/pactest/tests/fileconflict002.py
index c54f6daf..f70873ba 100644
--- a/pactest/tests/fileconflict002.py
+++ b/pactest/tests/fileconflict002.py
@@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2])
 self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
+
+self.expectfailure = True
diff --git a/pactest/tests/fileconflict004.py b/pactest/tests/fileconflict004.py
index a5347ccd..2396cedb 100644
--- a/pactest/tests/fileconflict004.py
+++ b/pactest/tests/fileconflict004.py
@@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("PKG_VERSION=pkg1|2.0-1")
 self.addrule("FILE_TYPE=test|link")
+
+self.expectfailure = True
diff --git a/pactest/tests/sync403.py b/pactest/tests/sync403.py
index b8d01016..d8ab763c 100644
--- a/pactest/tests/sync403.py
+++ b/pactest/tests/sync403.py
@@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0")
 self.addrule("PKG_EXIST=pkg1")
 self.addrule("!PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/trans001.py b/pactest/tests/trans001.py
index b9889b14..b3d7883b 100644
--- a/pactest/tests/trans001.py
+++ b/pactest/tests/trans001.py
@@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
 self.addrule("PKG_EXIST=pkg3")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade046.py b/pactest/tests/upgrade046.py
index 60164b72..12390647 100644
--- a/pactest/tests/upgrade046.py
+++ b/pactest/tests/upgrade046.py
@@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy")
 self.addrule("FILE_MODIFIED=bin/foobar")
 self.addrule("FILE_EXIST=usr/share/file")
 self.addrule("FILE_MODIFIED=usr/share/file")
+
+self.expectfailure = True
diff --git a/pactest/tests/upgrade051.py b/pactest/tests/upgrade051.py
index e8c69eb9..4346c24d 100644
--- a/pactest/tests/upgrade051.py
+++ b/pactest/tests/upgrade051.py
@@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename()
 self.addrule("PACMAN_RETCODE=0")
 self.addrule("!PKG_EXIST=pkg1")
 self.addrule("PKG_EXIST=pkg2")
+
+self.expectfailure = True