Continuing in Python's unittest when an assertion fails

Question

EDIT: switched to a better example, and clarified why this is a real problem.

I\'d like to write unit tests in Python that continue executing when an assertion fails

Answer 1

Another way to have non-fatal assertions is to capture the assertion exception and store the exceptions in a list. Then assert that that list is empty as part of the tearDown.

import unittest

class Car(object):
  def __init__(self, make, model):
    self.make = make
    self.model = make  # Copy and paste error: should be model.
    self.has_seats = True
    self.wheel_count = 3  # Typo: should be 4.

class CarTest(unittest.TestCase):
  def setUp(self):
    self.verificationErrors = []

  def tearDown(self):
    self.assertEqual([], self.verificationErrors)

  def test_init(self):
    make = "Ford"
    model = "Model T"
    car = Car(make=make, model=model)
    try: self.assertEqual(car.make, make)
    except AssertionError, e: self.verificationErrors.append(str(e))
    try: self.assertEqual(car.model, model)  # Failure!
    except AssertionError, e: self.verificationErrors.append(str(e))
    try: self.assertTrue(car.has_seats)
    except AssertionError, e: self.verificationErrors.append(str(e))
    try: self.assertEqual(car.wheel_count, 4)  # Failure!
    except AssertionError, e: self.verificationErrors.append(str(e))

if __name__ == "__main__":
    unittest.main()

Answer 2

I had a problem with the answer from @Anthony Batchelor because it would have forced me to use try...catch inside my unit tests. Instead, I encapsulated the try...catch logic in an override of the TestCase.assertEqual method. Here is the code:

import unittest
import traceback

class AssertionErrorData(object):

    def __init__(self, stacktrace, message):
        super(AssertionErrorData, self).__init__()
        self.stacktrace = stacktrace
        self.message = message

class MultipleAssertionFailures(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        self.verificationErrors = []
        super(MultipleAssertionFailures, self).__init__( *args, **kwargs )

    def tearDown(self):
        super(MultipleAssertionFailures, self).tearDown()

        if self.verificationErrors:
            index = 0
            errors = []

            for error in self.verificationErrors:
                index += 1
                errors.append( "%s\nAssertionError %s: %s" % ( 
                        error.stacktrace, index, error.message ) )

            self.fail( '\n\n' + "\n".join( errors ) )
            self.verificationErrors.clear()

    def assertEqual(self, goal, results, msg=None):

        try:
            super( MultipleAssertionFailures, self ).assertEqual( goal, results, msg )

        except unittest.TestCase.failureException as error:
            goodtraces = self._goodStackTraces()
            self.verificationErrors.append( 
                    AssertionErrorData( "\n".join( goodtraces[:-2] ), error ) )

    def _goodStackTraces(self):
        """
            Get only the relevant part of stacktrace.
        """
        stop = False
        found = False
        goodtraces = []

        # stacktrace = traceback.format_exc()
        # stacktrace = traceback.format_stack()
        stacktrace = traceback.extract_stack()

        # https://stackoverflow.com/questions/54499367/how-to-correctly-override-testcase
        for stack in stacktrace:
            filename = stack.filename

            if found and not stop and \
                    not filename.find( 'lib' ) < filename.find( 'unittest' ):
                stop = True

            if not found and filename.find( 'lib' ) < filename.find( 'unittest' ):
                found = True

            if stop and found:
                stackline = '  File "%s", line %s, in %s\n    %s' % ( 
                        stack.filename, stack.lineno, stack.name, stack.line )
                goodtraces.append( stackline )

        return goodtraces

# class DummyTestCase(unittest.TestCase):
class DummyTestCase(MultipleAssertionFailures):

    def setUp(self):
        self.maxDiff = None
        super(DummyTestCase, self).setUp()

    def tearDown(self):
        super(DummyTestCase, self).tearDown()

    def test_function_name(self):
        self.assertEqual( "var", "bar" )
        self.assertEqual( "1937", "511" )

if __name__ == '__main__':
    unittest.main()

Result output:

F
======================================================================
FAIL: test_function_name (__main__.DummyTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "D:\User\Downloads\test.py", line 77, in tearDown
    super(DummyTestCase, self).tearDown()
  File "D:\User\Downloads\test.py", line 29, in tearDown
    self.fail( '\n\n' + "\n\n".join( errors ) )
AssertionError: 

  File "D:\User\Downloads\test.py", line 80, in test_function_name
    self.assertEqual( "var", "bar" )
AssertionError 1: 'var' != 'bar'
- var
? ^
+ bar
? ^
 : 

  File "D:\User\Downloads\test.py", line 81, in test_function_name
    self.assertEqual( "1937", "511" )
AssertionError 2: '1937' != '511'
- 1937
+ 511
 : 

More alternative solutions for the correct stacktrace capture could be posted on How to correctly override TestCase.assertEqual(), producing the right stacktrace?

Answer 3

I realize this question was asked literally years ago, but there are now (at least) two Python packages that allow you to do this.

One is softest: https://pypi.org/project/softest/

The other is Python-Delayed-Assert: https://github.com/pr4bh4sh/python-delayed-assert

I haven't used either, but they look pretty similar to me.

Answer 4

What you'll probably want to do is derive unittest.TestCase since that's the class that throws when an assertion fails. You will have to re-architect your TestCase to not throw (maybe keep a list of failures instead). Re-architecting stuff can cause other issues that you would have to resolve. For example you may end up needing to derive TestSuite to make changes in support of the changes made to your TestCase.

Answer 5

Since Python 3.4 you can also use subtests:

def test_init(self):
    make = "Ford"
    model = "Model T"
    car = Car(make=make, model=model)
    with self.subTest(msg='Car.make check'):
        self.assertEqual(car.make, make)
    with self.subTest(msg='Car.model check'):
        self.assertEqual(car.model, model)
    with self.subTest(msg='Car.has_seats check'):
        self.assertTrue(car.has_seats)
    with self.subTest(msg='Car.wheel_count check'):
        self.assertEqual(car.wheel_count, 4)

(msg parameter is used to more easily determine which test failed.)

Output:

======================================================================
FAIL: test_init (__main__.CarTest) [Car.model check]
----------------------------------------------------------------------
Traceback (most recent call last):
  File "test.py", line 23, in test_init
    self.assertEqual(car.model, model)
AssertionError: 'Ford' != 'Model T'
- Ford
+ Model T


======================================================================
FAIL: test_init (__main__.CarTest) [Car.wheel_count check]
----------------------------------------------------------------------
Traceback (most recent call last):
  File "test.py", line 27, in test_init
    self.assertEqual(car.wheel_count, 4)
AssertionError: 3 != 4

----------------------------------------------------------------------
Ran 1 test in 0.001s

FAILED (failures=2)

Answer 6

expect is very useful in gtest. This is python way in gist, and code:

import sys
import unittest


class TestCase(unittest.TestCase):
    def run(self, result=None):
        if result is None:
            self.result = self.defaultTestResult()
        else:
            self.result = result

        return unittest.TestCase.run(self, result)

    def expect(self, val, msg=None):
        '''
        Like TestCase.assert_, but doesn't halt the test.
        '''
        try:
            self.assert_(val, msg)
        except:
            self.result.addFailure(self, sys.exc_info())

    def expectEqual(self, first, second, msg=None):
        try:
            self.failUnlessEqual(first, second, msg)
        except:
            self.result.addFailure(self, sys.exc_info())

    expect_equal = expectEqual

    assert_equal = unittest.TestCase.assertEqual
    assert_raises = unittest.TestCase.assertRaises


test_main = unittest.main