test discovery: yield results progressively

test/test_runner_test.py

@@ -116,14 +116,14 @@ def test_get_tests_for_suite_in_suite(self):
 
         instance = test_runner.TestRunner(mock.sentinel.test_class)
         ret = instance.get_tests_for_suite(mock.sentinel.selected_suite_name)
-        assert_equal(ret, [self.mock_test_method])
+        assert_equal(list(ret), [self.mock_test_method])
 
     def test_get_tests_for_suite_not_in_suite(self):
         self.in_suite_mock.return_value = False
 
         instance = test_runner.TestRunner(mock.sentinel.test_class)
         ret = instance.get_tests_for_suite(mock.sentinel.selected_suite_name)
-        assert_equal(ret, [])
+        assert_equal(list(ret), [])
 
 
 class TestTestRunnerPrintsTestNames(test_case.TestCase):

testify/test_discovery.py

@@ -83,20 +83,20 @@ def discover(what):
     try:
         what = to_module(what)
         mod = __import__(what, fromlist=[str('__trash')])
-        discovered = set()
-
-        if hasattr(mod, '__path__'):
-            # It's a package!
-            for _, module_name, _ in pkgutil.walk_packages(
-                mod.__path__,
-                prefix=mod.__name__ + '.',
-            ):
-                mod = __import__(module_name, fromlist=[str('__trash')])
-                discovered.update(get_test_classes_from_module(mod))
-        else:
-            discovered.update(get_test_classes_from_module(mod))
-
-        return discovered
+        for cls in get_test_classes_from_module(mod):
+            yield cls
+
+        if not hasattr(mod, '__path__'):
+            return
+
+        # It's a package!
+        for _, module_name, _ in pkgutil.walk_packages(
+            mod.__path__,
+            prefix=mod.__name__ + '.',
+        ):
+            submod = __import__(module_name, fromlist=[str('__trash')])
+            for cls in get_test_classes_from_module(submod):
+                yield cls
     except Exception:
         traceback.print_exc()
         raise DiscoveryError(

testify/test_reporter.py

@@ -27,10 +27,6 @@ def __init__(self, options):
         """
         self.options = options
 
-    def test_counts(self, test_case_count, test_method_count):
-        """Called after discovery finishes. May not be called by all test runners, e.g. TestRunnerClient."""
-        pass
-
     def test_start(self, result):
         """Called when a test method is being run. Gets passed a TestResult dict which should not be complete."""
         pass

testify/test_runner.py

@@ -115,13 +115,10 @@ def _construct_test(self, test_case_cls, **kwargs):
 
     def discover(self):
         def discover_tests():
-            return sorted(
-                [
-                    self._construct_test(test_case_class)
-                    for test_case_class in test_discovery.discover(self.test_path_or_test_case)
-                    if not self.module_method_overrides or test_case_class.__name__ in self.module_method_overrides
-                ],
-                key=lambda test_case: MetaTestCase._cmp_str(type(test_case)),
+            return (
+                self._construct_test(test_case_class)
+                for test_case_class in test_discovery.discover(self.test_path_or_test_case)
+                if not self.module_method_overrides or test_case_class.__name__ in self.module_method_overrides
             )
 
         def discover_tests_by_buckets():
@@ -153,23 +150,20 @@ def discover_tests_testing():
             # For testing purposes only
             return [self.test_path_or_test_case()]
 
-        discovered_tests = []
+        if isinstance(self.test_path_or_test_case, (TestCase, MetaTestCase)):
+            discovered_tests = discover_tests_testing()
+        elif self.bucket is not None:
+            discovered_tests = discover_tests_by_buckets()
+        else:
+            discovered_tests = discover_tests()
+
         try:
-            if isinstance(self.test_path_or_test_case, (TestCase, MetaTestCase)):
-                discovered_tests = discover_tests_testing()
-            elif self.bucket is not None:
-                discovered_tests = discover_tests_by_buckets()
-            else:
-                discovered_tests = discover_tests()
+            for test in discovered_tests:
+                yield test
         except test_discovery.DiscoveryError as exc:
             for reporter in self.test_reporters:
                 reporter.test_discovery_failure(exc)
             sys.exit(1)
-        test_case_count = len(discovered_tests)
-        test_method_count = sum(len(list(test_case.runnable_test_methods())) for test_case in discovered_tests)
-        for reporter in self.test_reporters:
-            reporter.test_counts(test_case_count, test_method_count)
-        return discovered_tests
 
     def run(self):
         """Instantiate our found test case classes and run their test methods.
@@ -252,22 +246,14 @@ def list_suites(self):
 
     def get_tests_for_suite(self, selected_suite_name):
         """Gets the test list for the suite"""
-        test_list = []
         for test_instance in self.discover():
             for test_method in test_instance.runnable_test_methods():
                 if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
-                    test_list.append(test_method)
-        return test_list
+                    yield test_method
 
     def list_tests(self, selected_suite_name=None):
         """Lists all tests, optionally scoped to a single suite."""
-        test_list = self.get_tests_for_suite(selected_suite_name)
-        test_method_names = sorted([
-            self.get_test_method_name(test) for test in test_list
-        ])
-        for test_method_name in test_method_names:
-            print(test_method_name)
-
-        return test_list
+        for test in self.get_tests_for_suite(selected_suite_name):
+            print(self.get_test_method_name(test))
 
 # vim: set ts=4 sts=4 sw=4 et:

testify/test_runner_json_replay.py

@@ -31,9 +31,6 @@ def run(self):
             test_cases.add((result['method']['module'], result['method']['class'],))
             test_methods.add((result['method']['module'], result['method']['class'], result['method']['name'],))
 
-        for reporter in self.test_reporters:
-            reporter.test_counts(len(test_cases), len(test_methods))
-
         for result in self.results:
             for reporter in self.test_reporters:
                 reporter.test_start(result)