From 131f694d347f773d2462424c40cb13f23235864a Mon Sep 17 00:00:00 2001 From: Buck Evan Date: Tue, 24 May 2016 17:32:40 -0700 Subject: [PATCH] test discovery: yield results progressively saves 20% wall time and 20% memory --- test/test_runner_test.py | 4 +-- testify/test_discovery.py | 28 +++++++++--------- testify/test_reporter.py | 4 --- testify/test_runner.py | 46 +++++++++++------------------- testify/test_runner_json_replay.py | 3 -- 5 files changed, 32 insertions(+), 53 deletions(-) diff --git a/test/test_runner_test.py b/test/test_runner_test.py index 0f980521..935061c1 100644 --- a/test/test_runner_test.py +++ b/test/test_runner_test.py @@ -116,14 +116,14 @@ def test_get_tests_for_suite_in_suite(self): instance = test_runner.TestRunner(mock.sentinel.test_class) ret = instance.get_tests_for_suite(mock.sentinel.selected_suite_name) - assert_equal(ret, [self.mock_test_method]) + assert_equal(list(ret), [self.mock_test_method]) def test_get_tests_for_suite_not_in_suite(self): self.in_suite_mock.return_value = False instance = test_runner.TestRunner(mock.sentinel.test_class) ret = instance.get_tests_for_suite(mock.sentinel.selected_suite_name) - assert_equal(ret, []) + assert_equal(list(ret), []) class TestTestRunnerPrintsTestNames(test_case.TestCase): diff --git a/testify/test_discovery.py b/testify/test_discovery.py index afe056ef..ec23a1af 100644 --- a/testify/test_discovery.py +++ b/testify/test_discovery.py @@ -83,20 +83,20 @@ def discover(what): try: what = to_module(what) mod = __import__(what, fromlist=[str('__trash')]) - discovered = set() - - if hasattr(mod, '__path__'): - # It's a package! - for _, module_name, _ in pkgutil.walk_packages( - mod.__path__, - prefix=mod.__name__ + '.', - ): - mod = __import__(module_name, fromlist=[str('__trash')]) - discovered.update(get_test_classes_from_module(mod)) - else: - discovered.update(get_test_classes_from_module(mod)) - - return discovered + for cls in get_test_classes_from_module(mod): + yield cls + + if not hasattr(mod, '__path__'): + return + + # It's a package! + for _, module_name, _ in pkgutil.walk_packages( + mod.__path__, + prefix=mod.__name__ + '.', + ): + submod = __import__(module_name, fromlist=[str('__trash')]) + for cls in get_test_classes_from_module(submod): + yield cls except Exception: traceback.print_exc() raise DiscoveryError( diff --git a/testify/test_reporter.py b/testify/test_reporter.py index 0f0f26a6..2e997f1a 100644 --- a/testify/test_reporter.py +++ b/testify/test_reporter.py @@ -27,10 +27,6 @@ def __init__(self, options): """ self.options = options - def test_counts(self, test_case_count, test_method_count): - """Called after discovery finishes. May not be called by all test runners, e.g. TestRunnerClient.""" - pass - def test_start(self, result): """Called when a test method is being run. Gets passed a TestResult dict which should not be complete.""" pass diff --git a/testify/test_runner.py b/testify/test_runner.py index 8f5d8434..590941bd 100644 --- a/testify/test_runner.py +++ b/testify/test_runner.py @@ -115,13 +115,10 @@ def _construct_test(self, test_case_cls, **kwargs): def discover(self): def discover_tests(): - return sorted( - [ - self._construct_test(test_case_class) - for test_case_class in test_discovery.discover(self.test_path_or_test_case) - if not self.module_method_overrides or test_case_class.__name__ in self.module_method_overrides - ], - key=lambda test_case: MetaTestCase._cmp_str(type(test_case)), + return ( + self._construct_test(test_case_class) + for test_case_class in test_discovery.discover(self.test_path_or_test_case) + if not self.module_method_overrides or test_case_class.__name__ in self.module_method_overrides ) def discover_tests_by_buckets(): @@ -153,23 +150,20 @@ def discover_tests_testing(): # For testing purposes only return [self.test_path_or_test_case()] - discovered_tests = [] + if isinstance(self.test_path_or_test_case, (TestCase, MetaTestCase)): + discovered_tests = discover_tests_testing() + elif self.bucket is not None: + discovered_tests = discover_tests_by_buckets() + else: + discovered_tests = discover_tests() + try: - if isinstance(self.test_path_or_test_case, (TestCase, MetaTestCase)): - discovered_tests = discover_tests_testing() - elif self.bucket is not None: - discovered_tests = discover_tests_by_buckets() - else: - discovered_tests = discover_tests() + for test in discovered_tests: + yield test except test_discovery.DiscoveryError as exc: for reporter in self.test_reporters: reporter.test_discovery_failure(exc) sys.exit(1) - test_case_count = len(discovered_tests) - test_method_count = sum(len(list(test_case.runnable_test_methods())) for test_case in discovered_tests) - for reporter in self.test_reporters: - reporter.test_counts(test_case_count, test_method_count) - return discovered_tests def run(self): """Instantiate our found test case classes and run their test methods. @@ -252,22 +246,14 @@ def list_suites(self): def get_tests_for_suite(self, selected_suite_name): """Gets the test list for the suite""" - test_list = [] for test_instance in self.discover(): for test_method in test_instance.runnable_test_methods(): if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name): - test_list.append(test_method) - return test_list + yield test_method def list_tests(self, selected_suite_name=None): """Lists all tests, optionally scoped to a single suite.""" - test_list = self.get_tests_for_suite(selected_suite_name) - test_method_names = sorted([ - self.get_test_method_name(test) for test in test_list - ]) - for test_method_name in test_method_names: - print(test_method_name) - - return test_list + for test in self.get_tests_for_suite(selected_suite_name): + print(self.get_test_method_name(test)) # vim: set ts=4 sts=4 sw=4 et: diff --git a/testify/test_runner_json_replay.py b/testify/test_runner_json_replay.py index dbf9afe7..2f4a2a9a 100644 --- a/testify/test_runner_json_replay.py +++ b/testify/test_runner_json_replay.py @@ -31,9 +31,6 @@ def run(self): test_cases.add((result['method']['module'], result['method']['class'],)) test_methods.add((result['method']['module'], result['method']['class'], result['method']['name'],)) - for reporter in self.test_reporters: - reporter.test_counts(len(test_cases), len(test_methods)) - for result in self.results: for reporter in self.test_reporters: reporter.test_start(result)