diff --git a/setup.py b/setup.py index f1177c5..1dd7409 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ ], license='Apache 2.0', install_requires=[ - 'coverage<5.0' + 'coverage>=6.0' ], packages=['testflo'], entry_points=""" diff --git a/testflo/__main__.py b/testflo/__main__.py new file mode 100644 index 0000000..7fa44db --- /dev/null +++ b/testflo/__main__.py @@ -0,0 +1,5 @@ +"""This file makes testflo runnable using 'python -m testflo'.""" +from testflo.main import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/testflo/cover.py b/testflo/cover.py index 48ca7f3..c88cde9 100644 --- a/testflo/cover.py +++ b/testflo/cover.py @@ -1,20 +1,74 @@ """ Methods to provide code coverage using coverage.py. """ - import os import sys +import shutil import webbrowser try: - from coverage import coverage + import coverage + from coverage.config import HandyConfigParser except ImportError: coverage = None +else: + coverage.process_startup() # use to hold a global coverage obj _coverobj = None +def _to_ini(lst): + if lst: + return ','.join(lst) + return '' + + +def _write_temp_config(options, rcfile): + """ + Read any .coveragerc file if it exists, and override parts of it then generate our temp config. + + Parameters + ---------- + options : cmd line options + Options from the command line parser. + rcfile : str + The name of our temporary coverage config file. + """ + tmp_cfg = { + 'run': { + 'branch': False, + 'parallel': True, + 'concurrency': 'multiprocessing', + }, + 'report': { + 'ignore_errors': True, + 'skip_empty': True, + 'sort': '-cover', + }, + 'html': { + 'skip_empty': True, + } + } + + if options.coverpkgs: + tmp_cfg['run']['source_pkgs'] = _to_ini(options.coverpkgs) + + if options.cover_omits: + tmp_cfg['run']['omit'] = _to_ini(options.cover_omits) + tmp_cfg['report']['omit'] = _to_ini(options.cover_omits) + + cfgparser = HandyConfigParser(our_file=True) + + if os.path.isfile('.coveragerc'): + cfgparser.read(['.coveragerc']) + + cfgparser.read_dict(tmp_cfg) + + with open(rcfile, 'w') as f: + cfgparser.write(f) + + def setup_coverage(options): global _coverobj if _coverobj is None and (options.coverage or options.coveragehtml): @@ -23,8 +77,19 @@ def setup_coverage(options): if not options.coverpkgs: raise RuntimeError("No packages specified for coverage. " "Use the --coverpkg option to add a package.") - _coverobj = coverage(data_suffix=True, source=options.coverpkgs, - omit=options.cover_omits) + oldcov = os.path.join(os.getcwd(), '.coverage') + if os.path.isfile(oldcov): + os.remove(oldcov) + covdir = os.path.join(os.getcwd(), '_covdir') + if os.path.isdir('_covdir'): + shutil.rmtree('_covdir') + os.mkdir('_covdir') + os.environ['COVERAGE_RUN'] = 'true' + os.environ['COVERAGE_RCFILE'] = rcfile = os.path.join(covdir, '_coveragerc_') + os.environ['COVERAGE_FILE'] = covfile = os.path.join(covdir, '.coverage') + os.environ['COVERAGE_PROCESS_START'] = rcfile + _write_temp_config(options, rcfile) + _coverobj = coverage.Coverage(data_file=covfile, data_suffix=True, config_file=rcfile) return _coverobj def start_coverage(): @@ -65,10 +130,7 @@ def finalize_coverage(options): morfs = list(find_files(dirs, match='*.py', exclude=excl)) _coverobj.combine() - - # write combined data to default filename (as used by coveralls) - # (NOTE: get_data() returns None, so using data attribute) - _coverobj.data.write_file('.coverage') + _coverobj.save() if options.coverage: _coverobj.report(morfs=morfs) @@ -81,3 +143,6 @@ def finalize_coverage(options): os.system('open %s' % outfile) else: webbrowser.get().open(outfile) + + shutil.copy(_coverobj.get_data().data_filename(), + os.path.join(os.getcwd(), '.coverage')) diff --git a/testflo/isolatedrun.py b/testflo/isolatedrun.py index 65637bd..c7ff4b0 100644 --- a/testflo/isolatedrun.py +++ b/testflo/isolatedrun.py @@ -5,12 +5,18 @@ """ if __name__ == '__main__': + try: + import coverage + except ImportError: + coverage = None + else: + coverage.process_startup() + import sys import os import traceback from testflo.test import Test - from testflo.cover import save_coverage from testflo.qman import get_client_queue from testflo.options import get_options @@ -29,8 +35,6 @@ test.status = 'FAIL' test.err_msg = traceback.format_exc() - save_coverage() - except: test.err_msg = traceback.format_exc() test.status = 'FAIL' diff --git a/testflo/main.py b/testflo/main.py index de8269b..5907209 100644 --- a/testflo/main.py +++ b/testflo/main.py @@ -29,6 +29,8 @@ def get_iter(self, input_iter) import time import warnings import multiprocessing +import atexit +import shutil from fnmatch import fnmatch, fnmatchcase @@ -42,8 +44,7 @@ def get_iter(self, input_iter) from testflo.discover import TestDiscoverer from testflo.filters import TimeFilter, FailFilter -from testflo.util import read_config_file, read_test_file, _get_parser -from testflo.cover import setup_coverage, finalize_coverage +from testflo.util import read_config_file, read_test_file from testflo.options import get_options from testflo.qman import get_server_queue @@ -165,7 +166,22 @@ def dir_exclude(d): # set this so code will know when it's running under testflo os.environ['TESTFLO_RUNNING'] = '1' - setup_coverage(options) + if options.coverage or options.coveragehtml: + os.environ['TESTFLO_MAIN_PID'] = str(os.getpid()) + # some coverage files aren't written until atexit of their processes, so put our finalize + # routine in atexit before their Coverage._atexit methods are registered so ours will be + # executed *after* theirs. + def _finalize(): + if os.getpid() == int(os.environ.get('TESTFLO_MAIN_PID', '0')): + from testflo.cover import finalize_coverage + finalize_coverage(options) + # clean up the temporary dir where we store the interim coverage files from all + # of the processes. + if os.path.isdir('_covdir'): + shutil.rmtree('_covdir') + atexit.register(_finalize) + from testflo.cover import setup_coverage + setup_coverage(options) if options.noreport: report_file = open(os.devnull, 'a') @@ -175,17 +191,6 @@ def dir_exclude(d): if not options.test_glob: options.test_glob = ['test*'] - def func_matcher(funcname): - for pattern in options.excludes: - if fnmatchcase(funcname, pattern): - return False - - for pattern in options.test_glob: - if fnmatchcase(funcname, pattern): - return True - - return False - if options.benchmark: options.num_procs = 1 options.isolated = True @@ -194,8 +199,18 @@ def func_matcher(funcname): dir_exclude=dir_exclude) benchmark_file = open(options.benchmarkfile, 'a') else: - discoverer = TestDiscoverer(options, dir_exclude=dir_exclude, - func_match=func_matcher) + def func_matcher(funcname): + for pattern in options.excludes: + if fnmatchcase(funcname, pattern): + return False + + for pattern in options.test_glob: + if fnmatchcase(funcname, pattern): + return True + + return False + + discoverer = TestDiscoverer(options, dir_exclude=dir_exclude, func_match=func_matcher) benchmark_file = open(os.devnull, 'a') retval = 0 @@ -253,8 +268,6 @@ def func_matcher(funcname): retval = run_pipeline(tests, pipeline, options.disallow_skipped) - finalize_coverage(options) - if manager is not None: manager.shutdown() diff --git a/testflo/mpirun.py b/testflo/mpirun.py index 66f0b21..044100f 100644 --- a/testflo/mpirun.py +++ b/testflo/mpirun.py @@ -6,15 +6,22 @@ """ if __name__ == '__main__': + try: + import coverage + except ImportError: + pass + else: + coverage.process_startup() + import sys import os import traceback + # when testing OpenMDAO, make sure that MPI is active os.environ['OPENMDAO_USE_MPI'] = '1' from mpi4py import MPI from testflo.test import Test - from testflo.cover import setup_coverage, save_coverage from testflo.qman import get_client_queue from testflo.options import get_options @@ -25,46 +32,34 @@ os.environ['TESTFLO_QUEUE'] = '' options = get_options() - setup_coverage(options) try: try: comm = MPI.COMM_WORLD test = Test(sys.argv[1], options) test.nocapture = True # so we don't lose stdout - tests = test.run() + test.run() except: print(traceback.format_exc()) test.status = 'FAIL' test.err_msg = traceback.format_exc() - tests = [test] - - # collect results - results = comm.gather(tests, root=0) - if comm.rank == 0: - for r in results: - for tst in r: - if not isinstance(tst, Test): - print("\nNot all results gathered are Test objects. " - "You may have out-of-sync collective MPI calls.\n") - break - total_mem_usage = 0. - for r in results: - for tst in r: - if isinstance(tst, Test): - total_mem_usage += tst.memory_usage - break # subtests don't track their own memory usage, so break after 1st one - for tst in tests: - tst.memory_usage = total_mem_usage - - # check for errors and record error message - for r in results: - for test, tst in zip(tests, r): - if test.status != 'FAIL' and tst.status in ('SKIP', 'FAIL'): - test.err_msg = tst.err_msg - test.status = tst.status + else: + # collect results + results = comm.gather(test, root=0) + if comm.rank == 0: + if not all([isinstance(r, Test) for r in results]): + print("\nNot all results gathered are Test objects. " + "You may have out-of-sync collective MPI calls.\n") + total_mem_usage = sum(r.memory_usage for r in results if isinstance(r, Test)) + test.memory_usage = total_mem_usage - save_coverage() + # check for errors and record error message + for r in results: + if test.status != 'FAIL' and r.status in ('SKIP', 'FAIL'): + test.err_msg = r.err_msg + test.status = r.status + if r.status == 'FAIL': + break except Exception: test.err_msg = traceback.format_exc() @@ -75,4 +70,4 @@ sys.stderr.flush() if comm.rank == 0: - queue.put(tests) + queue.put(test) diff --git a/testflo/test.py b/testflo/test.py index 34ba4c2..425ed8a 100644 --- a/testflo/test.py +++ b/testflo/test.py @@ -379,8 +379,6 @@ def run(self, queue=None): self.deprecations[msg] = dep finally: - stop_coverage() - sys.stderr = old_err sys.stdout = old_out diff --git a/testflo/util.py b/testflo/util.py index 329dc84..fb82393 100644 --- a/testflo/util.py +++ b/testflo/util.py @@ -394,12 +394,7 @@ def get_module(fname): else: raise ImportError("can't import %s" % modpath) - start_coverage() - - try: - mod = try_import(fname, modpath) - finally: - stop_coverage() + mod = try_import(fname, modpath) return fname, mod