Updated testflo to work with new version (>=6.0) of coverage

setup.py

@@ -32,7 +32,7 @@
       ],
       license='Apache 2.0',
       install_requires=[
-        'coverage<5.0'
+        'coverage>=6.0'
       ],
       packages=['testflo'],
       entry_points="""

testflo/__main__.py

@@ -0,0 +1,5 @@
+"""This file makes testflo runnable using 'python -m testflo'."""
+from testflo.main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

testflo/cover.py

@@ -1,20 +1,74 @@
 """
 Methods to provide code coverage using coverage.py.
 """
-
 import os
 import sys
+import shutil
 import webbrowser
 
 try:
-    from coverage import coverage
+    import coverage
+    from coverage.config import HandyConfigParser
 except ImportError:
     coverage = None
+else:
+    coverage.process_startup()
 
 
 # use to hold a global coverage obj
 _coverobj = None
 
+def _to_ini(lst):
+    if lst:
+        return ','.join(lst)
+    return ''
+
+
+def _write_temp_config(options, rcfile):
+    """
+    Read any .coveragerc file if it exists, and override parts of it then generate our temp config.
+
+    Parameters
+    ----------
+    options : cmd line options
+        Options from the command line parser.
+    rcfile : str
+        The name of our temporary coverage config file.
+    """
+    tmp_cfg = {
+        'run': {
+            'branch': False,
+            'parallel': True,
+            'concurrency': 'multiprocessing',
+        },
+        'report': {
+            'ignore_errors': True,
+            'skip_empty': True,
+            'sort': '-cover',
+        },
+        'html': {
+            'skip_empty': True,
+        }
+    }
+
+    if options.coverpkgs:
+        tmp_cfg['run']['source_pkgs'] = _to_ini(options.coverpkgs)
+
+    if options.cover_omits:
+        tmp_cfg['run']['omit'] = _to_ini(options.cover_omits)
+        tmp_cfg['report']['omit'] = _to_ini(options.cover_omits)
+
+    cfgparser = HandyConfigParser(our_file=True)
+
+    if os.path.isfile('.coveragerc'):
+        cfgparser.read(['.coveragerc'])
+
+    cfgparser.read_dict(tmp_cfg)
+
+    with open(rcfile, 'w') as f:
+        cfgparser.write(f)
+
+
 def setup_coverage(options):
     global _coverobj
     if _coverobj is None and (options.coverage or options.coveragehtml):
@@ -23,8 +77,19 @@ def setup_coverage(options):
         if not options.coverpkgs:
             raise RuntimeError("No packages specified for coverage. "
                                "Use the --coverpkg option to add a package.")
-        _coverobj = coverage(data_suffix=True, source=options.coverpkgs,
-                             omit=options.cover_omits)
+        oldcov = os.path.join(os.getcwd(), '.coverage')
+        if os.path.isfile(oldcov):
+            os.remove(oldcov)
+        covdir = os.path.join(os.getcwd(), '_covdir')
+        if os.path.isdir('_covdir'):
+            shutil.rmtree('_covdir')
+        os.mkdir('_covdir')
+        os.environ['COVERAGE_RUN'] = 'true'
+        os.environ['COVERAGE_RCFILE'] = rcfile = os.path.join(covdir, '_coveragerc_')
+        os.environ['COVERAGE_FILE'] = covfile = os.path.join(covdir, '.coverage')
+        os.environ['COVERAGE_PROCESS_START'] = rcfile
+        _write_temp_config(options, rcfile)
+        _coverobj = coverage.Coverage(data_file=covfile, data_suffix=True, config_file=rcfile)
     return _coverobj
 
 def start_coverage():
@@ -65,10 +130,7 @@ def finalize_coverage(options):
             morfs = list(find_files(dirs, match='*.py', exclude=excl))
 
             _coverobj.combine()
-
-            # write combined data to default filename (as used by coveralls)
-            # (NOTE: get_data() returns None, so using data attribute)
-            _coverobj.data.write_file('.coverage')
+            _coverobj.save()
 
             if options.coverage:
                 _coverobj.report(morfs=morfs)
@@ -81,3 +143,6 @@ def finalize_coverage(options):
                     os.system('open %s' % outfile)
                 else:
                     webbrowser.get().open(outfile)
+
+            shutil.copy(_coverobj.get_data().data_filename(),
+                        os.path.join(os.getcwd(), '.coverage'))

testflo/isolatedrun.py

@@ -5,12 +5,18 @@
 """
 
 if __name__ == '__main__':
+    try:
+        import coverage
+    except ImportError:
+        coverage = None
+    else:
+        coverage.process_startup()
+
     import sys
     import os
     import traceback
 
     from testflo.test import Test
-    from testflo.cover import save_coverage
     from testflo.qman import get_client_queue
     from testflo.options import get_options
 
@@ -29,8 +35,6 @@
             test.status = 'FAIL'
             test.err_msg = traceback.format_exc()
 
-        save_coverage()
-
     except:
         test.err_msg = traceback.format_exc()
         test.status = 'FAIL'

testflo/main.py

@@ -29,6 +29,8 @@ def get_iter(self, input_iter)
 import time
 import warnings
 import multiprocessing
+import atexit
+import shutil
 
 from fnmatch import fnmatch, fnmatchcase
 
@@ -42,8 +44,7 @@ def get_iter(self, input_iter)
 from testflo.discover import TestDiscoverer
 from testflo.filters import TimeFilter, FailFilter
 
-from testflo.util import read_config_file, read_test_file, _get_parser
-from testflo.cover import setup_coverage, finalize_coverage
+from testflo.util import read_config_file, read_test_file
 from testflo.options import get_options
 from testflo.qman import get_server_queue
 
@@ -165,7 +166,22 @@ def dir_exclude(d):
     # set this so code will know when it's running under testflo
     os.environ['TESTFLO_RUNNING'] = '1'
 
-    setup_coverage(options)
+    if options.coverage or options.coveragehtml:
+        os.environ['TESTFLO_MAIN_PID'] = str(os.getpid())
+        # some coverage files aren't written until atexit of their processes, so put our finalize
+        # routine in atexit before their Coverage._atexit methods are registered so ours will be
+        # executed *after* theirs.
+        def _finalize():
+            if os.getpid() == int(os.environ.get('TESTFLO_MAIN_PID', '0')):
+                from testflo.cover import finalize_coverage
+                finalize_coverage(options)
+                # clean up the temporary dir where we store the interim coverage files from all
+                # of the processes.
+                if os.path.isdir('_covdir'):
+                    shutil.rmtree('_covdir')
+        atexit.register(_finalize)
+        from testflo.cover import setup_coverage
+        setup_coverage(options)
 
     if options.noreport:
         report_file = open(os.devnull, 'a')
@@ -175,17 +191,6 @@ def dir_exclude(d):
     if not options.test_glob:
         options.test_glob = ['test*']
 
-    def func_matcher(funcname):
-        for pattern in options.excludes:
-            if fnmatchcase(funcname, pattern):
-                return False
-
-        for pattern in options.test_glob:
-            if fnmatchcase(funcname, pattern):
-                return True
-
-        return False
-
     if options.benchmark:
         options.num_procs = 1
         options.isolated = True
@@ -194,8 +199,18 @@ def func_matcher(funcname):
                                     dir_exclude=dir_exclude)
         benchmark_file = open(options.benchmarkfile, 'a')
     else:
-        discoverer = TestDiscoverer(options, dir_exclude=dir_exclude,
-                                    func_match=func_matcher)
+        def func_matcher(funcname):
+            for pattern in options.excludes:
+                if fnmatchcase(funcname, pattern):
+                    return False
+
+            for pattern in options.test_glob:
+                if fnmatchcase(funcname, pattern):
+                    return True
+
+            return False
+
+        discoverer = TestDiscoverer(options, dir_exclude=dir_exclude, func_match=func_matcher)
         benchmark_file = open(os.devnull, 'a')
 
     retval = 0
@@ -253,8 +268,6 @@ def func_matcher(funcname):
 
         retval = run_pipeline(tests, pipeline, options.disallow_skipped)
 
-        finalize_coverage(options)
-
         if manager is not None:
             manager.shutdown()
 

testflo/mpirun.py

@@ -6,15 +6,22 @@
 """
 
 if __name__ == '__main__':
+    try:
+        import coverage
+    except ImportError:
+        pass
+    else:
+        coverage.process_startup()
+
     import sys
     import os
     import traceback
 
+    # when testing OpenMDAO, make sure that MPI is active
     os.environ['OPENMDAO_USE_MPI'] = '1'
 
     from mpi4py import MPI
     from testflo.test import Test
-    from testflo.cover import setup_coverage, save_coverage
     from testflo.qman import get_client_queue
     from testflo.options import get_options
 
@@ -25,46 +32,34 @@
     os.environ['TESTFLO_QUEUE'] = ''
 
     options = get_options()
-    setup_coverage(options)
 
     try:
         try:
             comm = MPI.COMM_WORLD
             test = Test(sys.argv[1], options)
             test.nocapture = True # so we don't lose stdout
-            tests = test.run()
+            test.run()
         except:
             print(traceback.format_exc())
             test.status = 'FAIL'
             test.err_msg = traceback.format_exc()
-            tests = [test]
-
-        # collect results
-        results = comm.gather(tests, root=0)
-        if comm.rank == 0:
-            for r in results:
-                for tst in r:
-                    if not isinstance(tst, Test):
-                        print("\nNot all results gathered are Test objects.  "
-                              "You may have out-of-sync collective MPI calls.\n")
-                        break
-            total_mem_usage = 0.
-            for r in results:
-                for tst in r:
-                    if isinstance(tst, Test):
-                        total_mem_usage += tst.memory_usage
-                    break  # subtests don't track their own memory usage, so break after 1st one
-            for tst in tests:
-                tst.memory_usage = total_mem_usage
-
-            # check for errors and record error message
-            for r in results:
-                for test, tst in zip(tests, r):
-                    if test.status != 'FAIL' and tst.status in ('SKIP', 'FAIL'):
-                        test.err_msg = tst.err_msg
-                        test.status = tst.status
+        else:
+            # collect results
+            results = comm.gather(test, root=0)
+            if comm.rank == 0:
+                if not all([isinstance(r, Test) for r in results]):
+                    print("\nNot all results gathered are Test objects.  "
+                          "You may have out-of-sync collective MPI calls.\n")
+                total_mem_usage = sum(r.memory_usage for r in results if isinstance(r, Test))
+                test.memory_usage = total_mem_usage
 
-        save_coverage()
+                # check for errors and record error message
+                for r in results:
+                    if test.status != 'FAIL' and r.status in ('SKIP', 'FAIL'):
+                        test.err_msg = r.err_msg
+                        test.status = r.status
+                        if r.status == 'FAIL':
+                            break
 
     except Exception:
         test.err_msg = traceback.format_exc()
@@ -75,4 +70,4 @@
         sys.stderr.flush()
 
         if comm.rank == 0:
-            queue.put(tests)
+            queue.put(test)

testflo/test.py

@@ -379,8 +379,6 @@ def run(self, queue=None):
                             self.deprecations[msg] = dep
 
             finally:
-                stop_coverage()
-
                 sys.stderr = old_err
                 sys.stdout = old_out
 

testflo/util.py

@@ -394,12 +394,7 @@ def get_module(fname):
             else:
                 raise ImportError("can't import %s" % modpath)
 
-    start_coverage()
-
-    try:
-        mod = try_import(fname, modpath)
-    finally:
-        stop_coverage()
+    mod = try_import(fname, modpath)
 
     return fname, mod