test: make it easier to run tests for subsystems

You can now run suites for subsystem using shorthand, e.g., http.
Switch to black-list of default test folders from white-list.
Tests run by 'make test', 'make coverage', etc., now configurable.
Stop running known_issues suite when collecting test coverage.

PR-URL: nodejs/node#15450
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: Benjamin Coe

Makefile

@@ -10,7 +10,7 @@ TEST_CI_ARGS ?=
 STAGINGSERVER ?= node-www
 LOGLEVEL ?= silent
 OSTYPE := $(shell uname -s | tr '[A-Z]' '[a-z]')
-COVTESTS ?= test
+COVTESTS ?= test-cov
 GTEST_FILTER ?= "*"
 GNUMAKEFLAGS += --no-print-directory
 
@@ -207,10 +207,20 @@ test: all
 	$(PYTHON) tools/test.py --mode=release -J \
 		$(CI_ASYNC_HOOKS) \
 		$(CI_JS_SUITES) \
-		$(CI_NATIVE_SUITES)
+		$(CI_NATIVE_SUITES) \
+		known_issues
 	$(MAKE) lint
 endif
 
+test-cov: all
+	$(MAKE) build-addons
+	$(MAKE) build-addons-napi
+	# $(MAKE) cctest
+	$(PYTHON) tools/test.py --mode=release -J \
+		$(CI_JS_SUITES) \
+		$(CI_NATIVE_SUITES)
+	$(MAKE) lint
+
 test-parallel: all
 	$(PYTHON) tools/test.py --mode=release parallel -J
 
@@ -339,7 +349,7 @@ test-all-valgrind: test-build
 
 CI_NATIVE_SUITES := addons addons-napi
 CI_ASYNC_HOOKS := async-hooks
-CI_JS_SUITES := abort doctool es-module inspector known_issues message parallel pseudo-tty sequential
+CI_JS_SUITES ?= default
 
 # Build and test addons without building anything else
 test-ci-native: LOGLEVEL := silent
@@ -352,7 +362,7 @@ test-ci-native: | test/addons/.buildstamp test/addons-napi/.buildstamp
 test-ci-js: | clear-stalled
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
 		--mode=release --flaky-tests=$(FLAKY_TESTS) \
-		$(TEST_CI_ARGS) $(CI_ASYNC_HOOKS) $(CI_JS_SUITES)
+		$(TEST_CI_ARGS) $(CI_ASYNC_HOOKS) known_issues
 	# Clean up any leftover processes, error if found.
 	ps awwx | grep Release/node | grep -v grep | cat
 	@PS_OUT=`ps awwx | grep Release/node | grep -v grep | awk '{print $$1}'`; \
@@ -365,7 +375,7 @@ test-ci: | clear-stalled build-addons build-addons-napi
 	out/Release/cctest --gtest_output=tap:cctest.tap
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p oneline \
 		--mode=release --flaky-tests=$(FLAKY_TESTS) \
-		$(TEST_CI_ARGS) $(CI_ASYNC_HOOKS) $(CI_JS_SUITES) $(CI_NATIVE_SUITES)
+		$(TEST_CI_ARGS) $(CI_ASYNC_HOOKS) $(CI_JS_SUITES) known_issues
 	# Clean up any leftover processes, error if found.
 	ps awwx | grep Release/node | grep -v grep | cat
 	@PS_OUT=`ps awwx | grep Release/node | grep -v grep | awk '{print $$1}'`; \

tools/test.py

@@ -1548,23 +1548,6 @@ def ExpandCommand(args):
       return prefix + args + suffix
     return ExpandCommand
 
-
-BUILT_IN_TESTS = [
-  'sequential',
-  'parallel',
-  'pummel',
-  'message',
-  'internet',
-  'addons',
-  'addons-napi',
-  'gc',
-  'debugger',
-  'doctool',
-  'inspector',
-  'async-hooks',
-]
-
-
 def GetSuites(test_root):
   def IsSuite(path):
     return isdir(path) and exists(join(path, 'testcfg.py'))
@@ -1583,6 +1566,32 @@ def PrintCrashed(code):
     return "CRASHED (Signal: %d)" % -code
 
 
+# these suites represent special cases that should not be run as part of the
+# default JavaScript test-run, e.g., internet/ requires a network connection,
+# addons/ requires compilation.
+IGNORED_SUITES = [
+  'addons',
+  'addons-napi',
+  'gc',
+  'internet',
+  'pummel',
+  'test-known-issues',
+  'tick-processor',
+  'timers'
+]
+
+
+def ArgsToTestPaths(test_root, args, suites):
+  if len(args) == 0 or 'default' in args:
+    def_suites = filter(lambda s: s not in IGNORED_SUITES, suites)
+    args = filter(lambda a: a != 'default', args) + def_suites
+  subsystem_regex = re.compile(r'^[a-zA-Z-]*$')
+  check = lambda arg: subsystem_regex.match(arg) and (arg not in suites)
+  mapped_args = ["*/test*-%s-*" % arg if check(arg) else arg for arg in args]
+  paths = [SplitPath(NormalizePath(a)) for a in mapped_args]
+  return paths
+
+
 def Main():
   parser = BuildOptions()
   (options, args) = parser.parse_args()
@@ -1598,18 +1607,13 @@ def Main():
     logger.addHandler(fh)
 
   workspace = abspath(join(dirname(sys.argv[0]), '..'))
-  suites = GetSuites(join(workspace, 'test'))
+  test_root = join(workspace, 'test')
+  suites = GetSuites(test_root)
   repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
   repositories += [TestRepository(a) for a in options.suite]
 
   root = LiteralTestSuite(repositories)
-  if len(args) == 0:
-    paths = [SplitPath(t) for t in BUILT_IN_TESTS]
-  else:
-    paths = [ ]
-    for arg in args:
-      path = SplitPath(NormalizePath(arg))
-      paths.append(path)
+  paths = ArgsToTestPaths(test_root, args, suites)
 
   # Check for --valgrind option. If enabled, we overwrite the special
   # command flag with a command that uses the run-valgrind.py script.

vcbuild.bat

@@ -44,7 +44,7 @@ set enable_static=
 set build_addons_napi=
 set test_node_inspect=
 set test_check_deopts=
-set js_test_suites=abort async-hooks es-module inspector known_issues message parallel sequential
+set js_test_suites=default async-hooks known_issues
 set v8_test_options=
 set v8_build_options=
 set "common_test_suites=%js_test_suites% doctool addons addons-napi&set build_addons=1&set build_addons_napi=1"