Support multiple test suites and add helpful verifiers

test/array-test.cpp

@@ -16,9 +16,9 @@
 
 #include "../include/array.h"
 
-#include "tester.h"
+#include "verifiers.h"
 
-void runTests()
+void array_test()
 {
 	using sigcpp::array;
 
@@ -30,41 +30,41 @@ void runTests()
 
 
 	//capacity
-	verify(!s.empty(), "s.empty()");
-	verify(s.size() == 3, "s.size()");
-	verify(s.max_size() == s.size(), "s.max_size()");
+	is_false(s.empty(), "s.empty()");
+	is_true(s.size() == 3, "s.size()");
+	is_true(s.max_size() == s.size(), "s.max_size()");
 
-	verify(!p.empty(), "p.empty()");
-	verify(p.size() == 5, "p.size()");
-	verify(p.max_size() == p.size(), "p.max_size()");
+	is_false(p.empty(), "p.empty()");
+	is_true(p.size() == 5, "p.size()");
+	is_true(p.max_size() == p.size(), "p.max_size()");
 
 
 	//element access
-	verify(s[0] == 8, "s[0]");
-	verify(s[1] == -2, "s[1]");
-	verify(s[2] == 7, "s[2]");
-	verify(s[1] != 8, "s[1] != 8");
+	is_true(s[0] == 8, "s[0]");
+	is_true(s[1] == -2, "s[1]");
+	is_true(s[2] == 7, "s[2]");
+	is_true(s[1] != 8, "s[1] != 8");
 
-	verify(p[0] == 8, "p[0]");
-	verify(p[2] == 7, "p[2]");
-	verify(p[4] == 0, "p[4]");
+	is_true(p[0] == 8, "p[0]");
+	is_true(p[2] == 7, "p[2]");
+	is_true(p[4] == 0, "p[4]");
 
-	verify(s.at(0) == 8, "s.at(0)");
-	verify(s.at(1) == -2, "s.at(1)");
-	verify(s.at(2) == 7, "s.at(2)");
+	is_true(s.at(0) == 8, "s.at(0)");
+	is_true(s.at(1) == -2, "s.at(1)");
+	is_true(s.at(2) == 7, "s.at(2)");
 
-	verify(p.at(0) == 8, "p.at(0)");
-	verify(p.at(2) == 7, "p.at(2)");
-	verify(p.at(4) == 0, "p.at(4)");
+	is_true(p.at(0) == 8, "p.at(0)");
+	is_true(p.at(2) == 7, "p.at(2)");
+	is_true(p.at(4) == 0, "p.at(4)");
 
-	verify(s.front() == 8, "s.front()");
-	verify(s.front() != -2, "s.front() != -2");
-	verify(s.back() == 7, "s.back()");
-	verify(s.back() != -2, "s.back() != -2");
+	is_true(s.front() == 8, "s.front()");
+	is_true(s.front() != -2, "s.front() != -2");
+	is_true(s.back() == 7, "s.back()");
+	is_true(s.back() != -2, "s.back() != -2");
 
-	verify(p.front() == 8, "p.front()");
-	verify(p.front() != -2, "p.front() != -2");
-	verify(p.back() == 0, "p.back()");
+	is_true(p.front() == 8, "p.front()");
+	is_true(p.front() != -2, "p.front() != -2");
+	is_true(p.back() == 0, "p.back()");
 
 
 	//forward iterators
@@ -75,7 +75,7 @@ void runTests()
 	std::size_t i = 0;
 	for (auto it = u.begin(); it != u.end() && iteratorTest; ++it, ++i)
 		iteratorTest = *it == uExpected[i];
-	verify(iteratorTest, "forward iterator");
+	is_true(iteratorTest, "forward iterator");
 
 	//reverse iterators
 	unsigned urExpected[] = { 6, 1, 3, 9, 5 };
@@ -84,17 +84,17 @@ void runTests()
 	i = 0;
 	for (auto it = u.rbegin(); it != u.rend() && iteratorTest; ++it, ++i)
 		iteratorTest = *it == urExpected[i];
-	verify(iteratorTest, "reverse iterator");
+	is_true(iteratorTest, "reverse iterator");
 
 	//zero-size array
 	array<char, 0> c;
-	verify(c.empty(), "c.empty()");
+	is_true(c.empty(), "c.empty()");
 
 	//iterator on empty array: the loop body should not execute
 	iteratorTest = true;
 	for (const auto e : c)
 		iteratorTest = false;
-	verify(iteratorTest, "fwd iterator on empty array");
+	is_true(iteratorTest, "fwd iterator on empty array");
 
 
 	//fill
@@ -106,7 +106,7 @@ void runTests()
 	bool fillTest = !std::any_of(a.begin(), a.end(),
 		[](char c) { return c != 'x'; }
 	);
-	verify(fillTest, "a.fill()");
+	is_true(fillTest, "a.fill()");
 
 
 	//swap
@@ -120,5 +120,5 @@ void runTests()
 	bool swapTest = true;
 	for (std::size_t idx = 0; idx < m.size() && swapTest; ++idx)
 		swapTest = m[idx] == mExpected[idx] && n[idx] == nExpected[idx];
-	verify(swapTest, "m.swap(n)");
-}
+	is_true(swapTest, "m.swap(n)");
+}

test/driver.cpp

@@ -16,18 +16,33 @@
 #include <string>
 #include <exception>
 #include <filesystem>
+#include <unordered_map>
+#include <algorithm>
+#include <cassert>
 
+#include "utils.h"
+#include "suites.h"
 #include "tester.h"
 #include "options.h"
 #include "options-exceptions.h"
 
-//must be defined in a unit-specific source file such as "array-test.cpp"
-void runTests();
+//error codes returned back from main are negative
+enum class error_code {
+	cmd_line_initial = -1, file_initial = -2, unexpected_typed_initial = -3, unexpected_untyped_initial = -4,
+	cmd_line_run = -6, file_run = -7, suite_add_run = -8, unexpected_typed_run = -9,
+	unexpected_untyped_run = -10
+};
 
-static void show_error(const char* message);
+
+void run_suites(const Options& options);
+static int show_error(const char* message, error_code ec);
 static void show_usage(const char* program_path);
-static void show_error_and_usage(const char* message, const char* program_path);
+static int show_error_and_usage(const char* message, const char* program_path, error_code ec);
+
 
+//return  < 0: error
+//return == 0: no error, no tests failed
+//return  > 0: no error, tests failed; number of failed tests returned
 int main(int argc, char* argv[])
 {
 	Options options;
@@ -38,59 +53,107 @@ int main(int argc, char* argv[])
 		apply_options(options, fileOut);
 	}
 	catch (const cmd_line_error& cle) {
-		show_error_and_usage(cle.what(), argv[0]);
-		return -1;
+		return show_error_and_usage(cle.what(), argv[0], error_code::cmd_line_initial);
 	}
 	catch (const file_error& fe) {
-		show_error_and_usage(fe.what(), argv[0]);
-		return -2;
+		return show_error_and_usage(fe.what(), argv[0], error_code::file_initial);
 	}
 	catch (const std::exception& e) {
-		show_error((std::string{ "Unexpected error: " } +e.what()).data());
-		return -3;
+		auto message = format_message("Unexpected error", e.what());
+		return show_error(message.data(), error_code::unexpected_typed_initial);
 	}
 	catch (...) {
-		show_error("Unexpected error");
-		return -4;
+		show_error("Unexpected error", error_code::unexpected_untyped_initial);
 	}
 
 
 	try {
-		runTests();
+		run_suites(options);
+	}
+	catch (const cmd_line_error& cle) {
+		show_error_and_usage(cle.what(), argv[0], error_code::cmd_line_run);
 	}
-	catch (const std::string& msg) {
-		logLine(msg.data());
-		if (options.fom == file_open_mode::no_file)
-			show_error(msg.data());
+	catch (const test_suite_add_error& tae) {
+		show_error(tae.what(), error_code::suite_add_run);
+	}
+	catch (const std::exception& e) {
+		auto message = format_message("Unexpected error", e.what());
+		show_error(message.data(), error_code::unexpected_typed_run);
+	}
+	catch (...) {
+		show_error("Unexpected error", error_code::unexpected_untyped_run);
 	}
 
 	if (options.summary)
-		summarizeTests();
+		summarize_tests();
 
-	return getTestsFailed();
+	return get_tests_failed_total();
+}
+
+
+//run test suites in sequence: runs all suites defined or only those whose names are specified in options
+void run_suites(const Options& options)
+{
+	//retrieve all test suites defined
+	auto suites = get_test_suites();
+
+	//build a collection of suite names specified in the options structucre
+	//options.suites_to_run is empty or a semi-colon delimited list of suite names
+	const auto names_to_run = split(options.suites_to_run, ';');
+	auto run_all_suites = names_to_run.empty();
+
+	//check that the suite names specified in options correspond to suites defined
+	//this check is not required to run the suites, but is included to inform the user of any issues
+	//silently ignoring an unfound suite leaves the user unaware of the reason the suite doesn't run
+	if (!run_all_suites) {
+		auto end_suites = suites.cend();
+		for (auto& suite_name : names_to_run) {
+			if (suites.find(suite_name) == end_suites) {
+				assert(false);
+				throw invalid_option_value{ std::string{"test suite "} +suite_name + " not defined" };
+			}
+		}
+	}
+
+	//run all suites or only the suites indicated in options
+	auto size = suites.size();
+	auto begin_names_to_run = names_to_run.cbegin(), end_names_to_run = names_to_run.cend();
+	for (const auto& suite : suites) {
+		const auto& key = suite.first;
+		if (run_all_suites || std::find(begin_names_to_run, end_names_to_run, key) != end_names_to_run) {
+			start_suite(key);
+			suite.second();
+			if (size > 1 && options.summary)
+				summarize_suite();
+		}
+	}
 }
 
 
 //the following functions assume they are called only from main so that the parameters are always correct
 //assertion and error handling are included by design
 
-static void show_error_and_usage(const char* message, const char* program_path)
+static int show_error_and_usage(const char* message, const char* program_path, error_code ec)
 {
-	show_error(message);
+	show_error(message, ec);
 	std::cout << '\n';
 	show_usage(program_path);
+
+	return static_cast<int>(ec);
 }
 
 
-static void show_error(const char* message)
+static int show_error(const char* message, error_code ec)
 {
-	std::cout << message << '\n';
+	int code{ static_cast<int>(ec) };
+	std::cout << "Error " << code << "; " << message << '\n';
+	return code;
 }
 
 
 static void show_usage(const char* program_path)
 {
-	std::string program_filename = std::filesystem::path{ program_path }.filename().string();
+	auto program_filename = std::filesystem::path{ program_path }.filename().string();
 
 	std::cout << "Usage: " << program_filename << " {option_name option_value}\n\n";
 
@@ -104,14 +167,15 @@ static void show_usage(const char* program_path)
 		"Angle brackets are placeholders for option values:\n\n";
 
 	std::cout <<
-		"  -h  <yes, no>\n"
-		"  -ht <header text>\n"
-		"  -s  <yes, no>\n"
-		"  -p  <detail, indicate, none, auto>\n"
-		"  -t  <fail threshold>\n"
-		"  -fn <output file path>\n"
-		"  -fo <output file path>\n"
-		"  -fa <output file path>\n"
+		"  -h   <yes, no>\n"
+		"  -ht  <header text>\n"
+		"  -s   <yes, no>\n"
+		"  -p   <detail, indicate, none, auto>\n"
+		"  -t   <fail threshold>\n"
+		"  -run <semi-colon separated list of suite names>\n"
+		"  -fn  <output file path>\n"
+		"  -fo  <output file path>\n"
+		"  -fa  <output file path>\n"
 		"\n";
 
 	std::cout <<

test/options-exceptions.h

@@ -19,23 +19,16 @@
 #include <string>
 #include <filesystem>
 
-static std::string message(const std::string_view& base, const std::string& extra = "")
-{
-	std::string msg{ base };
-	if (!extra.empty())
-		msg += ": " + extra;
-
-	return msg;
-}
+#include "utils.h"
 
 //base class for cmd-line errors: no public ctors to force use of a specialized error
 class cmd_line_error : public std::runtime_error {
 
 protected:
-	cmd_line_error(const std::string_view& base) : std::runtime_error{ message(base) } {}
+	cmd_line_error(const std::string_view& base) : std::runtime_error{ format_message(base) } {}
 
 	cmd_line_error(const std::string_view& base, const std::string& details) :
-		std::runtime_error{ message(base, details) },
+		std::runtime_error{ format_message(base, details) },
 		details_{ details } {}
 
 	const std::string& details() const noexcept
@@ -118,10 +111,10 @@ class invalid_option_value : public cmd_line_error {
 class file_error : public std::runtime_error {
 
 public:
-	file_error(const std::string& base) : std::runtime_error{ message(base) } {}
+	file_error(const std::string& base) : std::runtime_error{ format_message(base) } {}
 
 	file_error(const std::string& base, const std::filesystem::path& filepath) :
-		std::runtime_error{ message(base, filepath.string()) },
+		std::runtime_error{ format_message(base, filepath.string()) },
 		filepath_{ filepath } {}
 
 	const std::filesystem::path& filepath() const noexcept