testeth print test suite suggestions

Author: Dimitry

test/CMakeLists.txt

@@ -53,6 +53,7 @@ list(REMOVE_ITEM sources ${unittest_sources})
 # search for test names and create ctest tests
 set(excludeSuites jsonrpc \"customTestSuite\" BlockQueueSuite)
 set(allSuites jsonrpc)
+set(allTests "")
 foreach(file ${sources})
     file(STRINGS ${file} test_list_raw REGEX "BOOST_.*TEST_(SUITE|CASE|SUITE_END)")
     set(TestSuite "DEFAULT")
@@ -81,6 +82,7 @@ foreach(file ${sources})
                     set(TestEthArgs -t ${TestSuitePathFixed} -- ${TESTETH_ARGS})
                     add_test(NAME ${TestSuitePathFixed} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND testeth ${TestEthArgs})
                     set_tests_properties(${TestSuitePathFixed} PROPERTIES TIMEOUT ${timeout})
+                    set(allTests "${allTests}   \"${TestSuitePathFixed}\",\n")
                 endif()
             endif()
         elseif(test MATCHES "^CASE .*")
@@ -92,6 +94,11 @@ foreach(file ${sources})
                     set(TestEthArgs -t ${TestSuitePathFixed}/${TestCase} -- ${TESTETH_ARGS})
                     add_test(NAME ${TestSuitePathFixed}/${TestCase} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND testeth ${TestEthArgs})
                     set_tests_properties(${TestSuitePathFixed}/${TestCase} PROPERTIES TIMEOUT ${timeout})
+                    set(allTests "${allTests}   \"${TestSuitePathFixed}/${TestCase}\",\n")
+                    if (NOT ";${allSuites};" MATCHES ";${TestSuitePathFixed};")
+                        list(APPEND allSuites ${TestSuitePathFixed})
+                        set(allTests "${allTests}   \"${TestSuitePathFixed}\",\n")
+                    endif()
                 endif()
             endif()
         elseif (";${test_raw};" MATCHES "BOOST_AUTO_TEST_SUITE_END()")
@@ -102,10 +109,14 @@ foreach(file ${sources})
     endforeach(test_raw)
 endforeach(file)
 
+#Save allTests into a variable in include file
+configure_file(tools/libtesteth/cmakeAllTests.h.in tools/libtesteth/cmakeAllTests.h)
+
 hunter_add_package(yaml-cpp)
 find_package(yaml-cpp CONFIG REQUIRED)
 
 add_executable(testeth ${sources})
+target_include_directories(testeth PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_include_directories(testeth PRIVATE ${UTILS_INCLUDE_DIR})
 target_link_libraries(testeth PRIVATE ethereum ethashseal web3jsonrpc devcrypto devcore aleth-buildinfo cryptopp-static yaml-cpp::yaml-cpp binaryen::binaryen libjson-rpc-cpp::client)
 install(TARGETS testeth DESTINATION ${CMAKE_INSTALL_BINDIR})

test/tools/libtesteth/TestHelper.cpp

@@ -637,6 +637,36 @@ string prepareLLLCVersionString()
     return "Error getting LLLC Version";
 }
 
+// A simple C++ implementation of the Levenshtein distance algorithm to measure the amount of
+// difference between two strings. https://gist.github.com/TheRayTracer/2644387
+size_t levenshteinDistance(char const* _s, size_t _n, char const* _t, size_t _m)
+{
+    ++_n;
+    ++_m;
+    size_t* d = new size_t[_n * _m];
+
+    memset(d, 0, sizeof(size_t) * _n * _m);
+    for (size_t i = 1, im = 0; i < _m; ++i, ++im)
+    {
+        for (size_t j = 1, jn = 0; j < _n; ++j, ++jn)
+        {
+            if (_s[jn] == _t[im])
+                d[(i * _n) + j] = d[((i - 1) * _n) + (j - 1)];
+            else
+            {
+                d[(i * _n) + j] = min(d[(i - 1) * _n + j] + 1, /* A deletion. */
+                    min(d[i * _n + (j - 1)] + 1,               /* An insertion. */
+                        d[(i - 1) * _n + (j - 1)] + 1));       /* A substitution. */
+            }
+        }
+    }
+
+    size_t r = d[_n * _m - 1];
+    delete[] d;
+    return r;
+}
+
+
 void copyFile(fs::path const& _source, fs::path const& _destination)
 {
     fs::ifstream src(_source, ios::binary);

test/tools/libtesteth/TestHelper.h

@@ -105,6 +105,7 @@ bytes importByteArray(std::string const& _str);
 void requireJsonFields(json_spirit::mObject const& _o, std::string const& _section,
     std::map<std::string, json_spirit::Value_type> const& _validationMap);
 void checkHexHasEvenLength(std::string const&);
+size_t levenshteinDistance(char const* _s, size_t _n, char const* _t, size_t _m);
 void copyFile(boost::filesystem::path const& _source, boost::filesystem::path const& _destination);
 eth::LogEntries importLog(json_spirit::mArray const& _o);
 std::string exportLog(eth::LogEntries const& _logs);

test/tools/libtesteth/boostTest.cpp

@@ -16,7 +16,9 @@
 #include <test/tools/jsontests/TransactionTests.h>
 #include <test/tools/jsontests/vm.h>
 #include <test/tools/libtesteth/TestHelper.h>
+#include <test/tools/libtesteth/cmakeAllTests.h>
 #include <boost/test/included/unit_test.hpp>
+#include <boost/tokenizer.hpp>
 #include <clocale>
 #include <cstdlib>
 #include <iostream>
@@ -81,6 +83,17 @@ int main(int argc, const char* argv[])
     std::string const dynamicTestSuiteName = "customTestSuite";
     setDefaultOrCLocale();
 
+    string sMinusTArg;
+    for (int i = 0; i < argc; i++)  // find -t boost arg
+    {
+        std::string const arg = std::string{argv[i]};
+        if (arg == "-t" && i + 1 < argc)
+        {
+            sMinusTArg = std::string{argv[i + 1]};
+            break;
+        }
+    }
+
     // Initialize options
     try
     {
@@ -137,6 +150,34 @@ int main(int argc, const char* argv[])
     int result = 0;
     auto fakeInit = [](int, char* []) -> boost::unit_test::test_suite* { return nullptr; };
     result = unit_test_main(fakeInit, argc, const_cast<char**>(argv));
+
+    // Print suggestions of a test case
+    if (result == boost::exit_exception_failure)  // test suite not found
+    {
+        size_t allTestsElementIndex = 0;
+        typedef std::pair<size_t, size_t> NameDistance;  // <index in availableTests, compared
+                                                         // distance>
+
+        // Use `vector` here because `set` does not work with sort
+        std::vector<NameDistance> distanceMap;  // <test name, compared distance>
+        for (auto& it : c_allTests)
+        {
+            int dist =
+                test::levenshteinDistance(sMinusTArg.c_str(), sMinusTArg.size(), it, strlen(it));
+            distanceMap.emplace_back(allTestsElementIndex++, dist);
+        }
+        std::sort(distanceMap.begin(), distanceMap.end(),
+            [](NameDistance const& _a, NameDistance const& _b) { return _a.second < _b.second; });
+        std::cerr << "Did you mean: \n";
+        std::set<string> suggestionSet;
+        for (size_t i = 0; i < 3 && i < distanceMap.size(); i++)
+        {
+            string name = c_allTests[distanceMap[i].first];
+            suggestionSet.emplace(name);
+        }
+        for (auto const& suggestion : suggestionSet)
+            std::cerr << "-t " << suggestion << "\n";
+    }
     dev::test::TestOutputHelper::get().printTestExecStats();
     return result;
 }

test/tools/libtesteth/cmakeAllTests.h.in

@@ -0,0 +1,6 @@
+#pragma once
+
+char const* c_allTests[] = 
+{
+@allTests@
+};