Merge pull request #4114 from ethereum/bcRandom

winsvega · web-flow · commit a58dae673af5 · 2017-06-13T13:56:15.000+04:00
random blockchain tests. Parse all files in bcRandom folder.
diff --git a/test/jsontests b/test/jsontests
@@ -1 +1 @@
-Subproject commit e4041ac74c569c2cf4f83f0233df804a1a93a278
+Subproject commit dcbc5e771e9e732ce369752707939649352965bd
diff --git a/test/tools/jsontests/BlockChainTests.cpp b/test/tools/jsontests/BlockChainTests.cpp
@@ -102,16 +102,24 @@ void checkBlocks(TestBlock const& _blockFromFields, TestBlock const& _blockFromR
 
 void doBlockchainTests(json_spirit::mValue& _v, bool _fillin)
 {
-	if (!Options::get().fillchain) //fill blockchain through state tests
-		TestOutputHelper::initTest(_v);
+	TestOutputHelper::initTest(_v);	//Count how many tests in the json object (read from .json file)
+	doBlockchainTestNoLog(_v, _fillin); //Do the test / test generation
+	TestOutputHelper::finishTest(); //Calculate the time of test execution and add it to the log
+}
+
+void doBlockchainTestNoLog(json_spirit::mValue& _v, bool _fillin)
+{
 	for (auto& i: _v.get_obj())
 	{
 		string testname = i.first;
 		json_spirit::mObject& o = i.second.get_obj();
 
-		if (!Options::get().fillchain)
-		if (!TestOutputHelper::passTest(o, testname))
+		//Select test by name if --singletest is set and not filling state tests as blockchain
+		if (!TestOutputHelper::passTest(testname) && !Options::get().fillchain)
+		{
+			o.clear(); //don't add irrelevant tests to the final file when filling
 			continue;
+		}
 
 		BOOST_REQUIRE(o.count("genesisBlockHeader"));
 		BOOST_REQUIRE(o.count("pre"));
@@ -407,7 +415,6 @@ void doBlockchainTests(json_spirit::mValue& _v, bool _fillin)
 			ImportTest::compareStates(postState, blockchain.topBlock().state());
 		}
 	}//for tests
-	TestOutputHelper::finishTest();
 }
 
 //TestFunction
diff --git a/test/tools/jsontests/BlockChainTestsBoost.cpp b/test/tools/jsontests/BlockChainTestsBoost.cpp
@@ -20,7 +20,6 @@
  * BlockChain boost test cases.
  */
 
-#include <boost/filesystem/operations.hpp>
 #include <boost/test/unit_test.hpp>
 #include <boost/filesystem.hpp>
 #include <test/tools/libtesteth/TestHelper.h>
@@ -173,4 +172,26 @@ BOOST_AUTO_TEST_CASE(userDefinedFile)
 	dev::test::userDefinedTest(dev::test::doBlockchainTests);
 }
 BOOST_AUTO_TEST_SUITE_END()
+
+//BlockChainTests
+BOOST_AUTO_TEST_SUITE(BlockChainTestsRandom)
+
+BOOST_AUTO_TEST_CASE(bcRandom)
+{
+	std::string fillersPath = dev::test::getTestPath() + "/src/BlockchainTestsFiller/RandomTests";
+
+	std::vector<boost::filesystem::path> files = test::getJsonFiles(fillersPath);
+	int fileCount = files.size();
+
+	//bcRandom tests are generated from random state tests and have 1 test case * 5 forks in each file
+	dev::test::TestOutputHelper::initTest(fileCount * 5);
+
+	for (auto const& file: files)
+		dev::test::executeTests(file.filename().string(), "/BlockchainTests/RandomTests", "/BlockchainTestsFiller/RandomTests", dev::test::doBlockchainTestNoLog);
+
+	//calculate the total time on bcRandom test cases
+	dev::test::TestOutputHelper::finishTest();
+}
+BOOST_AUTO_TEST_SUITE_END()
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/test/tools/jsontests/StateTests.cpp b/test/tools/jsontests/StateTests.cpp
@@ -46,10 +46,10 @@ void doStateTests(json_spirit::mValue& _v, bool _fillin)
 		string testname = i.first;
 		json_spirit::mObject& o = i.second.get_obj();
 
-		if (!TestOutputHelper::passTest(o, testname))
+		if (!TestOutputHelper::passTest(testname))
 			continue;
 
-		//For 100% at the log output
+		//For 100% at the log output when making blockchain tests out of state tests
 		if (_fillin == false && Options::get().fillchain)
 			continue;
 
@@ -115,25 +115,19 @@ class generaltestfixture
 
 	void fillAllFilesInFolder(string _folder)
 	{
-		std::string fillersPath = dev::test::getTestPath() + "/src/GeneralStateTestsFiller/" + _folder;
-
-		boost::filesystem::directory_iterator iterator_tmp(fillersPath);
-		int fileCount = 0;
-		for(; iterator_tmp != boost::filesystem::directory_iterator(); ++iterator_tmp)
-			if (boost::filesystem::is_regular_file(iterator_tmp->path()) && iterator_tmp->path().extension() == ".json")
-				fileCount++;
-		if (dev::test::Options::get().filltests)
+		std::string fillersPath = test::getTestPath() + "/src/GeneralStateTestsFiller/" + _folder;
+
+		std::vector<boost::filesystem::path> files = test::getJsonFiles(fillersPath);
+		int fileCount = files.size();
+
+		if (test::Options::get().filltests)
 			fileCount *= 2; //tests are checked when filled and after they been filled
-		dev::test::TestOutputHelper::initTest(fileCount);
+		test::TestOutputHelper::initTest(fileCount);
 
-		boost::filesystem::directory_iterator iterator(fillersPath);
-		for(; iterator != boost::filesystem::directory_iterator(); ++iterator)
-			if (boost::filesystem::is_regular_file(iterator->path()) && iterator->path().extension() == ".json")
-			{
-				string fileboost = iterator->path().filename().string();
-				dev::test::executeTests(fileboost, "/GeneralStateTests/"+_folder, "/GeneralStateTestsFiller/"+_folder, dev::test::doStateTests);
-			}
-		dev::test::TestOutputHelper::finishTest();
+		for (auto const& file: files)
+			test::executeTests(file.filename().string(), "/GeneralStateTests/"+_folder, "/GeneralStateTestsFiller/"+_folder, dev::test::doStateTests);
+
+		test::TestOutputHelper::finishTest();
 	}
 };
 
diff --git a/test/tools/jsontests/TransactionTests.cpp b/test/tools/jsontests/TransactionTests.cpp
@@ -43,8 +43,11 @@ void doTransactionTests(json_spirit::mValue& _v, bool _fillin)
 		string testname = i.first;
 		json_spirit::mObject& o = i.second.get_obj();
 
-		if (!TestOutputHelper::passTest(o, testname))
+		if (!TestOutputHelper::passTest(testname))
+		{
+			o.clear(); //don't add irrelevant tests to the final file when filling
 			continue;
+		}
 
 		BOOST_REQUIRE(o.count("blocknumber") > 0);
 		u256 transactionBlock = toInt(o["blocknumber"].get_str());
diff --git a/test/tools/jsontests/vm.cpp b/test/tools/jsontests/vm.cpp
@@ -301,8 +301,11 @@ void doVMTests(json_spirit::mValue& _v, bool _fillin)
 		string testname = i.first;
 		json_spirit::mObject& o = i.second.get_obj();
 
-		if (!TestOutputHelper::passTest(o, testname))
+		if (!TestOutputHelper::passTest(testname))
+		{
+			o.clear(); //don't add irrelevant tests to the final file when filling
 			continue;
+		}
 
 		BOOST_REQUIRE_MESSAGE(o.count("env") > 0, testname + "env not set!");
 		BOOST_REQUIRE_MESSAGE(o.count("pre") > 0, testname + "pre not set!");
@@ -529,11 +532,7 @@ BOOST_AUTO_TEST_CASE(vmRandom)
 	string testPath = getTestPath();
 	testPath += "/VMTests/RandomTests";
 
-	vector<boost::filesystem::path> testFiles;
-	boost::filesystem::directory_iterator iterator(testPath);
-	for(; iterator != boost::filesystem::directory_iterator(); ++iterator)
-		if (boost::filesystem::is_regular_file(iterator->path()) && iterator->path().extension() == ".json")
-			testFiles.push_back(iterator->path());
+	std::vector<boost::filesystem::path> testFiles = test::getJsonFiles(testPath);
 
 	test::TestOutputHelper::initTest();
 	test::TestOutputHelper::setMaxTests(testFiles.size());
diff --git a/test/tools/libtesteth/TestHelper.cpp b/test/tools/libtesteth/TestHelper.cpp
@@ -198,6 +198,16 @@ void replaceLLLinState(json_spirit::mObject& _o)
 	}
 }
 
+std::vector<boost::filesystem::path> getJsonFiles(std::string const& _dirPath)
+{
+	vector<boost::filesystem::path> jsonFiles;
+	using Bdit = boost::filesystem::directory_iterator;
+	for (Bdit it(_dirPath); it != Bdit(); ++it)
+		if (boost::filesystem::is_regular_file(it->path()) && it->path().extension() == ".json")
+			jsonFiles.push_back(it->path());
+	return jsonFiles;
+}
+
 string compileLLL(string const& _code)
 {
 	if (_code == "")
diff --git a/test/tools/libtesteth/TestHelper.h b/test/tools/libtesteth/TestHelper.h
@@ -121,6 +121,7 @@ class ZeroGasPricer: public eth::GasPricer
 };
 
 // helping functions
+std::vector<boost::filesystem::path> getJsonFiles(std::string const& _dirPath);
 std::string netIdToString(eth::Network _netId);
 eth::Network stringToNetId(std::string const& _netname);
 u256 toInt(json_spirit::mValue const& _v);
@@ -166,6 +167,7 @@ void doTransactionTests(json_spirit::mValue& _v, bool _fillin);
 void doStateTests(json_spirit::mValue& v, bool _fillin);
 void doVMTests(json_spirit::mValue& v, bool _fillin);
 void doBlockchainTests(json_spirit::mValue& _v, bool _fillin);
+void doBlockchainTestNoLog(json_spirit::mValue& _v, bool _fillin);
 void doRlpTests(json_spirit::mValue& v, bool _fillin);
 
 
diff --git a/test/tools/libtesteth/TestOutputHelper.cpp b/test/tools/libtesteth/TestOutputHelper.cpp
@@ -61,7 +61,7 @@ void TestOutputHelper::initTest(json_spirit::mValue& _v)
 	m_currTest = 0;
 }
 
-bool TestOutputHelper::passTest(json_spirit::mObject& _o, std::string& _testName)
+bool TestOutputHelper::passTest(std::string const& _testName)
 {
 	m_currTest++;
 	int m_testsPerProgs = std::max(1, (int)(m_maxTests / 4));
@@ -75,14 +75,9 @@ bool TestOutputHelper::passTest(json_spirit::mObject& _o, std::string& _testName
 	}
 
 	if (test::Options::get().singleTest && test::Options::get().singleTestName != _testName)
-	{
-		_o.clear();
 		return false;
-	}
 
 	cnote << _testName;
-	//Test name for old State Tests
-	//_testName = (m_currentTestFileName == "n/a") ? "(" + _testName + ") " : "(" + m_currentTestFileName + "/" +  _testName + ") ";
 	m_currentTestName = _testName;
 	return true;
 }
diff --git a/test/tools/libtesteth/TestOutputHelper.h b/test/tools/libtesteth/TestOutputHelper.h
@@ -33,7 +33,7 @@ class TestOutputHelper
 	TestOutputHelper() { TestOutputHelper::initTest(); }
 	static void initTest(int _maxTests = 1);
 	static void initTest(json_spirit::mValue& _v);
-	static bool passTest(json_spirit::mObject& _o, std::string& _testName);
+	static bool passTest(std::string const& _testName);
 	static void setMaxTests(int _count) { m_maxTests = _count; }
 	static void setCurrentTestFileName(std::string _name) { m_currentTestFileName = _name; }
 	static std::string const& testName() { return m_currentTestName; }
diff --git a/test/unittests/libdevcore/rlp.cpp b/test/unittests/libdevcore/rlp.cpp
@@ -300,12 +300,7 @@ BOOST_AUTO_TEST_CASE(rlpRandom)
 	string testPath = dev::test::getTestPath();
 	testPath += "/RLPTests/RandomRLPTests";
 
-	vector<boost::filesystem::path> testFiles;
-	boost::filesystem::directory_iterator iterator(testPath);
-	for(; iterator != boost::filesystem::directory_iterator(); ++iterator)
-		if (boost::filesystem::is_regular_file(iterator->path()) && iterator->path().extension() == ".json")
-			testFiles.push_back(iterator->path());
-
+	vector<boost::filesystem::path> testFiles = getJsonFiles(testPath);
 	for (auto& path: testFiles)
 	{
 		try
diff --git a/test/unittests/libethcore/difficulty.cpp b/test/unittests/libethcore/difficulty.cpp
@@ -113,9 +113,8 @@ void fillDifficulty(string const& _testFileFullName, Ethash& _sealEngine)
 		for (u256 blockNumber = 1; blockNumber < 1500000; blockNumber += 25000)
 		{
 			testN++;
-			json_spirit::mObject m;
 			string testName = "DifficultyTest"+toString(testN);
-			if (!dev::test::TestOutputHelper::passTest(m, testName))
+			if (!dev::test::TestOutputHelper::passTest(testName))
 				continue;
 
 			u256 pStamp = dev::test::RandomCode::randomUniInt();
@@ -165,8 +164,11 @@ void testDifficulty(string const& _testFileFullName, Ethash& _sealEngine, Networ
 	{
 		js::mObject o = i.second.get_obj();
 		string testname = i.first;
-		if (!dev::test::TestOutputHelper::passTest(o, testname))
+		if (!dev::test::TestOutputHelper::passTest(testname))
+		{
+			o.clear();
 			continue;
+		}
 
 		BlockHeader parent;
 		parent.setTimestamp(test::toInt(o["parentTimestamp"]));

Original file line number	Diff line number	Diff line change
`@@ -198,6 +198,16 @@ void replaceLLLinState(json_spirit::mObject& _o)`
`198`	`198`	`}`
`199`	`199`	`}`
`200`	`200`
	`201`	`+std::vector<boost::filesystem::path> getJsonFiles(std::string const& _dirPath)`
	`202`	`+{`
	`203`	`+ vector<boost::filesystem::path> jsonFiles;`
	`204`	`+ using Bdit = boost::filesystem::directory_iterator;`
	`205`	`+ for (Bdit it(_dirPath); it != Bdit(); ++it)`
	`206`	`+ if (boost::filesystem::is_regular_file(it->path()) && it->path().extension() == ".json")`
	`207`	`+ jsonFiles.push_back(it->path());`
	`208`	`+ return jsonFiles;`
	`209`	`+}`
	`210`	`+`
`201`	`211`	`string compileLLL(string const& _code)`
`202`	`212`	`{`
`203`	`213`	`if (_code == "")`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ void TestOutputHelper::initTest(json_spirit::mValue& _v)`
`61`	`61`	`m_currTest = 0;`
`62`	`62`	`}`
`63`	`63`
`64`		`-bool TestOutputHelper::passTest(json_spirit::mObject& _o, std::string& _testName)`
	`64`	`+bool TestOutputHelper::passTest(std::string const& _testName)`
`65`	`65`	`{`
`66`	`66`	`m_currTest++;`
`67`	`67`	`int m_testsPerProgs = std::max(1, (int)(m_maxTests / 4));`
`@@ -75,14 +75,9 @@ bool TestOutputHelper::passTest(json_spirit::mObject& _o, std::string& _testName`
`75`	`75`	`}`
`76`	`76`
`77`	`77`	`if (test::Options::get().singleTest && test::Options::get().singleTestName != _testName)`
`78`		`- {`
`79`		`- _o.clear();`
`80`	`78`	`return false;`
`81`		`- }`
`82`	`79`
`83`	`80`	`cnote << _testName;`
`84`		`- //Test name for old State Tests`
`85`		`- //_testName = (m_currentTestFileName == "n/a") ? "(" + _testName + ") " : "(" + m_currentTestFileName + "/" + _testName + ") ";`
`86`	`81`	`m_currentTestName = _testName;`
`87`	`82`	`return true;`
`88`	`83`	`}`