Skip to content
This repository was archived by the owner on Oct 28, 2021. It is now read-only.

Commit 9c4db26

Browse files
gumb0halfalicious
authored andcommitted
Support RocksDB as a database backend
Add support for RocksDB to both Aleth and TestEth. RocksDB is included via the Hunter package manager and is compiled into Aleth and TestEth. One can select RocksDB at runtime via the '--db' command-line argument. Note that Aleth still defaults to LevelDB and TestEth still defaults to MemoryDB. The plan is to perform more testing on RocksDB and then switch it on as default (and remove LevelDB support).
1 parent ec558b2 commit 9c4db26

File tree

11 files changed

+304
-25
lines changed

11 files changed

+304
-25
lines changed

doc/database_layout.rst

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ Database Layout
22
===============
33

44
Aleth uses three databases, all of them are essentially just
5-
key-value storages (LevelDB or RocksDB is used depending on build
6-
settings). Their physical disk locations are as follows:
5+
key-value storages (MemoryDB, LevelDB, or RocksDB is used depending on
6+
the runtime settings). Their physical disk locations are as follows (note
7+
that this doesn't apply to MemoryDB):
78

89
- Blocks - ``{ETH_DATABASE_DIR}/{GENESIS_HASH}/blocks``
910
- Extras -

libdevcore/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,7 @@ target_link_libraries(devcore PUBLIC aleth-buildinfo Boost::filesystem Boost::sy
1111
find_package(LevelDB)
1212
target_include_directories(devcore SYSTEM PUBLIC ${LEVELDB_INCLUDE_DIRS})
1313
target_link_libraries(devcore PRIVATE ${LEVELDB_LIBRARIES})
14+
15+
hunter_add_package(rocksdb)
16+
find_package(RocksDB CONFIG REQUIRED)
17+
target_link_libraries(devcore PRIVATE RocksDB::rocksdb)

libdevcore/DBFactory.cpp

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "DBFactory.h"
1919
#include "FileSystem.h"
2020
#include "LevelDB.h"
21+
#include "RocksDB.h"
2122
#include "MemoryDB.h"
2223
#include "libethcore/Exceptions.h"
2324

@@ -47,6 +48,7 @@ struct DBKindTableEntry
4748
/// so linear search only to parse command line arguments is not a problem.
4849
DBKindTableEntry dbKindsTable[] = {
4950
{DatabaseKind::LevelDB, "leveldb"},
51+
{DatabaseKind::RocksDB, "rocksdb"},
5052
{DatabaseKind::MemoryDB, "memorydb"},
5153
};
5254

@@ -75,9 +77,16 @@ void setDatabasePath(std::string const& _path)
7577
g_dbPath = fs::path(_path);
7678
}
7779

78-
bool isMemoryDB()
80+
bool isDiskDatabase()
7981
{
80-
return g_kind == DatabaseKind::MemoryDB;
82+
switch (g_kind)
83+
{
84+
case DatabaseKind::LevelDB:
85+
case DatabaseKind::RocksDB:
86+
return true;
87+
default:
88+
return false;
89+
}
8190
}
8291

8392
DatabaseKind databaseKind()
@@ -145,6 +154,9 @@ std::unique_ptr<DatabaseFace> DBFactory::create(DatabaseKind _kind, fs::path con
145154
case DatabaseKind::LevelDB:
146155
return std::unique_ptr<DatabaseFace>(new LevelDB(_path));
147156
break;
157+
case DatabaseKind::RocksDB:
158+
return std::unique_ptr<DatabaseFace>(new RocksDB(_path));
159+
break;
148160
case DatabaseKind::MemoryDB:
149161
// Silently ignore path since the concept of a db path doesn't make sense
150162
// when using an in-memory database

libdevcore/DBFactory.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ namespace db
3030
enum class DatabaseKind
3131
{
3232
LevelDB,
33+
RocksDB,
3334
MemoryDB
3435
};
3536

@@ -40,7 +41,7 @@ enum class DatabaseKind
4041
boost::program_options::options_description databaseProgramOptions(
4142
unsigned _lineLength = boost::program_options::options_description::m_default_line_length);
4243

43-
bool isMemoryDB();
44+
bool isDiskDatabase();
4445
DatabaseKind databaseKind();
4546
void setDatabaseKindByName(std::string const& _name);
4647
void setDatabaseKind(DatabaseKind _kind);

libdevcore/RocksDB.cpp

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
/*
2+
This file is part of cpp-ethereum.
3+
4+
cpp-ethereum is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU General Public License as published by
6+
the Free Software Foundation, either version 3 of the License, or
7+
(at your option) any later version.
8+
9+
cpp-ethereum is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU General Public License for more details.
13+
14+
You should have received a copy of the GNU General Public License
15+
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
18+
#include "RocksDB.h"
19+
#include "Assertions.h"
20+
21+
namespace dev
22+
{
23+
namespace db
24+
{
25+
namespace
26+
{
27+
28+
using errinfo_rocksdbStatusCode = boost::error_info<struct tag_rocksdbStatusCode, rocksdb::Status::Code>;
29+
using errinfo_rocksdbStatusSubCode = boost::error_info<struct tag_rocksdbStatusSubCode, rocksdb::Status::SubCode>;
30+
31+
DatabaseStatus toDatabaseStatus(rocksdb::Status const& _status)
32+
{
33+
switch (_status.code())
34+
{
35+
case rocksdb::Status::kOk:
36+
return DatabaseStatus::Ok;
37+
case rocksdb::Status::kNotFound:
38+
return DatabaseStatus::NotFound;
39+
case rocksdb::Status::kCorruption:
40+
return DatabaseStatus::Corruption;
41+
case rocksdb::Status::kNotSupported:
42+
return DatabaseStatus::NotSupported;
43+
case rocksdb::Status::kInvalidArgument:
44+
return DatabaseStatus::InvalidArgument;
45+
case rocksdb::Status::kIOError:
46+
return DatabaseStatus::IOError;
47+
default:
48+
return DatabaseStatus::Unknown;
49+
}
50+
}
51+
52+
void checkStatus(rocksdb::Status const& _status, boost::filesystem::path const& _path = {})
53+
{
54+
if (_status.ok())
55+
return;
56+
57+
DatabaseError ex;
58+
ex << errinfo_dbStatusCode(toDatabaseStatus(_status))
59+
<< errinfo_rocksdbStatusCode(_status.code())
60+
<< errinfo_rocksdbStatusSubCode(_status.subcode())
61+
<< errinfo_dbStatusString(_status.ToString());
62+
if (!_path.empty())
63+
ex << errinfo_path(_path.string());
64+
65+
BOOST_THROW_EXCEPTION(ex);
66+
}
67+
68+
class RocksDBWriteBatch : public WriteBatchFace
69+
{
70+
public:
71+
void insert(Slice _key, Slice _value) override;
72+
void kill(Slice _key) override;
73+
74+
rocksdb::WriteBatch const& writeBatch() const { return m_writeBatch; }
75+
rocksdb::WriteBatch& writeBatch() { return m_writeBatch; }
76+
77+
private:
78+
rocksdb::WriteBatch m_writeBatch;
79+
};
80+
81+
void RocksDBWriteBatch::insert(Slice _key, Slice _value)
82+
{
83+
auto const status = m_writeBatch.Put(
84+
rocksdb::Slice(_key.data(), _key.size()),
85+
rocksdb::Slice(_value.data(), _value.size())
86+
);
87+
checkStatus(status);
88+
}
89+
90+
void RocksDBWriteBatch::kill(Slice _key)
91+
{
92+
auto const status = m_writeBatch.Delete(rocksdb::Slice(_key.data(), _key.size()));
93+
checkStatus(status);
94+
}
95+
96+
} // namespace
97+
98+
rocksdb::ReadOptions RocksDB::defaultReadOptions()
99+
{
100+
return rocksdb::ReadOptions();
101+
}
102+
103+
rocksdb::WriteOptions RocksDB::defaultWriteOptions()
104+
{
105+
return rocksdb::WriteOptions();
106+
}
107+
108+
rocksdb::Options RocksDB::defaultDBOptions()
109+
{
110+
rocksdb::Options options;
111+
options.create_if_missing = true;
112+
options.max_open_files = 256;
113+
return options;
114+
}
115+
116+
RocksDB::RocksDB(boost::filesystem::path const& _path, rocksdb::ReadOptions _readOptions,
117+
rocksdb::WriteOptions _writeOptions, rocksdb::Options _dbOptions)
118+
: m_db(nullptr), m_readOptions(std::move(_readOptions)), m_writeOptions(std::move(_writeOptions))
119+
{
120+
auto db = static_cast<rocksdb::DB*>(nullptr);
121+
auto const status = rocksdb::DB::Open(_dbOptions, _path.string(), &db);
122+
checkStatus(status, _path);
123+
124+
assert(db);
125+
m_db.reset(db);
126+
}
127+
128+
std::string RocksDB::lookup(Slice _key) const
129+
{
130+
rocksdb::Slice const key(_key.data(), _key.size());
131+
std::string value;
132+
auto const status = m_db->Get(m_readOptions, key, &value);
133+
if (status.IsNotFound())
134+
return std::string();
135+
136+
checkStatus(status);
137+
return value;
138+
}
139+
140+
bool RocksDB::exists(Slice _key) const
141+
{
142+
std::string value;
143+
rocksdb::Slice const key(_key.data(), _key.size());
144+
if (!m_db->KeyMayExist(m_readOptions, key, &value, nullptr))
145+
return false;
146+
147+
auto const status = m_db->Get(m_readOptions, key, &value);
148+
if (status.IsNotFound())
149+
return false;
150+
151+
checkStatus(status);
152+
return true;
153+
}
154+
155+
void RocksDB::insert(Slice _key, Slice _value)
156+
{
157+
rocksdb::Slice const key(_key.data(), _key.size());
158+
rocksdb::Slice const value(_value.data(), _value.size());
159+
auto const status = m_db->Put(m_writeOptions, key, value);
160+
checkStatus(status);
161+
}
162+
163+
void RocksDB::kill(Slice _key)
164+
{
165+
rocksdb::Slice const key(_key.data(), _key.size());
166+
auto const status = m_db->Delete(m_writeOptions, key);
167+
checkStatus(status);
168+
}
169+
170+
std::unique_ptr<WriteBatchFace> RocksDB::createWriteBatch() const
171+
{
172+
return std::unique_ptr<WriteBatchFace>(new RocksDBWriteBatch());
173+
}
174+
175+
void RocksDB::commit(std::unique_ptr<WriteBatchFace> _batch)
176+
{
177+
if (!_batch)
178+
BOOST_THROW_EXCEPTION(DatabaseError() << errinfo_comment("Cannot commit null batch"));
179+
180+
auto* batchPtr = dynamic_cast<RocksDBWriteBatch*>(_batch.get());
181+
if (!batchPtr)
182+
BOOST_THROW_EXCEPTION(DatabaseError() << errinfo_comment("Invalid batch type passed to rocksdb::commit"));
183+
184+
auto const status = m_db->Write(m_writeOptions, &batchPtr->writeBatch());
185+
checkStatus(status);
186+
}
187+
188+
void RocksDB::forEach(std::function<bool(Slice, Slice)> f) const
189+
{
190+
std::unique_ptr<rocksdb::Iterator> itr(m_db->NewIterator(m_readOptions));
191+
if (itr == nullptr)
192+
BOOST_THROW_EXCEPTION(DatabaseError() << errinfo_comment("null iterator"));
193+
194+
auto keepIterating = true;
195+
for (itr->SeekToFirst(); keepIterating && itr->Valid(); itr->Next())
196+
{
197+
auto const dbKey = itr->key();
198+
auto const dbValue = itr->value();
199+
Slice const key(dbKey.data(), dbKey.size());
200+
Slice const value(dbValue.data(), dbValue.size());
201+
keepIterating = f(key, value);
202+
}
203+
}
204+
205+
} // namespace db
206+
} // namespace dev

libdevcore/RocksDB.h

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
This file is part of cpp-ethereum.
3+
4+
cpp-ethereum is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU General Public License as published by
6+
the Free Software Foundation, either version 3 of the License, or
7+
(at your option) any later version.
8+
9+
cpp-ethereum is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU General Public License for more details.
13+
14+
You should have received a copy of the GNU General Public License
15+
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
18+
#pragma once
19+
20+
#include "db.h"
21+
22+
#include <boost/filesystem.hpp>
23+
#include <rocksdb/db.h>
24+
#include <rocksdb/write_batch.h>
25+
26+
namespace dev
27+
{
28+
namespace db
29+
{
30+
class RocksDB : public DatabaseFace
31+
{
32+
public:
33+
static rocksdb::ReadOptions defaultReadOptions();
34+
static rocksdb::WriteOptions defaultWriteOptions();
35+
static rocksdb::Options defaultDBOptions();
36+
37+
explicit RocksDB(boost::filesystem::path const& _path,
38+
rocksdb::ReadOptions _readOptions = defaultReadOptions(),
39+
rocksdb::WriteOptions _writeOptions = defaultWriteOptions(),
40+
rocksdb::Options _dbOptions = defaultDBOptions());
41+
42+
std::string lookup(Slice _key) const override;
43+
bool exists(Slice _key) const override;
44+
void insert(Slice _key, Slice _value) override;
45+
void kill(Slice _key) override;
46+
47+
std::unique_ptr<WriteBatchFace> createWriteBatch() const override;
48+
void commit(std::unique_ptr<WriteBatchFace> _batch) override;
49+
50+
void forEach(std::function<bool(Slice, Slice)> f) const override;
51+
52+
private:
53+
std::unique_ptr<rocksdb::DB> m_db;
54+
rocksdb::ReadOptions const m_readOptions;
55+
rocksdb::WriteOptions const m_writeOptions;
56+
};
57+
58+
} // namespace db
59+
} // namespace dev

libethereum/BlockChain.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ unsigned BlockChain::open(fs::path const& _path, WithExisting _we)
216216
fs::path extrasPath = chainPath / fs::path(toString(c_databaseVersion));
217217
unsigned lastMinor = c_minorProtocolVersion;
218218

219-
if (!db::isMemoryDB())
219+
if (db::isDiskDatabase())
220220
{
221221
fs::create_directories(extrasPath);
222222
DEV_IGNORE_EXCEPTIONS(fs::permissions(extrasPath, fs::owner_all));
@@ -253,7 +253,7 @@ unsigned BlockChain::open(fs::path const& _path, WithExisting _we)
253253
if (*boost::get_error_info<db::errinfo_dbStatusCode>(ex) != db::DatabaseStatus::IOError)
254254
throw;
255255

256-
if (!db::isMemoryDB())
256+
if (db::isDiskDatabase())
257257
{
258258
if (fs::space(chainPath / fs::path("blocks")).available < 1024)
259259
{
@@ -340,7 +340,7 @@ void BlockChain::close()
340340

341341
void BlockChain::rebuild(fs::path const& _path, std::function<void(unsigned, unsigned)> const& _progress)
342342
{
343-
if (db::isMemoryDB())
343+
if (!db::isDiskDatabase())
344344
{
345345
cwarn <<"In-memory database detected, skipping rebuild (since there's no existing database to rebuild)";
346346
return;

0 commit comments

Comments
 (0)