Skip to content

Commit 857eb53

Browse files
quantumagiquantumagi
authored
Combine KeyValueStore implementations (stratisproject#1029)
* Combine KeyValueStore implementations * Update tests
1 parent 42a2066 commit 857eb53

File tree

12 files changed

+283
-163
lines changed

12 files changed

+283
-163
lines changed

‎src/Stratis.Bitcoin.Tests/Base/TipsManagerTests.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using Microsoft.Extensions.Logging;
55
using NBitcoin;
66
using Stratis.Bitcoin.Base;
7+
using Stratis.Bitcoin.Database;
78
using Stratis.Bitcoin.Persistence.KeyValueStores;
89
using Stratis.Bitcoin.Tests.Common;
910
using Stratis.Bitcoin.Utilities;
@@ -14,7 +15,7 @@ namespace Stratis.Bitcoin.Tests.Base
1415
public class TipsManagerTests : TestBase
1516
{
1617
private readonly LoggerFactory loggerFactory;
17-
private readonly LevelDbKeyValueRepository keyValueRepo;
18+
private readonly KeyValueRepository<LevelDb> keyValueRepo;
1819
private readonly ITipsManager tipsManager;
1920

2021
private readonly List<ChainedHeader> mainChainHeaders;
@@ -23,7 +24,7 @@ public TipsManagerTests() : base(KnownNetworks.StraxMain)
2324
{
2425
this.loggerFactory = new LoggerFactory();
2526
string dir = CreateTestDir(this);
26-
this.keyValueRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
27+
this.keyValueRepo = new KeyValueRepository<LevelDb>(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
2728

2829
this.tipsManager = new TipsManager(this.keyValueRepo, this.loggerFactory);
2930

‎src/Stratis.Bitcoin.Tests/Consensus/FinalizedBlockInfoRepositoryTest.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using NBitcoin;
55
using Stratis.Bitcoin.AsyncWork;
66
using Stratis.Bitcoin.Consensus;
7+
using Stratis.Bitcoin.Database;
78
using Stratis.Bitcoin.Persistence.KeyValueStores;
89
using Stratis.Bitcoin.Tests.Common;
910
using Stratis.Bitcoin.Utilities;
@@ -24,7 +25,7 @@ public FinalizedBlockInfoRepositoryTest() : base(KnownNetworks.StraxRegTest)
2425
public async Task FinalizedHeightSavedOnDiskAsync()
2526
{
2627
string dir = CreateTestDir(this);
27-
var kvRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
28+
var kvRepo = new KeyValueRepository<LevelDb>(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
2829
var asyncMock = new Mock<IAsyncProvider>();
2930
asyncMock.Setup(a => a.RegisterTask(It.IsAny<string>(), It.IsAny<Task>()));
3031

@@ -45,7 +46,7 @@ public async Task FinalizedHeightSavedOnDiskAsync()
4546
public async Task FinalizedHeightCantBeDecreasedAsync()
4647
{
4748
string dir = CreateTestDir(this);
48-
var kvRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
49+
var kvRepo = new KeyValueRepository<LevelDb>(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory));
4950
var asyncMock = new Mock<IAsyncProvider>();
5051
asyncMock.Setup(a => a.RegisterTask(It.IsAny<string>(), It.IsAny<Task>()));
5152

‎src/Stratis.Bitcoin/Base/BaseFeature.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -461,13 +461,13 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil
461461
if (dbType == DbType.Leveldb)
462462
{
463463
chainStore = new ChainStore<LevelDb>(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer);
464-
services.AddSingleton<IKeyValueRepository, LevelDbKeyValueRepository>();
464+
services.AddSingleton<IKeyValueRepository, KeyValueRepository<LevelDb>>();
465465
}
466466

467467
if (dbType == DbType.RocksDb)
468468
{
469469
chainStore = new ChainStore<RocksDb>(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer);
470-
services.AddSingleton<IKeyValueRepository, RocksDbKeyValueRepository>();
470+
services.AddSingleton<IKeyValueRepository, KeyValueRepository<RocksDb>>();
471471
}
472472

473473
chainIndexer[0].SetChainStore(chainStore);

‎src/Stratis.Bitcoin/Database/IDb.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,26 @@ public interface IDb : IDisposable
2525
/// <returns>The value for the specified table and key.</returns>
2626
byte[] Get(byte table, byte[] key);
2727

28+
/// <summary>
29+
/// Gets the value associated with a key.
30+
/// </summary>
31+
/// <param name="key">The key of the value to retrieve.</param>
32+
/// <returns>The value for the specified key.</returns>
33+
byte[] Get(byte[] key);
34+
2835
/// <summary>
2936
/// Gets an iterator that allows iteration over keys in a table.
3037
/// </summary>
3138
/// <param name="table">The table that will be iterated.</param>
3239
/// <returns>See <see cref="IDbIterator"/>.</returns>
3340
IDbIterator GetIterator(byte table);
3441

42+
/// <summary>
43+
/// Gets an iterator that allows iteration over keys.
44+
/// </summary>
45+
/// <returns>See <see cref="IDbIterator"/>.</returns>
46+
IDbIterator GetIterator();
47+
3548
/// <summary>
3649
/// Gets a batch that can be used to record changes that can be applied atomically.
3750
/// </summary>
@@ -69,6 +82,21 @@ public interface IDbBatch : IDisposable
6982
/// <returns>This class for fluent operations.</returns>
7083
IDbBatch Delete(byte table, byte[] key);
7184

85+
/// <summary>
86+
/// Records a value that will be written to the database when the <see cref="Write"/> method is invoked.
87+
/// </summary>
88+
/// <param name="key">The table key that identifies the value to be updated.</param>
89+
/// <param name="value">The value to be written to the table.</param>
90+
/// <returns>This class for fluent operations.</returns>
91+
IDbBatch Put(byte[] key, byte[] value);
92+
93+
/// <summary>
94+
/// Records a key that will be deleted from the database when the <see cref="Write"/> method is invoked.
95+
/// </summary>
96+
/// <param name="key">The table key that will be removed.</param>
97+
/// <returns>This class for fluent operations.</returns>
98+
IDbBatch Delete(byte[] key);
99+
72100
/// <summary>
73101
/// Writes the recorded changes to the database.
74102
/// </summary>

‎src/Stratis.Bitcoin/Database/IDbIteratorExt.cs‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using NBitcoin;
4+
5+
namespace Stratis.Bitcoin.Database
6+
{
7+
/// <summary>
8+
/// Extension methods that build on the <see cref="IDbIterator"/> interface.
9+
/// </summary>
10+
public static class IDbIteratorExt
11+
{
12+
private static ByteArrayComparer byteArrayComparer = new ByteArrayComparer();
13+
14+
/// <summary>
15+
/// Gets all the keys in the relevant table subject to any supplied constraints.
16+
/// </summary>
17+
/// <param name="iterator">The iterator that also identifies the table being iterated.</param>
18+
/// <param name="keysOnly">Defaults to <c>false</c>. Set to <c>true</c> if values should be ommitted - i.e. set to <c>null</c>.</param>
19+
/// <param name="ascending">Defaults to <c>true</c>. Set to <c>false</c> to return keys in ascending order.</param>
20+
/// <param name="firstKey">Can be set optionally to specify the lower bound of keys to return.</param>
21+
/// <param name="lastKey">Can be set optionally to specify the upper bound of keys to return.</param>
22+
/// <param name="includeFirstKey">Defaults to <c>true</c>. Set to <c>false</c> to omit the key specified in <paramref name="firstKey"/>.</param>
23+
/// <param name="includeLastKey">Defaults to <c>true</c>. Set to <c>false</c> to omit the key specified in <paramref name="lastKey"/>.</param>
24+
/// <returns>An enumeration containing all the keys and values according to the specified constraints.</returns>
25+
public static IEnumerable<(byte[], byte[])> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true,
26+
byte[] firstKey = null, byte[] lastKey = null, bool includeFirstKey = true, bool includeLastKey = true)
27+
{
28+
bool done = false;
29+
Func<byte[], bool> breakLoop;
30+
Action next;
31+
32+
if (!ascending)
33+
{
34+
// Seek to the last key if it was provided.
35+
if (lastKey == null)
36+
iterator.SeekToLast();
37+
else
38+
{
39+
iterator.Seek(lastKey);
40+
if (iterator.IsValid())
41+
{
42+
if (!(includeLastKey && byteArrayComparer.Equals(iterator.Key(), lastKey)))
43+
iterator.Prev();
44+
}
45+
else
46+
iterator.SeekToLast();
47+
}
48+
49+
breakLoop = (firstKey == null) ? (Func<byte[], bool>)null : (keyBytes) =>
50+
{
51+
int compareResult = byteArrayComparer.Compare(keyBytes, firstKey);
52+
if (compareResult <= 0)
53+
{
54+
// If this is the first key and its not included or we've overshot the range then stop without yielding a value.
55+
if (!includeFirstKey || compareResult < 0)
56+
return true;
57+
58+
// Stop after yielding the value.
59+
done = true;
60+
}
61+
62+
// Keep going.
63+
return false;
64+
};
65+
66+
next = () => iterator.Prev();
67+
}
68+
else /* Ascending */
69+
{
70+
// Seek to the first key if it was provided.
71+
if (firstKey == null)
72+
iterator.Seek(new byte[0]);
73+
else
74+
{
75+
iterator.Seek(firstKey);
76+
if (iterator.IsValid())
77+
{
78+
if (!(includeFirstKey && byteArrayComparer.Equals(iterator.Key(), firstKey)))
79+
iterator.Next();
80+
}
81+
}
82+
83+
breakLoop = (lastKey == null) ? (Func<byte[], bool>)null : (keyBytes) =>
84+
{
85+
int compareResult = byteArrayComparer.Compare(keyBytes, lastKey);
86+
if (compareResult >= 0)
87+
{
88+
// If this is the last key and its not included or we've overshot the range then stop without yielding a value.
89+
if (!includeLastKey || compareResult > 0)
90+
return true;
91+
92+
// Stop after yielding the value.
93+
done = true;
94+
}
95+
96+
// Keep going.
97+
return false;
98+
};
99+
100+
next = () => iterator.Next();
101+
}
102+
103+
while (iterator.IsValid())
104+
{
105+
byte[] keyBytes = iterator.Key();
106+
107+
if (breakLoop != null && breakLoop(keyBytes))
108+
break;
109+
110+
yield return (keyBytes, keysOnly ? null : iterator.Value());
111+
112+
if (done)
113+
break;
114+
115+
next();
116+
}
117+
}
118+
}
119+
}

‎src/Stratis.Bitcoin/Database/LevelDb.cs‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ public IDbIterator GetIterator(byte table)
1515
return new LevelDbIterator(table, this.db.CreateIterator());
1616
}
1717

18+
public IDbIterator GetIterator()
19+
{
20+
return new LevelDbIterator(this.db.CreateIterator());
21+
}
22+
1823
public void Open(string dbPath)
1924
{
2025
this.dbPath = dbPath;
@@ -35,6 +40,11 @@ public byte[] Get(byte table, byte[] key)
3540
return this.db.Get(new[] { table }.Concat(key).ToArray());
3641
}
3742

43+
public byte[] Get(byte[] key)
44+
{
45+
return this.db.Get(key);
46+
}
47+
3848
public void Dispose()
3949
{
4050
this.db.Dispose();
@@ -51,14 +61,30 @@ public LevelDbBatch(DB db)
5161
this.db = db;
5262
}
5363

64+
// Methods when using tables.
65+
5466
public IDbBatch Put(byte table, byte[] key, byte[] value)
5567
{
56-
return (IDbBatch)this.Put(new[] { table }.Concat(key).ToArray(), value);
68+
return this.Put(new[] { table }.Concat(key).ToArray(), value);
5769
}
5870

5971
public IDbBatch Delete(byte table, byte[] key)
6072
{
61-
return (IDbBatch)this.Delete(new[] { table }.Concat(key).ToArray());
73+
return this.Delete(new[] { table }.Concat(key).ToArray());
74+
}
75+
76+
// Table-less operations.
77+
78+
public new IDbBatch Put(byte[] key, byte[] value)
79+
{
80+
base.Put(key, value);
81+
return this;
82+
}
83+
84+
public new IDbBatch Delete(byte[] key)
85+
{
86+
base.Delete(key);
87+
return this;
6288
}
6389

6490
public void Write()
@@ -70,7 +96,7 @@ public void Write()
7096
/// <summary>A minimal LevelDb wrapper that makes it compliant with the <see cref="IDbIterator"/> interface.</summary>
7197
public class LevelDbIterator : IDbIterator
7298
{
73-
private byte table;
99+
private byte? table;
74100
private Iterator iterator;
75101

76102
public LevelDbIterator(byte table, Iterator iterator)
@@ -79,13 +105,25 @@ public LevelDbIterator(byte table, Iterator iterator)
79105
this.iterator = iterator;
80106
}
81107

108+
// Table-less constructor.
109+
public LevelDbIterator(Iterator iterator)
110+
{
111+
this.iterator = iterator;
112+
}
113+
82114
public void Seek(byte[] key)
83115
{
84-
this.iterator.Seek(new[] { this.table }.Concat(key).ToArray());
116+
this.iterator.Seek(this.table.HasValue ? (new[] { this.table.Value }.Concat(key).ToArray()) : key);
85117
}
86118

87119
public void SeekToLast()
88120
{
121+
if (!this.table.HasValue)
122+
{
123+
this.iterator.SeekToLast();
124+
return;
125+
}
126+
89127
if (this.table != 255)
90128
{
91129
// First seek past the last record in the table by attempting to seek to the start of the next table (if any).
@@ -115,12 +153,12 @@ public void Prev()
115153

116154
public bool IsValid()
117155
{
118-
return this.iterator.IsValid() && this.iterator.Key()[0] == this.table;
156+
return this.iterator.IsValid() && (!this.table.HasValue || this.iterator.Key()[0] == this.table);
119157
}
120158

121159
public byte[] Key()
122160
{
123-
return this.iterator.Key().Skip(1).ToArray();
161+
return this.table.HasValue ? this.iterator.Key().Skip(1).ToArray() : this.iterator.Key();
124162
}
125163

126164
public byte[] Value()

0 commit comments

Comments
 (0)