Skip to content

AddressIndexer configured for unitysdk #788

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
noescape00 merged 6 commits into from
Nov 29, 2021
Merged

AddressIndexer configured for unitysdk #788

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1e976f0
fix 1
noescape00 Nov 29, 2021
d2fbf89
disable compaction
noescape00 Nov 29, 2021
a518960
rarer rewind purge loging
noescape00 Nov 29, 2021
cd958f8
Show error if state is incorrect
noescape00 Nov 29, 2021
6368ec4
increase purge performance
noescape00 Nov 29, 2021
2486ced
upgraded litedb and incresed purging efficency
noescape00 Nov 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Stratis.Benchmark/Stratis.Benchmark.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
<PackageReference Include="FASTER" Version="2019.3.16.1" />
<PackageReference Include="LiteDB" Version="4.1.4" />
<PackageReference Include="LiteDB" Version="5.0.11" />
</ItemGroup>

<ItemGroup>
Expand Down
3 changes: 1 addition & 2 deletions src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ public class AddressIndexerOutpointsRepositoryTests

public AddressIndexerOutpointsRepositoryTests()
{
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb", Mode = fileMode });
var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb" });

this.repository = new AddressIndexerOutpointsRepository(db, new ExtendedLoggerFactory(), this.maxItems);
}
Expand Down
19 changes: 6 additions & 13 deletions src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
using Stratis.Bitcoin.Tests.Common;
using Stratis.Bitcoin.Utilities;
using Xunit;
using FileMode = LiteDB.FileMode;
using Script = NBitcoin.Script;

namespace Stratis.Bitcoin.Features.BlockStore.Tests
Expand Down Expand Up @@ -179,9 +178,8 @@ public void OutPointCacheCanRetrieveExisting()
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory());

var outPoint = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 0);
Expand All @@ -202,9 +200,8 @@ public void OutPointCacheCannotRetrieveNonexistent()
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory());

Assert.False(cache.TryGetOutPointData(new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1), out OutPointData retrieved));
Expand All @@ -217,9 +214,8 @@ public void OutPointCacheEvicts()
const string CollectionName = "OutputsData";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory(), 2);

Assert.Equal(0, cache.Count);
Expand Down Expand Up @@ -268,9 +264,8 @@ public void AddressCacheCanRetrieveExisting()
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory());

string address = "xyz";
Expand All @@ -297,9 +292,8 @@ public void AddressCacheRetrievesBlankRecordForNonexistent()
const string CollectionName = "DummyCollection";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory());

AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz");
Expand All @@ -316,9 +310,8 @@ public void AddressCacheEvicts()
const string CollectionName = "AddrData";
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));
string dbPath = Path.Combine(dataFolder.RootPath, CollectionName);
FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;

var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
var database = new LiteDatabase(new ConnectionString() { Filename = dbPath });
var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory(), 4);

// Recall, each index entry counts as 1 and each balance change associated with it is an additional 1.
Expand Down
2 changes: 1 addition & 1 deletion src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public class AddressIndexRepository : MemorySizeCache<string, AddressIndexerData
{
private const string DbAddressDataKey = "AddrData";

private readonly LiteCollection<AddressIndexerData> addressIndexerDataCollection;
private readonly ILiteCollection<AddressIndexerData> addressIndexerDataCollection;

private readonly ILogger logger;

Expand Down
27 changes: 16 additions & 11 deletions src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Primitives;
using Stratis.Bitcoin.Utilities;
using FileMode = LiteDB.FileMode;
using Script = NBitcoin.Script;

namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing
Expand Down Expand Up @@ -90,7 +89,7 @@ public class AddressIndexer : IAddressIndexer

private LiteDatabase db;

private LiteCollection<AddressIndexerTipData> tipDataStore;
private ILiteCollection<AddressIndexerTipData> tipDataStore;

/// <summary>A mapping between addresses and their balance changes.</summary>
/// <remarks>All access should be protected by <see cref="lockObject"/>.</remarks>
Expand Down Expand Up @@ -119,7 +118,7 @@ public class AddressIndexer : IAddressIndexer

private DateTime lastFlushTime;

private const int PurgeIntervalSeconds = 60;
private const int PurgeIntervalSeconds = 120;

/// <summary>Last time rewind data was purged.</summary>
private DateTime lastPurgeTime;
Expand All @@ -140,6 +139,9 @@ public class AddressIndexer : IAddressIndexer
/// </summary>
public const int SyncBuffer = 50;

// Compaction removes info that is used by unity sdk to get UTXOs. Therefore this feature is disabled.
public const bool CompactionEnabled = false;
Copy link
Collaborator

@zeptin zeptin Nov 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should only be false when the Unity API is being used, to preserve the previous behaviour and performance characteristics of the indexer.

Copy link
Contributor Author

@noescape00 noescape00 Nov 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With compaction off behavior is the same, performance is increased, but hard drive usage is increased as well.
With compaction on cirrus test it's around 1.1gb. Currently syncing indexer, will know hard drive impact once it's done.

Copy link
Contributor Author

@noescape00 noescape00 Nov 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should only be false when the Unity API is being used

address indexer is used only for unity api as far as I know.

Also once it was on it can't be turned off because it requires reindexing.

So in case compaction feature is not cosnt bool then data if it was enabled before should be persisted to the database.
I think this can be done in a separate PR once we need addr indexer for smth not unity related.


public IFullNodeFeature InitializingFeature { get; set; }

public AddressIndexer(StoreSettings storeSettings, DataFolder dataFolder, ILoggerFactory loggerFactory, Network network, INodeStats nodeStats,
Expand Down Expand Up @@ -189,9 +191,8 @@ public void Initialize()
}

string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename);

FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });

this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath });

this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory);

Expand Down Expand Up @@ -393,7 +394,7 @@ private void SaveAll()
private void AddInlineStats(StringBuilder benchLog)
{
benchLog.AppendLine("AddressIndexer Height".PadRight(LoggingConfiguration.ColumnLength) + $": {this.IndexerTip.Height}".PadRight(9) +
"AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) +
" AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) +
"OutPointCache%: " + this.outpointsRepository.GetLoadPercentage().ToString().PadRight(8) +
$"Ms/block: {Math.Round(this.averageTimePerBlock.Average, 2)}");
}
Expand Down Expand Up @@ -433,8 +434,8 @@ private bool ProcessBlock(Block block, ChainedHeader header)
// Process inputs.
var inputs = new List<TxIn>();

// Collect all inputs excluding coinbases.
foreach (TxInList inputsCollection in block.Transactions.Where(x => !x.IsCoinBase).Select(x => x.Inputs))
// Collect all inputs.
foreach (TxInList inputsCollection in block.Transactions.Select(x => x.Inputs))
inputs.AddRange(inputsCollection);

lock (this.lockObject)
Expand All @@ -445,6 +446,10 @@ private bool ProcessBlock(Block block, ChainedHeader header)
{
OutPoint consumedOutput = input.PrevOut;

// Ignore coinbase.
if (consumedOutput.Hash == uint256.Zero)
continue;

if (!this.outpointsRepository.TryGetOutPointData(consumedOutput, out OutPointData consumedOutputData))
{
this.logger.LogError("Missing outpoint data for {0}.", consumedOutput);
Expand Down Expand Up @@ -506,7 +511,7 @@ private bool ProcessBlock(Block block, ChainedHeader header)
}

// Remove outpoints that were consumed.
foreach (OutPoint consumedOutPoint in inputs.Select(x => x.PrevOut))
foreach (OutPoint consumedOutPoint in inputs.Where(x => x.PrevOut.Hash != uint256.Zero).Select(x => x.PrevOut))
this.outpointsRepository.RemoveOutPointData(consumedOutPoint);
}

Expand Down Expand Up @@ -534,7 +539,7 @@ private void ProcessBalanceChangeLocked(int height, string address, Money amount
// Anything less than that should be compacted.
int heightThreshold = this.consensusManager.Tip.Height - this.compactionTriggerDistance;

bool compact = (indexData.BalanceChanges.Count > CompactingThreshold) &&
bool compact = CompactionEnabled && (indexData.BalanceChanges.Count > CompactingThreshold) &&
(indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold);

if (!compact)
Expand Down
16 changes: 5 additions & 11 deletions src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ public sealed class AddressIndexerOutpointsRepository : MemoryCache<string, OutP

/// <summary>Represents the output collection.</summary>
/// <remarks>Should be protected by <see cref="LockObject"/></remarks>
private readonly LiteCollection<OutPointData> addressIndexerOutPointData;
private readonly ILiteCollection<OutPointData> addressIndexerOutPointData;

/// <summary>Represents the rewind data collection.</summary>
/// <remarks>Should be protected by <see cref="LockObject"/></remarks>
private readonly LiteCollection<AddressIndexerRewindData> addressIndexerRewindData;
private readonly ILiteCollection<AddressIndexerRewindData> addressIndexerRewindData;

private readonly ILogger logger;

Expand Down Expand Up @@ -118,15 +118,9 @@ public void PurgeOldRewindData(int height)
{
lock (this.LockObject)
{
var itemsToPurge = this.addressIndexerRewindData.Find(x => x.BlockHeight < height).ToArray();

for (int i = 0; i < itemsToPurge.Count(); i++)
{
this.addressIndexerRewindData.Delete(itemsToPurge[i].BlockHash);

if (i % 100 == 0)
this.logger.LogInformation("Purging {0}/{1} rewind data items.", i, itemsToPurge.Count());
}
this.logger.LogInformation("AddressIndexer: started purging rewind data items.");
int purgedCount = this.addressIndexerRewindData.DeleteMany(x => x.BlockHeight < height);
this.logger.LogInformation("AddressIndexer: Purged {0} rewind data items.", purgedCount);
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/Stratis.Bitcoin/Stratis.Bitcoin.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<ItemGroup>
<PackageReference Include="ConcurrentHashSet" Version="1.0.2" />
<PackageReference Include="DBreeze" Version="1.89.0" />
<PackageReference Include="LiteDB" Version="4.1.4" />
<PackageReference Include="LiteDB" Version="5.0.11" />
<PackageReference Include="LevelDB.Standard" Version="2.1.6.1" />
<PackageReference Include="Microsoft.AspNetCore.Diagnostics" Version="2.1.1" />
<PackageReference Include="Microsoft.AspNetCore.Hosting" Version="2.1.1" />
Expand Down
10 changes: 9 additions & 1 deletion src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address)
AddressIndexerData addressBalances = balancesResult.BalancesData.First();

List<AddressBalanceChange> deposits = addressBalances.BalanceChanges.Where(x => x.Deposited).ToList();

long totalDeposited = deposits.Sum(x => x.Satoshi);
long totalWithdrawn = addressBalances.BalanceChanges.Where(x => !x.Deposited).Sum(x => x.Satoshi);

Expand All @@ -140,7 +141,7 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address)

foreach (List<Transaction> txList in blocks.Select(x => x.Transactions))
{
foreach (Transaction transaction in txList.Where(x => !x.IsCoinBase && !x.IsCoinStake))
foreach (Transaction transaction in txList)
{
for (int i = 0; i < transaction.Outputs.Count; i++)
{
Expand All @@ -160,6 +161,8 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address)
UTXOs = new List<UTXOModel>()
};

Money totalM = Money.Zero;

foreach (KeyValuePair<OutPoint, UnspentOutput> unspentOutput in fetchCoinsResponse.UnspentOutputs)
{
if (unspentOutput.Value.Coins == null)
Expand All @@ -168,9 +171,14 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address)
OutPoint outPoint = unspentOutput.Key;
Money value = unspentOutput.Value.Coins.TxOut.Value;

totalM += value;

response.UTXOs.Add(new UTXOModel(outPoint, value));
}

if (totalM != balanceSat)
this.logger.LogError(string.Format("Should be {0}, is: {1}", new Money(balanceSat), totalM));

return response;
}

Expand Down
8 changes: 3 additions & 5 deletions src/Stratis.Features.Unity3dApi/NFTTransferIndexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
using Stratis.Bitcoin.Features.BlockStore.AddressIndexing;
using Stratis.Bitcoin.Features.SmartContracts.Models;
using Stratis.Bitcoin.Features.SmartContracts.Wallet;
using FileMode = LiteDB.FileMode;

namespace Stratis.Features.Unity3dApi
{
Expand Down Expand Up @@ -52,7 +51,7 @@ public class NFTTransferIndexer : INFTTransferIndexer
private readonly ISmartContractTransactionService smartContractTransactionService;

private LiteDatabase db;
private LiteCollection<NFTContractModel> NFTContractCollection;
private ILiteCollection<NFTContractModel> NFTContractCollection;
private CancellationTokenSource cancellation;
private Task indexingTask;

Expand All @@ -74,9 +73,8 @@ public void Initialize()
throw new Exception("NFTTransferIndexer already initialized!");

string dbPath = Path.Combine(this.dataFolder.RootPath, DatabaseFilename);

FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });

this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath });
this.NFTContractCollection = this.db.GetCollection<NFTContractModel>(DbOwnedNFTsKey);

this.indexingTask = Task.Run(async () => await this.IndexNFTsContinuouslyAsync().ConfigureAwait(false));
Expand Down