From 1e976f0ae5599c0e51f4a458ae915e8689d706e3 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 16:34:51 +0300 Subject: [PATCH 1/6] fix 1 --- .../Controllers/Unity3dController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs index fdb8e3b4ab..5b6a82dcae 100644 --- a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs +++ b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs @@ -140,7 +140,7 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address) foreach (List txList in blocks.Select(x => x.Transactions)) { - foreach (Transaction transaction in txList.Where(x => !x.IsCoinBase && !x.IsCoinStake)) + foreach (Transaction transaction in txList) { for (int i = 0; i < transaction.Outputs.Count; i++) { From d2fbf89657a30b0f090831c5611f55a8c3775b20 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 21:59:07 +0300 Subject: [PATCH 2/6] disable compaction --- .../AddressIndexing/AddressIndexer.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index f9a3239f00..206041395b 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -140,6 +140,9 @@ public class AddressIndexer : IAddressIndexer /// public const int SyncBuffer = 50; + // Compaction removes info that is used by unity sdk to get UTXOs. Therefore this feature is disabled. + public const bool CompactionEnabled = false; + public IFullNodeFeature InitializingFeature { get; set; } public AddressIndexer(StoreSettings storeSettings, DataFolder dataFolder, ILoggerFactory loggerFactory, Network network, INodeStats nodeStats, @@ -393,7 +396,7 @@ private void SaveAll() private void AddInlineStats(StringBuilder benchLog) { benchLog.AppendLine("AddressIndexer Height".PadRight(LoggingConfiguration.ColumnLength) + $": {this.IndexerTip.Height}".PadRight(9) + - "AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) + + " AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) + "OutPointCache%: " + this.outpointsRepository.GetLoadPercentage().ToString().PadRight(8) + $"Ms/block: {Math.Round(this.averageTimePerBlock.Average, 2)}"); } @@ -433,8 +436,8 @@ private bool ProcessBlock(Block block, ChainedHeader header) // Process inputs. var inputs = new List(); - // Collect all inputs excluding coinbases. - foreach (TxInList inputsCollection in block.Transactions.Where(x => !x.IsCoinBase).Select(x => x.Inputs)) + // Collect all inputs. + foreach (TxInList inputsCollection in block.Transactions.Select(x => x.Inputs)) inputs.AddRange(inputsCollection); lock (this.lockObject) @@ -445,6 +448,10 @@ private bool ProcessBlock(Block block, ChainedHeader header) { OutPoint consumedOutput = input.PrevOut; + // Ignore coinbase. + if (consumedOutput.Hash == uint256.Zero) + continue; + if (!this.outpointsRepository.TryGetOutPointData(consumedOutput, out OutPointData consumedOutputData)) { this.logger.LogError("Missing outpoint data for {0}.", consumedOutput); @@ -506,7 +513,7 @@ private bool ProcessBlock(Block block, ChainedHeader header) } // Remove outpoints that were consumed. - foreach (OutPoint consumedOutPoint in inputs.Select(x => x.PrevOut)) + foreach (OutPoint consumedOutPoint in inputs.Where(x => x.PrevOut.Hash != uint256.Zero).Select(x => x.PrevOut)) this.outpointsRepository.RemoveOutPointData(consumedOutPoint); } @@ -534,7 +541,7 @@ private void ProcessBalanceChangeLocked(int height, string address, Money amount // Anything less than that should be compacted. int heightThreshold = this.consensusManager.Tip.Height - this.compactionTriggerDistance; - bool compact = (indexData.BalanceChanges.Count > CompactingThreshold) && + bool compact = CompactionEnabled && (indexData.BalanceChanges.Count > CompactingThreshold) && (indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold); if (!compact) From a518960313d355cfdc25566534d2462024974bc5 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 21:59:19 +0300 Subject: [PATCH 3/6] rarer rewind purge loging --- .../AddressIndexing/AddressIndexerOutpointsRepository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs index 73b7f93f31..8a342213af 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs @@ -124,7 +124,7 @@ public void PurgeOldRewindData(int height) { this.addressIndexerRewindData.Delete(itemsToPurge[i].BlockHash); - if (i % 100 == 0) + if (i % 500 == 0) this.logger.LogInformation("Purging {0}/{1} rewind data items.", i, itemsToPurge.Count()); } } From cd958f863f52b594f0fd5cdc1e56a7395b25c4d7 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 21:59:50 +0300 Subject: [PATCH 4/6] Show error if state is incorrect --- .../Controllers/Unity3dController.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs index 5b6a82dcae..55f96ee2db 100644 --- a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs +++ b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs @@ -121,6 +121,7 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address) AddressIndexerData addressBalances = balancesResult.BalancesData.First(); List deposits = addressBalances.BalanceChanges.Where(x => x.Deposited).ToList(); + long totalDeposited = deposits.Sum(x => x.Satoshi); long totalWithdrawn = addressBalances.BalanceChanges.Where(x => !x.Deposited).Sum(x => x.Satoshi); @@ -160,6 +161,8 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address) UTXOs = new List() }; + Money totalM = Money.Zero; + foreach (KeyValuePair unspentOutput in fetchCoinsResponse.UnspentOutputs) { if (unspentOutput.Value.Coins == null) @@ -168,9 +171,14 @@ public GetUTXOsResponseModel GetUTXOsForAddress([FromQuery] string address) OutPoint outPoint = unspentOutput.Key; Money value = unspentOutput.Value.Coins.TxOut.Value; + totalM += value; + response.UTXOs.Add(new UTXOModel(outPoint, value)); } + if (totalM != balanceSat) + this.logger.LogError(string.Format("Should be {0}, is: {1}", new Money(balanceSat), totalM)); + return response; } From 6368ec435abe64428f5f9af444a9c77073ad2a09 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 23:19:04 +0300 Subject: [PATCH 5/6] increase purge performance --- .../AddressIndexing/AddressIndexer.cs | 2 +- .../AddressIndexerOutpointsRepository.cs | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index 206041395b..0810801baa 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -119,7 +119,7 @@ public class AddressIndexer : IAddressIndexer private DateTime lastFlushTime; - private const int PurgeIntervalSeconds = 60; + private const int PurgeIntervalSeconds = 120; /// Last time rewind data was purged. private DateTime lastPurgeTime; diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs index 8a342213af..72d55c5b6b 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs @@ -118,15 +118,8 @@ public void PurgeOldRewindData(int height) { lock (this.LockObject) { - var itemsToPurge = this.addressIndexerRewindData.Find(x => x.BlockHeight < height).ToArray(); - - for (int i = 0; i < itemsToPurge.Count(); i++) - { - this.addressIndexerRewindData.Delete(itemsToPurge[i].BlockHash); - - if (i % 500 == 0) - this.logger.LogInformation("Purging {0}/{1} rewind data items.", i, itemsToPurge.Count()); - } + int purgedCount = this.addressIndexerRewindData.Delete(x => x.BlockHeight < height); + this.logger.LogInformation("Purged {0} rewind data items.", purgedCount); } } From 2486ced7ccad3e97d09b514f6b7c2bf9a87738d3 Mon Sep 17 00:00:00 2001 From: noescape00 Date: Mon, 29 Nov 2021 23:34:33 +0300 Subject: [PATCH 6/6] upgraded litedb and incresed purging efficency --- .../Stratis.Benchmark.csproj | 2 +- .../AddressIndexerOutpointsRepositoryTests.cs | 3 +-- .../AddressIndexerTests.cs | 19 ++++++------------- .../AddressIndexing/AddressIndexRepository.cs | 2 +- .../AddressIndexing/AddressIndexer.cs | 8 +++----- .../AddressIndexerOutpointsRepository.cs | 9 +++++---- src/Stratis.Bitcoin/Stratis.Bitcoin.csproj | 2 +- .../NFTTransferIndexer.cs | 8 +++----- 8 files changed, 21 insertions(+), 32 deletions(-) diff --git a/src/Stratis.Benchmark/Stratis.Benchmark.csproj b/src/Stratis.Benchmark/Stratis.Benchmark.csproj index a9fd7d2b55..470aae6450 100644 --- a/src/Stratis.Benchmark/Stratis.Benchmark.csproj +++ b/src/Stratis.Benchmark/Stratis.Benchmark.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs index 26e37e5e27..340fc5ecd6 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs @@ -20,8 +20,7 @@ public class AddressIndexerOutpointsRepositoryTests public AddressIndexerOutpointsRepositoryTests() { - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb", Mode = fileMode }); + var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb" }); this.repository = new AddressIndexerOutpointsRepository(db, new ExtendedLoggerFactory(), this.maxItems); } diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs index 3cad2fc9f9..6ff7f1d123 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs @@ -16,7 +16,6 @@ using Stratis.Bitcoin.Tests.Common; using Stratis.Bitcoin.Utilities; using Xunit; -using FileMode = LiteDB.FileMode; using Script = NBitcoin.Script; namespace Stratis.Bitcoin.Features.BlockStore.Tests @@ -179,9 +178,8 @@ public void OutPointCacheCanRetrieveExisting() const string CollectionName = "DummyCollection"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); var outPoint = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 0); @@ -202,9 +200,8 @@ public void OutPointCacheCannotRetrieveNonexistent() const string CollectionName = "DummyCollection"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); Assert.False(cache.TryGetOutPointData(new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1), out OutPointData retrieved)); @@ -217,9 +214,8 @@ public void OutPointCacheEvicts() const string CollectionName = "OutputsData"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory(), 2); Assert.Equal(0, cache.Count); @@ -268,9 +264,8 @@ public void AddressCacheCanRetrieveExisting() const string CollectionName = "DummyCollection"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); string address = "xyz"; @@ -297,9 +292,8 @@ public void AddressCacheRetrievesBlankRecordForNonexistent() const string CollectionName = "DummyCollection"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz"); @@ -316,9 +310,8 @@ public void AddressCacheEvicts() const string CollectionName = "AddrData"; var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory(), 4); // Recall, each index entry counts as 1 and each balance change associated with it is an additional 1. diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs index a202e8e4e6..c210457d22 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs @@ -13,7 +13,7 @@ public class AddressIndexRepository : MemorySizeCache addressIndexerDataCollection; + private readonly ILiteCollection addressIndexerDataCollection; private readonly ILogger logger; diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index 0810801baa..19a2143c04 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -20,7 +20,6 @@ using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Primitives; using Stratis.Bitcoin.Utilities; -using FileMode = LiteDB.FileMode; using Script = NBitcoin.Script; namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing @@ -90,7 +89,7 @@ public class AddressIndexer : IAddressIndexer private LiteDatabase db; - private LiteCollection tipDataStore; + private ILiteCollection tipDataStore; /// A mapping between addresses and their balance changes. /// All access should be protected by . @@ -192,9 +191,8 @@ public void Initialize() } string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename); - - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + + this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath }); this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs index 72d55c5b6b..2878fc0a05 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs @@ -17,11 +17,11 @@ public sealed class AddressIndexerOutpointsRepository : MemoryCacheRepresents the output collection. /// Should be protected by - private readonly LiteCollection addressIndexerOutPointData; + private readonly ILiteCollection addressIndexerOutPointData; /// Represents the rewind data collection. /// Should be protected by - private readonly LiteCollection addressIndexerRewindData; + private readonly ILiteCollection addressIndexerRewindData; private readonly ILogger logger; @@ -118,8 +118,9 @@ public void PurgeOldRewindData(int height) { lock (this.LockObject) { - int purgedCount = this.addressIndexerRewindData.Delete(x => x.BlockHeight < height); - this.logger.LogInformation("Purged {0} rewind data items.", purgedCount); + this.logger.LogInformation("AddressIndexer: started purging rewind data items."); + int purgedCount = this.addressIndexerRewindData.DeleteMany(x => x.BlockHeight < height); + this.logger.LogInformation("AddressIndexer: Purged {0} rewind data items.", purgedCount); } } diff --git a/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj b/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj index 3d27cfac79..d4a727d9e4 100644 --- a/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj +++ b/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj @@ -27,7 +27,7 @@ - + diff --git a/src/Stratis.Features.Unity3dApi/NFTTransferIndexer.cs b/src/Stratis.Features.Unity3dApi/NFTTransferIndexer.cs index 83d8413b64..d010151414 100644 --- a/src/Stratis.Features.Unity3dApi/NFTTransferIndexer.cs +++ b/src/Stratis.Features.Unity3dApi/NFTTransferIndexer.cs @@ -14,7 +14,6 @@ using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.SmartContracts.Models; using Stratis.Bitcoin.Features.SmartContracts.Wallet; -using FileMode = LiteDB.FileMode; namespace Stratis.Features.Unity3dApi { @@ -52,7 +51,7 @@ public class NFTTransferIndexer : INFTTransferIndexer private readonly ISmartContractTransactionService smartContractTransactionService; private LiteDatabase db; - private LiteCollection NFTContractCollection; + private ILiteCollection NFTContractCollection; private CancellationTokenSource cancellation; private Task indexingTask; @@ -74,9 +73,8 @@ public void Initialize() throw new Exception("NFTTransferIndexer already initialized!"); string dbPath = Path.Combine(this.dataFolder.RootPath, DatabaseFilename); - - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + + this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath }); this.NFTContractCollection = this.db.GetCollection(DbOwnedNFTsKey); this.indexingTask = Task.Run(async () => await this.IndexNFTsContinuouslyAsync().ConfigureAwait(false));