From 16acc1d6b116d1557fccace7aef184fffc7f9e87 Mon Sep 17 00:00:00 2001 From: Joe Amenta Date: Tue, 12 Jan 2021 09:57:08 -0500 Subject: [PATCH 1/2] Update Cursively benchmark code. - VisitPartialFieldContents is required for correctness - Use the original byte array, since we have one, instead of forcing a stream around it - Set iteration time to 1 second, to work around dotnet/BenchmarkDotNet#837 while we wait for a version of this package that includes dotnet/BenchmarkDotNet#1573 --- benchmark.cmd | 2 +- source/CsvBenchmark/CsvReaderBenchmarks.cs | 51 ++++++++++++++++++---- source/CsvBenchmark/TestData.cs | 5 +++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/benchmark.cmd b/benchmark.cmd index c1bb5c7..11744d3 100644 --- a/benchmark.cmd +++ b/benchmark.cmd @@ -1,2 +1,2 @@ dotnet build -c Release source/CsvBenchmark.sln -bin\release\net5.0\CsvBenchmark.exe +bin\release\net5.0\CsvBenchmark.exe --iterationTime 1000 diff --git a/source/CsvBenchmark/CsvReaderBenchmarks.cs b/source/CsvBenchmark/CsvReaderBenchmarks.cs index e77e816..44eb7d9 100644 --- a/source/CsvBenchmark/CsvReaderBenchmarks.cs +++ b/source/CsvBenchmark/CsvReaderBenchmarks.cs @@ -201,12 +201,29 @@ public void MgholamFastCSV() class CursivelyStringVisitor : CsvReaderVisitorBase { + readonly bool doPooling; + readonly byte[] bytes = new byte[1024]; + int bytesUsed = 0; + // in any realistic scenario we'd need to at least know the column oridnal to do anything with the record int ordinal = 0; + public CursivelyStringVisitor(bool doPooling) + { + this.doPooling = doPooling; + } + public override void VisitEndOfField(System.ReadOnlySpan chunk) { - var str = Encoding.UTF8.GetString(chunk); + if (bytesUsed != 0) + { + chunk.CopyTo(bytes.AsSpan(bytesUsed, chunk.Length)); + chunk = new ReadOnlySpan(bytes, 0, bytesUsed + chunk.Length); + bytesUsed = 0; + } + var str = chunk.Length == 1 && chunk[0] < 128 && doPooling + ? pool[chunk[0]] + : Encoding.UTF8.GetString(chunk); ordinal++; } @@ -217,17 +234,20 @@ public override void VisitEndOfRecord() public override void VisitPartialFieldContents(System.ReadOnlySpan chunk) { + chunk.CopyTo(bytes.AsSpan(bytesUsed, chunk.Length)); + bytesUsed += chunk.Length; } } [Benchmark] - public void CursivelyCsv() + [Arguments(false)] + [Arguments(true)] + public void CursivelyCsv(bool doPooling) { - var s = TestData.GetUtf8Stream(); - var proc = new CursivelyStringVisitor(); + var d = TestData.GetUtf8Array(); + var proc = new CursivelyStringVisitor(doPooling); CsvSyncInput - .ForStream(s) - .WithMinReadBufferByteCount(BufferSize) + .ForMemory(d) .Process(proc); } @@ -352,6 +372,9 @@ public void NRecoSelect() class CursivelySelectVisitor : CsvReaderVisitorBase { + readonly byte[] bytes = new byte[1024]; + int bytesUsed = 0; + int ordinal = 0; int row = 0; @@ -361,6 +384,12 @@ class CursivelySelectVisitor : CsvReaderVisitorBase public override void VisitEndOfField(ReadOnlySpan chunk) { + if (bytesUsed != 0) + { + chunk.CopyTo(bytes.AsSpan(bytesUsed, chunk.Length)); + chunk = new ReadOnlySpan(bytes, 0, bytesUsed + chunk.Length); + bytesUsed = 0; + } if (row != 0) // skip the header row { switch (ordinal) @@ -398,17 +427,21 @@ public override void VisitEndOfRecord() public override void VisitPartialFieldContents(System.ReadOnlySpan chunk) { + if (row > 0) + { + chunk.CopyTo(bytes.AsSpan(bytesUsed, chunk.Length)); + bytesUsed += chunk.Length; + } } } [Benchmark] public void CursivelyCsvSelect() { - var s = TestData.GetUtf8Stream(); + var d = TestData.GetUtf8Array(); var proc = new CursivelySelectVisitor(); CsvSyncInput - .ForStream(s) - .WithMinReadBufferByteCount(BufferSize) + .ForMemory(d) .Process(proc); } diff --git a/source/CsvBenchmark/TestData.cs b/source/CsvBenchmark/TestData.cs index 4d95996..94cbd26 100644 --- a/source/CsvBenchmark/TestData.cs +++ b/source/CsvBenchmark/TestData.cs @@ -112,6 +112,11 @@ public static Stream GetUtf8Stream() return new MemoryStream(CachedUtfData); } + public static ReadOnlyMemory GetUtf8Array() + { + return CachedUtfData; + } + public static DbDataReader GetData() { From e7d0c2b6d720a9712982b407ce8e18bc743b5acf Mon Sep 17 00:00:00 2001 From: Joe Amenta Date: Tue, 12 Jan 2021 11:00:47 -0500 Subject: [PATCH 2/2] Change the order of this check Whether or not this is the optimal order is almost certain to depend on the data: data sets that would almost never use the pool would likely hurt more by doing it this way, especially if this were a real-world application since this would likely push a line out of the CPU cache for no reason. Then again, a real-world application probably wouldn't use the pool conditionally like this, so there's not much of a reason NOT to either. --- source/CsvBenchmark/CsvReaderBenchmarks.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/CsvBenchmark/CsvReaderBenchmarks.cs b/source/CsvBenchmark/CsvReaderBenchmarks.cs index 44eb7d9..7e20b79 100644 --- a/source/CsvBenchmark/CsvReaderBenchmarks.cs +++ b/source/CsvBenchmark/CsvReaderBenchmarks.cs @@ -221,7 +221,7 @@ public override void VisitEndOfField(System.ReadOnlySpan chunk) chunk = new ReadOnlySpan(bytes, 0, bytesUsed + chunk.Length); bytesUsed = 0; } - var str = chunk.Length == 1 && chunk[0] < 128 && doPooling + var str = doPooling && chunk.Length == 1 && chunk[0] < 128 ? pool[chunk[0]] : Encoding.UTF8.GetString(chunk); ordinal++;