diff --git a/src/FastSerialization/FastSerialization.cs b/src/FastSerialization/FastSerialization.cs
index 250cc9466..73eb31807 100644
--- a/src/FastSerialization/FastSerialization.cs
+++ b/src/FastSerialization/FastSerialization.cs
@@ -67,6 +67,33 @@ namespace FastSerialization
//
// This is MUCH leaner, and now dominated by actual work of copying the data to the output buffer.
+ ///
+ /// Allows users of serialization and de-serialization mechanisms to specify the size of the StreamLabel.
+ /// As traces get larger, there is a need to support larger file sizes, and thus to increase the addressable
+ /// space within the files. StreamLabel instances are 8-bytes in-memory, but all serialization and de-serialization
+ /// of them results in the upper 4-bytes being lost. This setting will allow Serializer and Deserializer to read
+ /// and write 8-byte StreamLabel instances.
+ ///
+#if FASTSERIALIZATION_PUBLIC
+ public
+#endif
+ enum StreamLabelWidth
+ {
+ FourBytes = 0,
+ EightBytes = 1
+ };
+
+ ///
+ /// These settings apply to use of Serializer and Deserializer specifically.
+ ///
+#if FASTSERIALIZATION_PUBLIC
+ public
+#endif
+ sealed class SerializationConfiguration
+ {
+ public StreamLabelWidth StreamLabelWidth { get; set; }
+ }
+
///
/// A StreamLabel represents a position in a IStreamReader or IStreamWriter.
/// In memory it is represented as a 64 bit signed value but to preserve compat
@@ -991,14 +1018,19 @@ sealed class Deserializer : IDisposable
///
/// Create a Deserializer that reads its data from a given file
///
- public Deserializer(string filePath) : this(new IOStreamStreamReader(filePath), filePath) { }
+ public Deserializer(string filePath, SerializationConfiguration config = null)
+ {
+ IOStreamStreamReader reader = new IOStreamStreamReader(filePath, config);
+ Initialize(reader, filePath);
+ }
///
/// Create a Deserializer that reads its data from a given System.IO.Stream. The stream will be closed when the Deserializer is done with it.
///
- public Deserializer(Stream inputStream, string streamName)
- : this(new IOStreamStreamReader(inputStream), streamName)
+ public Deserializer(Stream inputStream, string streamName, SerializationConfiguration config = null)
{
+ IOStreamStreamReader reader = new IOStreamStreamReader(inputStream, config: config);
+ Initialize(reader, streamName);
}
///
@@ -1006,15 +1038,21 @@ public Deserializer(Stream inputStream, string streamName)
/// parameter determines whether the deserializer will close the stream when it
/// closes.
///
- public Deserializer(Stream inputStream, string streamName, bool leaveOpen)
- : this(new IOStreamStreamReader(inputStream, leaveOpen: leaveOpen), streamName)
+ public Deserializer(Stream inputStream, string streamName, bool leaveOpen, SerializationConfiguration config = null)
{
+ IOStreamStreamReader reader = new IOStreamStreamReader(inputStream, leaveOpen: leaveOpen, config: config);
+ Initialize(reader, streamName);
}
///
/// Create a Deserializer that reads its data from a given IStreamReader. The stream will be closed when the Deserializer is done with it.
///
public Deserializer(IStreamReader reader, string streamName)
+ {
+ Initialize(reader, streamName);
+ }
+
+ private void Initialize(IStreamReader reader, string streamName)
{
ObjectsInGraph = new Dictionary();
this.reader = reader;
diff --git a/src/FastSerialization/StreamReaderWriter.cs b/src/FastSerialization/StreamReaderWriter.cs
index 80816003f..6edc853fd 100644
--- a/src/FastSerialization/StreamReaderWriter.cs
+++ b/src/FastSerialization/StreamReaderWriter.cs
@@ -24,19 +24,39 @@ class MemoryStreamReader : IStreamReader
///
public MemoryStreamReader(byte[] data) : this(data, 0, data.Length) { }
///
- /// Create a IStreamReader (reads binary data) from a given subregion of a byte buffer
+ /// Create a IStreamReader (reads binary data) from a given subregion of a byte buffer.
///
- public MemoryStreamReader(byte[] data, int start, int length)
+ public MemoryStreamReader(byte[] data, int start, int length, SerializationConfiguration config = null)
{
bytes = data;
position = start;
endPosition = length;
+ SerializationConfiguration = config != null ? config : new SerializationConfiguration();
+
+ if(SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes)
+ {
+ readLabel = () =>
+ {
+ return (StreamLabel)(uint)ReadInt32();
+ };
+ sizeOfSerializedStreamLabel = 4;
+ }
+ else
+ {
+ readLabel = () =>
+ {
+ return (StreamLabel)(ulong)ReadInt64();
+ };
+ sizeOfSerializedStreamLabel = 8;
+ }
}
+
///
/// The total length of bytes that this reader can read.
///
public virtual long Length { get { return endPosition; } }
public virtual bool HasLength { get { return true; } }
+ public SerializationConfiguration SerializationConfiguration { get; private set; }
#region implemenation of IStreamReader
public virtual void Read(byte[] data, int offset, int length)
@@ -150,7 +170,8 @@ public string ReadString()
///
public StreamLabel ReadLabel()
{
- return (StreamLabel)(uint)ReadInt32();
+ // Delegate set in the constructor based on the size of StreamLabel.
+ return readLabel();
}
///
/// Implementation of IStreamReader
@@ -176,7 +197,6 @@ public virtual StreamLabel Current
///
public virtual void GotoSuffixLabel()
{
- const int sizeOfSerializedStreamLabel = 4;
Goto((StreamLabel)(Length - sizeOfSerializedStreamLabel));
Goto(ReadLabel());
}
@@ -203,6 +223,8 @@ protected virtual void Dispose(bool disposing) { }
internal /*protected*/ int position;
internal /*protected*/ int endPosition;
private StringBuilder sb;
+ private Func readLabel;
+ private readonly int sizeOfSerializedStreamLabel;
#endregion
}
@@ -222,11 +244,31 @@ class MemoryStreamWriter : IStreamWriter
///
/// Call 'GetBytes' call to get the raw array. Only the first 'Length' bytes are valid
///
- public MemoryStreamWriter(int initialSize = 64)
+ public MemoryStreamWriter(int initialSize = 64, SerializationConfiguration config = null)
{
bytes = new byte[initialSize];
+ SerializationConfiguration = config != null ? config : new SerializationConfiguration();
+
+ if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes)
+ {
+ writeLabel = (value) =>
+ {
+ Debug.Assert((long)value <= int.MaxValue);
+ Write((int)value);
+ };
+ }
+ else
+ {
+ writeLabel = (value) =>
+ {
+ Debug.Assert((long)value <= long.MaxValue);
+ Write((long)value);
+ };
+ }
}
+ public SerializationConfiguration SerializationConfiguration { get; set; }
+
///
/// Returns a IStreamReader that will read the written bytes. You cannot write additional bytes to the stream after making this call.
///
@@ -327,8 +369,7 @@ public unsafe void Write(long value)
///
public void Write(StreamLabel value)
{
- Debug.Assert((long)value <= int.MaxValue);
- Write((int)value);
+ writeLabel(value);
}
///
/// Implementation of IStreamWriter
@@ -424,6 +465,7 @@ protected virtual void Dispose(bool disposing) { }
Array.Copy(bytes, newBytes, bytes.Length);
bytes = newBytes;
}
+ private Action writeLabel;
internal /* protected */ byte[] bytes;
internal /* protected */ int endPosition;
#endregion
@@ -597,15 +639,15 @@ class IOStreamStreamReader : MemoryStreamReader, IDisposable
/// Create a new IOStreamStreamReader from the given file.
///
///
- public IOStreamStreamReader(string fileName)
- : this(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { }
+ public IOStreamStreamReader(string fileName, SerializationConfiguration config = null)
+ : this(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete), config: config) { }
///
/// Create a new IOStreamStreamReader from the given System.IO.Stream. Optionally you can specify the size of the read buffer
/// The stream will be closed by the IOStreamStreamReader when it is closed.
///
- public IOStreamStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, bool leaveOpen = false)
- : base(new byte[bufferSize + align], 0, 0)
+ public IOStreamStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, bool leaveOpen = false, SerializationConfiguration config = null)
+ : base(new byte[bufferSize + align], 0, 0, config)
{
Debug.Assert(bufferSize % align == 0);
this.inputStream = inputStream;
@@ -839,17 +881,17 @@ sealed unsafe class PinnedStreamReader : IOStreamStreamReader
///
/// Create a new PinnedStreamReader that gets its data from a given file. You can optionally set the size of the read buffer.
///
- public PinnedStreamReader(string fileName, int bufferSize = defaultBufferSize)
+ public PinnedStreamReader(string fileName, int bufferSize = defaultBufferSize, SerializationConfiguration config = null)
: this(new FileStream(fileName, FileMode.Open, FileAccess.Read,
- FileShare.Read | FileShare.Delete), bufferSize)
+ FileShare.Read | FileShare.Delete), bufferSize, config)
{ }
///
/// Create a new PinnedStreamReader that gets its data from a given System.IO.Stream. You can optionally set the size of the read buffer.
/// The stream will be closed by the PinnedStreamReader when it is closed.
///
- public PinnedStreamReader(Stream inputStream, int bufferSize = defaultBufferSize)
- : base(inputStream, bufferSize)
+ public PinnedStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, SerializationConfiguration config = null)
+ : base(inputStream, bufferSize, config: config)
{
// Pin the array
pinningHandle = System.Runtime.InteropServices.GCHandle.Alloc(bytes, System.Runtime.InteropServices.GCHandleType.Pinned);
@@ -1016,13 +1058,13 @@ class IOStreamStreamWriter : MemoryStreamWriter, IDisposable
/// Create a IOStreamStreamWriter that writes its data to a given file that it creates
///
///
- public IOStreamStreamWriter(string fileName) : this(new FileStream(fileName, FileMode.Create)) { }
+ public IOStreamStreamWriter(string fileName, SerializationConfiguration config = null) : this(new FileStream(fileName, FileMode.Create), config: config) { }
///
/// Create a IOStreamStreamWriter that writes its data to a System.IO.Stream
///
- public IOStreamStreamWriter(Stream outputStream, int bufferSize = defaultBufferSize + sizeof(long), bool leaveOpen = false)
- : base(bufferSize)
+ public IOStreamStreamWriter(Stream outputStream, int bufferSize = defaultBufferSize + sizeof(long), bool leaveOpen = false, SerializationConfiguration config = null)
+ : base(bufferSize, config)
{
this.outputStream = outputStream;
this.leaveOpen = leaveOpen;
@@ -1071,7 +1113,7 @@ public override long Length
public override StreamLabel GetLabel()
{
long len = Length;
- if (len != (uint)len)
+ if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes && len != (uint)len)
{
throw new NotSupportedException("Streams larger than 4 GB. You need to use /MaxEventCount to limit the size.");
}
diff --git a/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs b/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs
index 01d359c55..a7d3ffa29 100644
--- a/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs
+++ b/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs
@@ -12,7 +12,26 @@ namespace TraceEventTests
public class FastSerializerTests
{
[Fact]
- public void ParseStreamLabel()
+ public void ParseEightByteStreamLabel()
+ {
+ MemoryStream ms = new MemoryStream();
+ BinaryWriter writer = new BinaryWriter(ms);
+ WriteString(writer, "!FastSerialization.1");
+ writer.Write((long)0);
+ writer.Write((long)19);
+ writer.Write((long)1_000_000);
+ writer.Write((long)0xf1234567);
+
+ ms.Position = 0;
+ Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), "name");
+ Assert.Equal((StreamLabel)0, d.ReadLabel());
+ Assert.Equal((StreamLabel)19, d.ReadLabel());
+ Assert.Equal((StreamLabel)1_000_000, d.ReadLabel());
+ Assert.Equal((StreamLabel)0xf1234567, d.ReadLabel());
+ }
+
+ [Fact]
+ public void ParseFourByteStreamLabel()
{
MemoryStream ms = new MemoryStream();
BinaryWriter writer = new BinaryWriter(ms);
@@ -23,7 +42,7 @@ public void ParseStreamLabel()
writer.Write(0xf1234567);
ms.Position = 0;
- Deserializer d = new Deserializer(new PinnedStreamReader(ms), "name");
+ Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), "name");
Assert.Equal((StreamLabel)0, d.ReadLabel());
Assert.Equal((StreamLabel)19, d.ReadLabel());
Assert.Equal((StreamLabel)1_000_000, d.ReadLabel());
@@ -35,5 +54,64 @@ private void WriteString(BinaryWriter writer, string val)
writer.Write(val.Length);
writer.Write(Encoding.UTF8.GetBytes(val));
}
+
+ [Fact]
+ public void WriteAndParseFourByteStreamLabel()
+ {
+ SampleSerializableType sample = new SampleSerializableType(SampleSerializableType.ConstantValue);
+ MemoryStream ms = new MemoryStream();
+ Serializer s = new Serializer(new IOStreamStreamWriter(ms, leaveOpen:true, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), sample);
+ s.Dispose();
+
+ Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), "name");
+ d.RegisterFactory(typeof(SampleSerializableType), () => new SampleSerializableType(0));
+ SampleSerializableType serializable = (SampleSerializableType)d.ReadObject();
+ Assert.Equal(SampleSerializableType.ConstantValue, serializable.BeforeValue);
+ Assert.Equal(SampleSerializableType.ConstantValue, serializable.AfterValue);
+ }
+
+ [Fact]
+ public void WriteAndParseEightByteStreamLabel()
+ {
+ SampleSerializableType sample = new SampleSerializableType(SampleSerializableType.ConstantValue);
+ MemoryStream ms = new MemoryStream();
+ Serializer s = new Serializer(new IOStreamStreamWriter(ms, leaveOpen: true, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), sample);
+ s.Dispose();
+
+ Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), "name");
+ d.RegisterFactory(typeof(SampleSerializableType), () => new SampleSerializableType(0));
+ SampleSerializableType serializable = (SampleSerializableType)d.ReadObject();
+ Assert.Equal(SampleSerializableType.ConstantValue, serializable.BeforeValue);
+ Assert.Equal(SampleSerializableType.ConstantValue, serializable.AfterValue);
+ }
+ }
+
+ public sealed class SampleSerializableType : IFastSerializable
+ {
+ public const int ConstantValue = 42;
+
+ public SampleSerializableType(int value)
+ {
+ BeforeValue = value;
+ AfterValue = value;
+ }
+
+ public int BeforeValue { get; set; }
+ public int AfterValue { get; set; }
+
+ void IFastSerializable.ToStream(Serializer serializer)
+ {
+ serializer.Write(BeforeValue);
+ serializer.Writer.Write((StreamLabel)0x7EADBEEF);
+ serializer.Write(AfterValue);
+ }
+
+ void IFastSerializable.FromStream(Deserializer deserializer)
+ {
+ BeforeValue = deserializer.ReadInt();
+ StreamLabel label = deserializer.Reader.ReadLabel();
+ Assert.Equal((ulong)0x7EADBEEF, (ulong)label);
+ AfterValue = deserializer.ReadInt();
+ }
}
}