diff --git a/src/FastSerialization/FastSerialization.cs b/src/FastSerialization/FastSerialization.cs index 250cc9466..73eb31807 100644 --- a/src/FastSerialization/FastSerialization.cs +++ b/src/FastSerialization/FastSerialization.cs @@ -67,6 +67,33 @@ namespace FastSerialization // // This is MUCH leaner, and now dominated by actual work of copying the data to the output buffer. + /// + /// Allows users of serialization and de-serialization mechanisms to specify the size of the StreamLabel. + /// As traces get larger, there is a need to support larger file sizes, and thus to increase the addressable + /// space within the files. StreamLabel instances are 8-bytes in-memory, but all serialization and de-serialization + /// of them results in the upper 4-bytes being lost. This setting will allow Serializer and Deserializer to read + /// and write 8-byte StreamLabel instances. + /// +#if FASTSERIALIZATION_PUBLIC + public +#endif + enum StreamLabelWidth + { + FourBytes = 0, + EightBytes = 1 + }; + + /// + /// These settings apply to use of Serializer and Deserializer specifically. + /// +#if FASTSERIALIZATION_PUBLIC + public +#endif + sealed class SerializationConfiguration + { + public StreamLabelWidth StreamLabelWidth { get; set; } + } + /// /// A StreamLabel represents a position in a IStreamReader or IStreamWriter. /// In memory it is represented as a 64 bit signed value but to preserve compat @@ -991,14 +1018,19 @@ sealed class Deserializer : IDisposable /// /// Create a Deserializer that reads its data from a given file /// - public Deserializer(string filePath) : this(new IOStreamStreamReader(filePath), filePath) { } + public Deserializer(string filePath, SerializationConfiguration config = null) + { + IOStreamStreamReader reader = new IOStreamStreamReader(filePath, config); + Initialize(reader, filePath); + } /// /// Create a Deserializer that reads its data from a given System.IO.Stream. The stream will be closed when the Deserializer is done with it. /// - public Deserializer(Stream inputStream, string streamName) - : this(new IOStreamStreamReader(inputStream), streamName) + public Deserializer(Stream inputStream, string streamName, SerializationConfiguration config = null) { + IOStreamStreamReader reader = new IOStreamStreamReader(inputStream, config: config); + Initialize(reader, streamName); } /// @@ -1006,15 +1038,21 @@ public Deserializer(Stream inputStream, string streamName) /// parameter determines whether the deserializer will close the stream when it /// closes. /// - public Deserializer(Stream inputStream, string streamName, bool leaveOpen) - : this(new IOStreamStreamReader(inputStream, leaveOpen: leaveOpen), streamName) + public Deserializer(Stream inputStream, string streamName, bool leaveOpen, SerializationConfiguration config = null) { + IOStreamStreamReader reader = new IOStreamStreamReader(inputStream, leaveOpen: leaveOpen, config: config); + Initialize(reader, streamName); } /// /// Create a Deserializer that reads its data from a given IStreamReader. The stream will be closed when the Deserializer is done with it. /// public Deserializer(IStreamReader reader, string streamName) + { + Initialize(reader, streamName); + } + + private void Initialize(IStreamReader reader, string streamName) { ObjectsInGraph = new Dictionary(); this.reader = reader; diff --git a/src/FastSerialization/StreamReaderWriter.cs b/src/FastSerialization/StreamReaderWriter.cs index 80816003f..6edc853fd 100644 --- a/src/FastSerialization/StreamReaderWriter.cs +++ b/src/FastSerialization/StreamReaderWriter.cs @@ -24,19 +24,39 @@ class MemoryStreamReader : IStreamReader /// public MemoryStreamReader(byte[] data) : this(data, 0, data.Length) { } /// - /// Create a IStreamReader (reads binary data) from a given subregion of a byte buffer + /// Create a IStreamReader (reads binary data) from a given subregion of a byte buffer. /// - public MemoryStreamReader(byte[] data, int start, int length) + public MemoryStreamReader(byte[] data, int start, int length, SerializationConfiguration config = null) { bytes = data; position = start; endPosition = length; + SerializationConfiguration = config != null ? config : new SerializationConfiguration(); + + if(SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + readLabel = () => + { + return (StreamLabel)(uint)ReadInt32(); + }; + sizeOfSerializedStreamLabel = 4; + } + else + { + readLabel = () => + { + return (StreamLabel)(ulong)ReadInt64(); + }; + sizeOfSerializedStreamLabel = 8; + } } + /// /// The total length of bytes that this reader can read. /// public virtual long Length { get { return endPosition; } } public virtual bool HasLength { get { return true; } } + public SerializationConfiguration SerializationConfiguration { get; private set; } #region implemenation of IStreamReader public virtual void Read(byte[] data, int offset, int length) @@ -150,7 +170,8 @@ public string ReadString() /// public StreamLabel ReadLabel() { - return (StreamLabel)(uint)ReadInt32(); + // Delegate set in the constructor based on the size of StreamLabel. + return readLabel(); } /// /// Implementation of IStreamReader @@ -176,7 +197,6 @@ public virtual StreamLabel Current /// public virtual void GotoSuffixLabel() { - const int sizeOfSerializedStreamLabel = 4; Goto((StreamLabel)(Length - sizeOfSerializedStreamLabel)); Goto(ReadLabel()); } @@ -203,6 +223,8 @@ protected virtual void Dispose(bool disposing) { } internal /*protected*/ int position; internal /*protected*/ int endPosition; private StringBuilder sb; + private Func readLabel; + private readonly int sizeOfSerializedStreamLabel; #endregion } @@ -222,11 +244,31 @@ class MemoryStreamWriter : IStreamWriter /// /// Call 'GetBytes' call to get the raw array. Only the first 'Length' bytes are valid /// - public MemoryStreamWriter(int initialSize = 64) + public MemoryStreamWriter(int initialSize = 64, SerializationConfiguration config = null) { bytes = new byte[initialSize]; + SerializationConfiguration = config != null ? config : new SerializationConfiguration(); + + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + writeLabel = (value) => + { + Debug.Assert((long)value <= int.MaxValue); + Write((int)value); + }; + } + else + { + writeLabel = (value) => + { + Debug.Assert((long)value <= long.MaxValue); + Write((long)value); + }; + } } + public SerializationConfiguration SerializationConfiguration { get; set; } + /// /// Returns a IStreamReader that will read the written bytes. You cannot write additional bytes to the stream after making this call. /// @@ -327,8 +369,7 @@ public unsafe void Write(long value) /// public void Write(StreamLabel value) { - Debug.Assert((long)value <= int.MaxValue); - Write((int)value); + writeLabel(value); } /// /// Implementation of IStreamWriter @@ -424,6 +465,7 @@ protected virtual void Dispose(bool disposing) { } Array.Copy(bytes, newBytes, bytes.Length); bytes = newBytes; } + private Action writeLabel; internal /* protected */ byte[] bytes; internal /* protected */ int endPosition; #endregion @@ -597,15 +639,15 @@ class IOStreamStreamReader : MemoryStreamReader, IDisposable /// Create a new IOStreamStreamReader from the given file. /// /// - public IOStreamStreamReader(string fileName) - : this(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { } + public IOStreamStreamReader(string fileName, SerializationConfiguration config = null) + : this(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete), config: config) { } /// /// Create a new IOStreamStreamReader from the given System.IO.Stream. Optionally you can specify the size of the read buffer /// The stream will be closed by the IOStreamStreamReader when it is closed. /// - public IOStreamStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, bool leaveOpen = false) - : base(new byte[bufferSize + align], 0, 0) + public IOStreamStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, bool leaveOpen = false, SerializationConfiguration config = null) + : base(new byte[bufferSize + align], 0, 0, config) { Debug.Assert(bufferSize % align == 0); this.inputStream = inputStream; @@ -839,17 +881,17 @@ sealed unsafe class PinnedStreamReader : IOStreamStreamReader /// /// Create a new PinnedStreamReader that gets its data from a given file. You can optionally set the size of the read buffer. /// - public PinnedStreamReader(string fileName, int bufferSize = defaultBufferSize) + public PinnedStreamReader(string fileName, int bufferSize = defaultBufferSize, SerializationConfiguration config = null) : this(new FileStream(fileName, FileMode.Open, FileAccess.Read, - FileShare.Read | FileShare.Delete), bufferSize) + FileShare.Read | FileShare.Delete), bufferSize, config) { } /// /// Create a new PinnedStreamReader that gets its data from a given System.IO.Stream. You can optionally set the size of the read buffer. /// The stream will be closed by the PinnedStreamReader when it is closed. /// - public PinnedStreamReader(Stream inputStream, int bufferSize = defaultBufferSize) - : base(inputStream, bufferSize) + public PinnedStreamReader(Stream inputStream, int bufferSize = defaultBufferSize, SerializationConfiguration config = null) + : base(inputStream, bufferSize, config: config) { // Pin the array pinningHandle = System.Runtime.InteropServices.GCHandle.Alloc(bytes, System.Runtime.InteropServices.GCHandleType.Pinned); @@ -1016,13 +1058,13 @@ class IOStreamStreamWriter : MemoryStreamWriter, IDisposable /// Create a IOStreamStreamWriter that writes its data to a given file that it creates /// /// - public IOStreamStreamWriter(string fileName) : this(new FileStream(fileName, FileMode.Create)) { } + public IOStreamStreamWriter(string fileName, SerializationConfiguration config = null) : this(new FileStream(fileName, FileMode.Create), config: config) { } /// /// Create a IOStreamStreamWriter that writes its data to a System.IO.Stream /// - public IOStreamStreamWriter(Stream outputStream, int bufferSize = defaultBufferSize + sizeof(long), bool leaveOpen = false) - : base(bufferSize) + public IOStreamStreamWriter(Stream outputStream, int bufferSize = defaultBufferSize + sizeof(long), bool leaveOpen = false, SerializationConfiguration config = null) + : base(bufferSize, config) { this.outputStream = outputStream; this.leaveOpen = leaveOpen; @@ -1071,7 +1113,7 @@ public override long Length public override StreamLabel GetLabel() { long len = Length; - if (len != (uint)len) + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes && len != (uint)len) { throw new NotSupportedException("Streams larger than 4 GB. You need to use /MaxEventCount to limit the size."); } diff --git a/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs b/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs index 01d359c55..a7d3ffa29 100644 --- a/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs +++ b/src/TraceEvent/TraceEvent.Tests/FastSerializerTests.cs @@ -12,7 +12,26 @@ namespace TraceEventTests public class FastSerializerTests { [Fact] - public void ParseStreamLabel() + public void ParseEightByteStreamLabel() + { + MemoryStream ms = new MemoryStream(); + BinaryWriter writer = new BinaryWriter(ms); + WriteString(writer, "!FastSerialization.1"); + writer.Write((long)0); + writer.Write((long)19); + writer.Write((long)1_000_000); + writer.Write((long)0xf1234567); + + ms.Position = 0; + Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), "name"); + Assert.Equal((StreamLabel)0, d.ReadLabel()); + Assert.Equal((StreamLabel)19, d.ReadLabel()); + Assert.Equal((StreamLabel)1_000_000, d.ReadLabel()); + Assert.Equal((StreamLabel)0xf1234567, d.ReadLabel()); + } + + [Fact] + public void ParseFourByteStreamLabel() { MemoryStream ms = new MemoryStream(); BinaryWriter writer = new BinaryWriter(ms); @@ -23,7 +42,7 @@ public void ParseStreamLabel() writer.Write(0xf1234567); ms.Position = 0; - Deserializer d = new Deserializer(new PinnedStreamReader(ms), "name"); + Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), "name"); Assert.Equal((StreamLabel)0, d.ReadLabel()); Assert.Equal((StreamLabel)19, d.ReadLabel()); Assert.Equal((StreamLabel)1_000_000, d.ReadLabel()); @@ -35,5 +54,64 @@ private void WriteString(BinaryWriter writer, string val) writer.Write(val.Length); writer.Write(Encoding.UTF8.GetBytes(val)); } + + [Fact] + public void WriteAndParseFourByteStreamLabel() + { + SampleSerializableType sample = new SampleSerializableType(SampleSerializableType.ConstantValue); + MemoryStream ms = new MemoryStream(); + Serializer s = new Serializer(new IOStreamStreamWriter(ms, leaveOpen:true, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), sample); + s.Dispose(); + + Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.FourBytes }), "name"); + d.RegisterFactory(typeof(SampleSerializableType), () => new SampleSerializableType(0)); + SampleSerializableType serializable = (SampleSerializableType)d.ReadObject(); + Assert.Equal(SampleSerializableType.ConstantValue, serializable.BeforeValue); + Assert.Equal(SampleSerializableType.ConstantValue, serializable.AfterValue); + } + + [Fact] + public void WriteAndParseEightByteStreamLabel() + { + SampleSerializableType sample = new SampleSerializableType(SampleSerializableType.ConstantValue); + MemoryStream ms = new MemoryStream(); + Serializer s = new Serializer(new IOStreamStreamWriter(ms, leaveOpen: true, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), sample); + s.Dispose(); + + Deserializer d = new Deserializer(new PinnedStreamReader(ms, config: new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes }), "name"); + d.RegisterFactory(typeof(SampleSerializableType), () => new SampleSerializableType(0)); + SampleSerializableType serializable = (SampleSerializableType)d.ReadObject(); + Assert.Equal(SampleSerializableType.ConstantValue, serializable.BeforeValue); + Assert.Equal(SampleSerializableType.ConstantValue, serializable.AfterValue); + } + } + + public sealed class SampleSerializableType : IFastSerializable + { + public const int ConstantValue = 42; + + public SampleSerializableType(int value) + { + BeforeValue = value; + AfterValue = value; + } + + public int BeforeValue { get; set; } + public int AfterValue { get; set; } + + void IFastSerializable.ToStream(Serializer serializer) + { + serializer.Write(BeforeValue); + serializer.Writer.Write((StreamLabel)0x7EADBEEF); + serializer.Write(AfterValue); + } + + void IFastSerializable.FromStream(Deserializer deserializer) + { + BeforeValue = deserializer.ReadInt(); + StreamLabel label = deserializer.Reader.ReadLabel(); + Assert.Equal((ulong)0x7EADBEEF, (ulong)label); + AfterValue = deserializer.ReadInt(); + } } }