diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index f15c41d9..900cec5a 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -37,7 +37,7 @@ class IngestCommand : Command readonly FileInputFeature _fileInputFeature; readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; - string _filter; + string _filter, _pattern; bool _json; public IngestCommand(SeqConnectionFactory connectionFactory) @@ -47,6 +47,10 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); + Options.Add("x=|extract=", + "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)", + v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); + Options.Add("json", "Read the events as JSON (the default assumes plain text)", v => _json = true); @@ -54,7 +58,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) Options.Add("f=|filter=", "Filter expression to select a subset of events", v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); - + _connection = Enable(); } @@ -82,7 +86,7 @@ protected override async Task Run() var reader = _json ? (ILogEventReader)new ClefLogEventReader(input) : - new PlainTextLogEventReader(input); + new PlainTextLogEventReader(input, _pattern); using (reader as IDisposable) { diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/BuiltInPatterns.cs deleted file mode 100644 index 01f649d7..00000000 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ /dev/null @@ -1,60 +0,0 @@ -using SeqCli.PlainText.Parsers; -using Superpower; -using Superpower.Model; -using Superpower.Parsers; - -namespace SeqCli.PlainText -{ - static class BuiltInPatterns - { - public static TextParser Identifier { get; } = - IdentifierEx.CStyle - .Select(span => (object) span); - - public static TextParser Token { get; } = - SpanEx.NonWhiteSpace.Select(span => (object)span); - - public static TextParser MultiLineMessage { get; } = - SpanEx.MatchedBy( - Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") - .IgnoreThen(Character.AnyChar.Many())) - .Select(span => (object)span); - - public static TextParser MultiLineContent { get; } = - SpanEx.MatchedBy(Character.AnyChar.Many()) - .Select(span => (object)span); - - public static TextParser SingleLineContent { get; } = - SpanEx.MatchedBy(Character.ExceptIn('\r', '\n').Many()) - .Select(span => (object)span); - - public static TextParser LiteralText(string literalText) - { - return Span.EqualTo(literalText).Select(span => (object) span); - } - - public static TextParser NonGreedyContent(PatternElement[] following) - { - if (following.Length == 0) - return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); - - var rest = following[0].Parser; - for (var i = 1; i < following.Length; ++i) - { - rest = rest.IgnoreThen(following[i].Parser); - } - - return i => - { - var remainder = i; - while (!rest.IsMatch(remainder)) - { - remainder = remainder.ConsumeChar().Remainder; - } - - var span = i.Until(remainder); - return Result.Value((object) span, i, remainder); - }; - } - } -} diff --git a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs new file mode 100644 index 00000000..16834b93 --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SeqCli.PlainText.Patterns; + +namespace SeqCli.PlainText.Extraction +{ + static class ExtractionPatternInterpreter + { + public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] + { + new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) + }); + + public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern) + { + if (pattern == null) throw new ArgumentNullException(nameof(pattern)); + + var patternElements = new PatternElement[pattern.Elements.Count]; + for (var i = pattern.Elements.Count - 1; i >= 0; --i) + { + var element = pattern.Elements[i]; + switch (element) + { + case LiteralTextPatternExpression text: + patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); + break; + case CapturePatternExpression capture + when capture.Content is NonGreedyContentExpression ngc: + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()), + capture.Name); + break; + case CapturePatternExpression capture + when capture.Content is MatchTypeContentExpression mtc: + patternElements[i] = new PatternElement( + mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type), + capture.Name); + break; + default: + throw new InvalidOperationException($"Element `{element}` not recognized."); + } + } + + return new NameValueExtractor(patternElements); + } + + // What we need to do here is: + // - for each parsed token + // - if it's literal text, map it an anonymous PatternElement with + // BuiltInPatterns.LiteralText() + // - otherwise, if it specifies no format, it's a named element with + // the BuiltInPatterns.Token parser + // - if it does specify a format, look up the parser based on the name, except + // - if the format is `$` it is BuiltInPatterns.SingleLineContent + // - if the format is `$$` it is BuiltInPatterns.MultiLineContent + // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the + // parser that follows it + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs new file mode 100644 index 00000000..903c00cd --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace SeqCli.PlainText.Extraction +{ + [AttributeUsage(AttributeTargets.Property)] + class MatcherAttribute : Attribute + { + public string Name { get; } + + public MatcherAttribute(string name) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs new file mode 100644 index 00000000..ec2725bd --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Extraction +{ + // ReSharper disable UnusedMember.Global + static class Matchers + { + [Matcher("ident")] + public static TextParser Identifier { get; } = + IdentifierEx.CStyle + .Select(span => (object) span); + + [Matcher("nat")] + public static TextParser Natural { get; } = + Numerics.NaturalUInt64 + .Select(span => (object) span); + + [Matcher("int")] + public static TextParser Integer { get; } = + Numerics.IntegerInt64 + .Select(span => (object) span); + + [Matcher("dec")] + public static TextParser Decimal { get; } = + NumericsEx.Decimal + .Select(span => (object) span); + + [Matcher("alpha")] + public static TextParser Alphabetical { get; } = + Span.WithAll(char.IsLetter) + .Select(span => (object) span); + + [Matcher("alphanum")] + public static TextParser Alphanumeric { get; } = + Span.WithAll(char.IsLetterOrDigit) + .Select(span => (object) span); + + [Matcher("token")] + public static TextParser Token { get; } = + SpanEx.NonWhiteSpace.Select(span => (object)span); + + [Matcher("iso8601dt")] + // A date and time are required by this pattern, though not necessarily by the spec. + public static TextParser Iso8601DateTime { get; } = + DateTimesEx.Iso8601DateTime + .Select(span => (object) span); + + public static TextParser SerilogFileTimestamp { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?") + .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)); + + [Matcher("timestamp")] + public static TextParser Timestamp { get; } = + Iso8601DateTime.Try().Or(SerilogFileTimestamp); + + // Unclear whether we need to name this + public static TextParser MultiLineMessage { get; } = + SpanEx.MatchedBy( + Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") + .IgnoreThen(Character.AnyChar.Many())) + .Select(span => (object)span); + + // Equivalent to :* at end-of-pattern + public static TextParser MultiLineContent { get; } = + Span.WithAll(ch => true) + .Select(span => (object)span); + + [Matcher("line")] + public static TextParser SingleLineContent { get; } = + from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n') + from _ in NewLine.OptionalOrDefault() + select (object) content; + + [Matcher("n")] + public static TextParser NewLine { get; } = + Span.EqualTo("\r\n").Or(Span.EqualTo("\n")) + .Select(span => (object)span); + + [Matcher("t")] + public static TextParser Tab { get; } = + Span.EqualTo("\t") + .Select(span => (object)span); + + static readonly Dictionary> ByType = new Dictionary>( + from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties + let attr = pi.GetCustomAttribute() + where attr != null + select KeyValuePair.Create(attr.Name, (TextParser) pi.GetValue(null))); + + public static TextParser GetByType(string type) + { + if (type == null) throw new ArgumentNullException(nameof(type)); + return ByType[type]; + } + + public static TextParser LiteralText(string literalText) + { + return Span.EqualTo(literalText).Select(span => (object) span); + } + + public static TextParser NonGreedyContent(params PatternElement[] following) + { + if (following.Length == 0) + return SpanEx.MatchedBy(Character.AnyChar.Many()) + .Select(span => span.Length > 0 ? (object) span : null); + + var rest = following[0].Parser; + for (var i = 1; i < following.Length; ++i) + { + rest = rest.IgnoreThen(following[i].Parser); + } + + return i => + { + var remainder = i; + var attempt = rest(remainder); + while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything + { + remainder = remainder.ConsumeChar().Remainder; + attempt = rest(remainder); + } + + var span = i.Until(remainder); + return Result.Value(span.Length > 0 ? (object) span : null, i, remainder); + }; + } + } +} diff --git a/src/SeqCli/PlainText/Pattern.cs b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs similarity index 64% rename from src/SeqCli/PlainText/Pattern.cs rename to src/SeqCli/PlainText/Extraction/NameValueExtractor.cs index 8faeabc3..243fe6a3 100644 --- a/src/SeqCli/PlainText/Pattern.cs +++ b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs @@ -1,29 +1,28 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using Superpower; using Superpower.Model; using Superpower.Parsers; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { - class Pattern + class NameValueExtractor { readonly PatternElement[] _elements; - public Pattern(IEnumerable elements) + public NameValueExtractor(IEnumerable elements) { _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements)); if (_elements.Length == 0) - throw new ArgumentException("A match pattern must contain at least one element."); + throw new ArgumentException("An extraction pattern must contain at least one element."); } - public TextParser FrameStart => _elements[0].Parser; + public TextParser StartMarker => _elements[0].Parser; - public (IDictionary, string) Match(string frame) + public (IDictionary, string) ExtractValues(string plainText) { - var input = new TextSpan(frame); + var input = new TextSpan(plainText); var result = new Dictionary(); var remainder = input; @@ -42,8 +41,7 @@ public Pattern(IEnumerable elements) if (!element.IsIgnored) { - if (match.Value != null || !element.IsOptional) - result.Add(element.Name, match.Value); + result.Add(element.Name, match.Value); } } diff --git a/src/SeqCli/PlainText/PatternElement.cs b/src/SeqCli/PlainText/Extraction/PatternElement.cs similarity index 73% rename from src/SeqCli/PlainText/PatternElement.cs rename to src/SeqCli/PlainText/Extraction/PatternElement.cs index 53d30eae..29670dd9 100644 --- a/src/SeqCli/PlainText/PatternElement.cs +++ b/src/SeqCli/PlainText/Extraction/PatternElement.cs @@ -1,20 +1,18 @@ using System; using Superpower; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { class PatternElement { - public PatternElement(TextParser parser, string name = null, bool isOptional = false) + public PatternElement(TextParser parser, string name = null) { Parser = parser ?? throw new ArgumentNullException(nameof(parser)); Name = name; - IsOptional = isOptional; } public TextParser Parser { get; } public string Name { get; } - public bool IsOptional { get; } public bool IsIgnored => Name == null; } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/Frame.cs b/src/SeqCli/PlainText/Framing/Frame.cs similarity index 100% rename from src/SeqCli/PlainText/Frame.cs rename to src/SeqCli/PlainText/Framing/Frame.cs diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/Framing/FrameReader.cs similarity index 100% rename from src/SeqCli/PlainText/FrameReader.cs rename to src/SeqCli/PlainText/Framing/FrameReader.cs diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs similarity index 92% rename from src/SeqCli/PlainText/LogEventBuilder.cs rename to src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs index f5b252c2..786b27cd 100644 --- a/src/SeqCli/PlainText/LogEventBuilder.cs +++ b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs @@ -87,13 +87,17 @@ static IEnumerable GetLogEventProperties(IDictionary properties) { - var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && - t is TextSpan span && - DateTimeOffset.TryParse(span.ToStringValue(), CultureInfo.InvariantCulture, - DateTimeStyles.AssumeLocal, out var ts) - ? ts - : DateTimeOffset.Now; - return timestamp; + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; } static readonly Dictionary LevelsByName = new Dictionary(StringComparer.OrdinalIgnoreCase) diff --git a/src/SeqCli/PlainText/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs similarity index 100% rename from src/SeqCli/PlainText/TextOnlyException.cs rename to src/SeqCli/PlainText/LogEvents/TextOnlyException.cs diff --git a/src/SeqCli/PlainText/Parsers/DateTimesEx.cs b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs new file mode 100644 index 00000000..7653505a --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs @@ -0,0 +1,12 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class DateTimesEx + { + public static TextParser Iso8601DateTime { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?(([+-]\\d\\d:\\d\\d)|Z)?"); + } +} diff --git a/src/SeqCli/PlainText/Parsers/NumericsEx.cs b/src/SeqCli/PlainText/Parsers/NumericsEx.cs new file mode 100644 index 00000000..1810372b --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/NumericsEx.cs @@ -0,0 +1,21 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class NumericsEx + { + public static TextParser Decimal { get; } = + Numerics.Integer + .Then(n => Character.EqualTo('.').IgnoreThen(Numerics.Integer).OptionalOrDefault() + .Select(f => f == TextSpan.None ? n : new TextSpan(n.Source, n.Position, n.Length + f.Length + 1))); + + public static TextParser HexNatural { get; } = + SpanEx.MatchedBy(Span.EqualTo("0x") + .IgnoreThen(Character.Digit + .Or(Character.Matching(ch => ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F', "a-f")) + .Named("hex digit") + .AtLeastOnce())); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Parsers/StringsEx.cs b/src/SeqCli/PlainText/Parsers/StringsEx.cs new file mode 100644 index 00000000..a39ffff9 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/StringsEx.cs @@ -0,0 +1,16 @@ +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class StringsEx + { + static readonly TextParser SqlStringContentChar = + Span.EqualTo("''").Value('\'').Try().Or(Character.ExceptIn('\'', '\r', '\n')); + + public static TextParser SqlStyle { get; } = + Character.EqualTo('\'') + .IgnoreThen(SqlStringContentChar.Many()) + .Then(s => Character.EqualTo('\'').Value(new string(s))); + } +} diff --git a/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs new file mode 100644 index 00000000..52d4aacb --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs @@ -0,0 +1,13 @@ +using Superpower; + +namespace SeqCli.PlainText.Parsers +{ + public static class TextParserExtensions + { + public static TextParser Cast(this TextParser parser) + where T : U + { + return parser.Select(t => (U) t); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs deleted file mode 100644 index 7c3c3a61..00000000 --- a/src/SeqCli/PlainText/PatternBuilder.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace SeqCli.PlainText -{ - static class PatternBuilder - { - public static Pattern DefaultPattern { get; } = new Pattern(new[] { - new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) - }); - - // What we need to do here is: - // - for each parsed token - // - if it's literal text, map it an anonymous PatternElement with - // BuiltInPatterns.LiteralText() - // - otherwise, if it specifies no format, it's a named element with - // the BuiltInPatterns.Token parser - // - if it does specify a format, look up the parser based on the name, except - // - if the format is `$` it is BuiltInPatterns.SingleLineContent - // - if the format is `$$` it is BuiltInPatterns.MultiLineContent - // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the - // parser that follows it - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs new file mode 100644 index 00000000..ef4d8ec1 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class CaptureContentExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs new file mode 100644 index 00000000..4e76bca5 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs @@ -0,0 +1,14 @@ +namespace SeqCli.PlainText.Patterns +{ + class CapturePatternExpression : ExtractionPatternExpression + { + public string Name { get; } + public CaptureContentExpression Content { get; } + + public CapturePatternExpression(string name, CaptureContentExpression content) + { + Name = name; + Content = content; + } + } +} diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs new file mode 100644 index 00000000..4b810ed9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SeqCli.PlainText.Patterns +{ + class ExtractionPattern + { + public IReadOnlyList Elements { get; } + + public ExtractionPattern(IEnumerable items) + { + if (items == null) throw new ArgumentNullException(nameof(items)); + Elements = items.ToArray(); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs new file mode 100644 index 00000000..fca4ec75 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class ExtractionPatternExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs new file mode 100644 index 00000000..493edb1d --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -0,0 +1,53 @@ +using System; +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Patterns +{ + static class ExtractionPatternParser + { + static readonly TextParser LiteralText = + Span.EqualTo("{{").Value('{').Try() + .Or(Span.EqualTo("}}").Value('}').Try()) + .Or(Character.ExceptIn('{', '}')) + .AtLeastOnce() + .Select(ch => new LiteralTextPatternExpression(new string(ch))); + + static readonly TextParser CaptureName = + SpanEx.MatchedBy( + Character.Letter.Or(Character.In('@', '_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => s.ToStringValue()); + + static readonly TextParser CaptureContent = + Character.EqualTo('*').AtLeastOnce().Select(chs => (CaptureContentExpression)new NonGreedyContentExpression(chs.Length)) + .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => (CaptureContentExpression)new MatchTypeContentExpression(s.ToStringValue()))); + + static readonly TextParser Capture = + from _ in Character.EqualTo('{') + from name in CaptureName.OptionalOrDefault() + from content in Character.EqualTo(':') + .IgnoreThen(CaptureContent) + .OptionalOrDefault() + where name != null || content != null + from __ in Character.EqualTo('}') + select new CapturePatternExpression(name, content); + + static readonly TextParser Element = + LiteralText.Cast() + .Or(Capture.Cast()); + + static readonly TextParser Pattern = + Element.AtLeastOnce().AtEnd().Select(e => new ExtractionPattern(e)); + + public static ExtractionPattern Parse(string extractionPattern) + { + if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); + if (extractionPattern == "") throw new ParseException("Zero-length extraction patterns are not allowed."); + return Pattern.Parse(extractionPattern); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs new file mode 100644 index 00000000..a8399e81 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs @@ -0,0 +1,14 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class LiteralTextPatternExpression : ExtractionPatternExpression + { + public string Text { get; } + + public LiteralTextPatternExpression(string text) + { + Text = text ?? throw new ArgumentNullException(nameof(text)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs new file mode 100644 index 00000000..1a0f9bf9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class MatchTypeContentExpression : CaptureContentExpression + { + public string Type { get; } + + public MatchTypeContentExpression(string type) + { + Type = type; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs new file mode 100644 index 00000000..bb4a2969 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class NonGreedyContentExpression : CaptureContentExpression + { + public int Lookahead { get; } + + public NonGreedyContentExpression(int lookahead) + { + Lookahead = lookahead; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs index 8f413c4b..af1cac3a 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -2,7 +2,9 @@ using System.IO; using System.Threading.Tasks; using SeqCli.Ingestion; +using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Parsers; +using SeqCli.PlainText.Patterns; using Serilog.Events; namespace SeqCli.PlainText @@ -11,13 +13,16 @@ class PlainTextLogEventReader : ILogEventReader, IDisposable { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - readonly Pattern _pattern; + readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input) + public PlainTextLogEventReader(TextReader input, string extractionPattern) { - _pattern = PatternBuilder.DefaultPattern; - _reader = new FrameReader(input, SpanEx.MatchedBy(_pattern.FrameStart), TrailingLineArrivalDeadline); + _nameValueExtractor = string.IsNullOrEmpty(extractionPattern) ? + ExtractionPatternInterpreter.MultilineMessageExtractor : + ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); + + _reader = new FrameReader(input, SpanEx.MatchedBy(_nameValueExtractor.StartMarker), TrailingLineArrivalDeadline); } public async Task TryReadAsync() @@ -29,7 +34,7 @@ public async Task TryReadAsync() if (frame.IsOrphan) throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - var (properties, remainder) = _pattern.Match(frame.Value); + var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); return LogEventBuilder.FromProperties(properties, remainder); } diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs new file mode 100644 index 00000000..f14b1b77 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; +using SeqCli.PlainText.Patterns; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternInterpreterTests + { + static (IDictionary, string) ExtractValues(string pattern, string candidate) + { + var parsed = ExtractionPatternParser.Parse(pattern); + var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); + return extractor.ExtractValues(candidate); + } + + [Fact] + public void NonGreedyMatchCanLookaheadMultipleTokens() + { + var (properties, remainder) = ExtractValues("[{test:**}]!", "[0]abc[1]!"); + Assert.Null(remainder); + Assert.Equal("0]abc[1", properties["test"].ToString()); + } + + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() + { + // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting + + // {@l:ident} is required so that the default "token" pattern doesn't greedily eat up the `]`. + // "timestamp" is intended to be an aggregate timestamp parser that tries ISO 8601, RFC 2822, and various other + // popular timestamp formats. + + var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}"; + + var candidate = +@"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed +System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +"; + + var (properties, remainder) = ExtractValues(pattern, candidate); + + Assert.Equal( + DateTimeOffset.ParseExact("2018-02-21 13:29:00.123 +10:00", "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture), + properties["@t"]); + Assert.Equal("ERR", properties["@l"].ToString()); + Assert.Equal("The operation failed", properties["@m"].ToString()); + Assert.Equal(@"System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +", properties["@x"].ToString()); + Assert.Null(remainder); + } + + // Work-in-progress... + + // [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() + { + // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-console#output-templates + + // "localtime" will add the closest non-future date to the time component that is matched + // by the pattern + + // The pattern language needs to be extended here so that the brackets, timestamp, spacing and + // level are all used as the start-frame marker. The strawman syntax proposes that to the + // right of `:` will always be either an alphanumeric matcher name, or a subexpression. This + // does have the issue that `{?:foo}` would be ambiguous (optional 'foo' matcher or optional 'foo' + // literal, so some escaping would be necessary - e.g. `{?:\foo}` to indicate a literal 'foo' and + // `{?:\*}` for an optional literal asterisk, `{?:\\}` for an optional literal backslash. + +#pragma warning disable 219 + var pattern = "{:[{@t:localtime} {@l:ident}] }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 + } + + // [Fact] + public void OptionalSourceContextCanBeExtracted() + { + // The {?: optional grouping is just an anonymous optional property, e.g. if the formatting was + // not dynamic, it might be written {SourceContext?:*}; using the grouping means the surrounding + // whitespace and parens are required only if the optional group is matched. +#pragma warning disable 219 + var pattern = "{:[{@t} {@l:ident}] }{?:({SourceContext:*}) }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs new file mode 100644 index 00000000..85fd84fc --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -0,0 +1,64 @@ +using System; +using System.Linq; +using SeqCli.PlainText.Patterns; +using Superpower; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternParserTests + { + [Fact] + public void ARegularStringIsASingleTextLiteral() + { + var pattern = ExtractionPatternParser.Parse("Hello!"); + Assert.Single(pattern.Elements); + var tt = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("Hello!", tt.Text); + } + + [Fact] + public void CaptureNameAndTypeAreParsed() + { + var pattern = ExtractionPatternParser.Parse("{abc:def}"); + Assert.Single(pattern.Elements); + var ct = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("abc", ct.Name); + Assert.Equal("def", ((MatchTypeContentExpression)ct.Content).Type); + } + + [Theory] + [InlineData("", false)] + [InlineData("{}", false)] + [InlineData("{a", false)] + [InlineData("a", true)] + [InlineData("{a}", true)] + [InlineData("{@m}", true)] + [InlineData("{@@m}", false)] + [InlineData("{m@}", false)] + [InlineData("{@m:n}", true)] + [InlineData("{@m:*}", true)] + [InlineData("{@m:***}", true)] + [InlineData("{:*}", true)] + [InlineData("{a:}", false)] + [InlineData("{@m:n}", true)] + [InlineData("{m_N}", true)] + [InlineData("{_9}", true)] + [InlineData("{:n}", true)] + [InlineData("{:}", false)] + [InlineData("{{@m}}", true)] + [InlineData("{{a", true)] + [InlineData("a}}", true)] + [InlineData("{", false)] + [InlineData("}", false)] + [InlineData("{a} b{c} ", true)] + [InlineData("d {a}b {c}", true)] + public void OnlyValidPatternsAreAccepted(string attempt, bool isValid) + { + if (isValid) + ExtractionPatternParser.Parse(attempt); + else + Assert.Throws(() => ExtractionPatternParser.Parse(attempt)); + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs index 50a67383..7cf38310 100644 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs @@ -45,5 +45,13 @@ public void MissingValuesAreDefaulted() Assert.Null(evt.Exception); Assert.Empty(evt.Properties); } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then, evt.Timestamp); + } } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs similarity index 55% rename from test/SeqCli.Tests/PlainText/PatternTests.cs rename to test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs index e899cc4b..785ede83 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs @@ -1,18 +1,19 @@ using System; using System.Linq; using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; using Superpower.Model; using Xunit; namespace SeqCli.Tests.PlainText { - public class PatternTests + public class NameValueExtractorTests { [Fact] public void TheDefaultPatternMatchesMultilineMessages() { var frame = $"Hello,{Environment.NewLine} world!"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Null(remainder); Assert.Single(properties, p => p.Key == ReifiedProperties.Message && ((TextSpan)p.Value).ToStringValue() == frame); @@ -22,16 +23,16 @@ public void TheDefaultPatternMatchesMultilineMessages() public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() { var frame = " world"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Empty(properties); Assert.Equal(frame, remainder); } - static Pattern ClassMethodPattern { get; } = new Pattern(new[] + static NameValueExtractor ClassMethodPattern { get; } = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "class"), - new PatternElement(BuiltInPatterns.LiteralText(".")), - new PatternElement(BuiltInPatterns.Identifier, "method") + new PatternElement(Matchers.Identifier, "class"), + new PatternElement(Matchers.LiteralText(".")), + new PatternElement(Matchers.Identifier, "method") }); [Fact] @@ -40,7 +41,7 @@ public void PatternsExtractElements() var pattern = ClassMethodPattern; var frame = "this.that"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("this", properties["class"].ToString()); Assert.Equal("that", properties["method"].ToString()); @@ -49,24 +50,24 @@ public void PatternsExtractElements() [Fact] public void TheFirstPatternElementIsExposed() { - Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); + Assert.Same(Matchers.Identifier, ClassMethodPattern.StartMarker); } [Fact] public void SingleLineContentMatchesUntilEol() { - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.SingleLineContent, "content"), - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.SingleLineContent, "content"), + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi (jkl)", properties["content"].ToString()); @@ -79,20 +80,20 @@ public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() // the "following" list, since they effectively become "mandatory" var following = new[] { - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }; - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.NonGreedyContent(following), "content"), + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.NonGreedyContent(following), "content"), }.Concat(following)); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi", properties["content"].ToString());