From 3d179624786f69ff39901634b5d84eeb135b13e3 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 23 Feb 2018 21:25:07 +1000 Subject: [PATCH 1/5] Initial extraction pattern support --- src/SeqCli/Cli/Commands/IngestCommand.cs | 10 ++- .../PlainText/Extraction/MatcherAttribute.cs | 15 +++++ .../Matchers.cs} | 58 ++++++++++++++--- .../NameValueExtractor.cs} | 18 +++--- .../PlainText/Extraction/PatternBuilder.cs | 62 +++++++++++++++++++ .../{ => Extraction}/PatternElement.cs | 6 +- src/SeqCli/PlainText/{ => Framing}/Frame.cs | 0 .../PlainText/{ => Framing}/FrameReader.cs | 0 .../{ => LogEvents}/LogEventBuilder.cs | 0 .../{ => LogEvents}/TextOnlyException.cs | 0 .../PlainText/Parsers/TextParserExtensions.cs | 13 ++++ src/SeqCli/PlainText/PatternBuilder.cs | 21 ------- .../Patterns/CapturePatternExpression.cs | 16 +++++ .../PlainText/Patterns/ExtractionPattern.cs | 17 +++++ .../Patterns/ExtractionPatternExpression.cs | 6 ++ .../Patterns/ExtractionPatternParser.cs | 53 ++++++++++++++++ .../Patterns/LiteralTextPatternExpression.cs | 14 +++++ .../PlainText/PlainTextLogEventReader.cs | 15 +++-- .../PlainText/ExtractionPatternParserTests.cs | 59 ++++++++++++++++++ test/SeqCli.Tests/PlainText/PatternTests.cs | 49 ++++++++------- 20 files changed, 355 insertions(+), 77 deletions(-) create mode 100644 src/SeqCli/PlainText/Extraction/MatcherAttribute.cs rename src/SeqCli/PlainText/{BuiltInPatterns.cs => Extraction/Matchers.cs} (50%) rename src/SeqCli/PlainText/{Pattern.cs => Extraction/NameValueExtractor.cs} (64%) create mode 100644 src/SeqCli/PlainText/Extraction/PatternBuilder.cs rename src/SeqCli/PlainText/{ => Extraction}/PatternElement.cs (73%) rename src/SeqCli/PlainText/{ => Framing}/Frame.cs (100%) rename src/SeqCli/PlainText/{ => Framing}/FrameReader.cs (100%) rename src/SeqCli/PlainText/{ => LogEvents}/LogEventBuilder.cs (100%) rename src/SeqCli/PlainText/{ => LogEvents}/TextOnlyException.cs (100%) create mode 100644 src/SeqCli/PlainText/Parsers/TextParserExtensions.cs delete mode 100644 src/SeqCli/PlainText/PatternBuilder.cs create mode 100644 src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPattern.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs create mode 100644 src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs create mode 100644 test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index f15c41d9..ed7817ae 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -37,7 +37,7 @@ class IngestCommand : Command readonly FileInputFeature _fileInputFeature; readonly PropertiesFeature _properties; readonly ConnectionFeature _connection; - string _filter; + string _filter, _pattern; bool _json; public IngestCommand(SeqConnectionFactory connectionFactory) @@ -47,6 +47,10 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); + Options.Add("p=|pattern=", + "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)", + v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); + Options.Add("json", "Read the events as JSON (the default assumes plain text)", v => _json = true); @@ -54,7 +58,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) Options.Add("f=|filter=", "Filter expression to select a subset of events", v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); - + _connection = Enable(); } @@ -82,7 +86,7 @@ protected override async Task Run() var reader = _json ? (ILogEventReader)new ClefLogEventReader(input) : - new PlainTextLogEventReader(input); + new PlainTextLogEventReader(input, _pattern); using (reader as IDisposable) { diff --git a/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs new file mode 100644 index 00000000..903c00cd --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/MatcherAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace SeqCli.PlainText.Extraction +{ + [AttributeUsage(AttributeTargets.Property)] + class MatcherAttribute : Attribute + { + public string Name { get; } + + public MatcherAttribute(string name) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/BuiltInPatterns.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs similarity index 50% rename from src/SeqCli/PlainText/BuiltInPatterns.cs rename to src/SeqCli/PlainText/Extraction/Matchers.cs index 01f649d7..9f5c9fcc 100644 --- a/src/SeqCli/PlainText/BuiltInPatterns.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -1,39 +1,77 @@ -using SeqCli.PlainText.Parsers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using SeqCli.PlainText.Parsers; using Superpower; using Superpower.Model; using Superpower.Parsers; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { - static class BuiltInPatterns + // ReSharper disable UnusedMember.Global + static class Matchers { + [Matcher("ident")] public static TextParser Identifier { get; } = IdentifierEx.CStyle .Select(span => (object) span); - + + [Matcher("nat")] + public static TextParser Natural { get; } = + Numerics.Natural + .Select(span => (object) span); + + [Matcher("int")] + public static TextParser Integer { get; } = + Numerics.Integer + .Select(span => (object) span); + + [Matcher("token")] public static TextParser Token { get; } = SpanEx.NonWhiteSpace.Select(span => (object)span); - + + // Unclear whether we need to name this public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character") .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); - + + [Matcher("lines")] public static TextParser MultiLineContent { get; } = - SpanEx.MatchedBy(Character.AnyChar.Many()) + Span.WithAll(ch => true) .Select(span => (object)span); - + + [Matcher("line")] public static TextParser SingleLineContent { get; } = - SpanEx.MatchedBy(Character.ExceptIn('\r', '\n').Many()) + from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n') + from _ in NewLine.OptionalOrDefault() + select (object) content; + + [Matcher("n")] + public static TextParser NewLine { get; } = + Span.EqualTo("\r\n").Or(Span.EqualTo("\n")) .Select(span => (object)span); + static readonly Dictionary> ByType = new Dictionary>( + from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties + let attr = pi.GetCustomAttribute() + where attr != null + select KeyValuePair.Create(attr.Name, (TextParser) pi.GetValue(null))); + + public static TextParser GetByType(string type) + { + if (type == null) throw new ArgumentNullException(nameof(type)); + return ByType[type]; + } + public static TextParser LiteralText(string literalText) { return Span.EqualTo(literalText).Select(span => (object) span); } - public static TextParser NonGreedyContent(PatternElement[] following) + public static TextParser NonGreedyContent(params PatternElement[] following) { if (following.Length == 0) return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); diff --git a/src/SeqCli/PlainText/Pattern.cs b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs similarity index 64% rename from src/SeqCli/PlainText/Pattern.cs rename to src/SeqCli/PlainText/Extraction/NameValueExtractor.cs index 8faeabc3..243fe6a3 100644 --- a/src/SeqCli/PlainText/Pattern.cs +++ b/src/SeqCli/PlainText/Extraction/NameValueExtractor.cs @@ -1,29 +1,28 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using Superpower; using Superpower.Model; using Superpower.Parsers; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { - class Pattern + class NameValueExtractor { readonly PatternElement[] _elements; - public Pattern(IEnumerable elements) + public NameValueExtractor(IEnumerable elements) { _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements)); if (_elements.Length == 0) - throw new ArgumentException("A match pattern must contain at least one element."); + throw new ArgumentException("An extraction pattern must contain at least one element."); } - public TextParser FrameStart => _elements[0].Parser; + public TextParser StartMarker => _elements[0].Parser; - public (IDictionary, string) Match(string frame) + public (IDictionary, string) ExtractValues(string plainText) { - var input = new TextSpan(frame); + var input = new TextSpan(plainText); var result = new Dictionary(); var remainder = input; @@ -42,8 +41,7 @@ public Pattern(IEnumerable elements) if (!element.IsIgnored) { - if (match.Value != null || !element.IsOptional) - result.Add(element.Name, match.Value); + result.Add(element.Name, match.Value); } } diff --git a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs b/src/SeqCli/PlainText/Extraction/PatternBuilder.cs new file mode 100644 index 00000000..0ae2aae7 --- /dev/null +++ b/src/SeqCli/PlainText/Extraction/PatternBuilder.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using SeqCli.PlainText.Patterns; + +namespace SeqCli.PlainText.Extraction +{ + static class PatternCompiler + { + public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] + { + new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) + }); + + public static NameValueExtractor Compile(ExtractionPattern pattern) + { + if (pattern == null) throw new ArgumentNullException(nameof(pattern)); + + var patternElements = new PatternElement[pattern.Elements.Count]; + for (var i = 0; i < pattern.Elements.Count; ++i) + { + var element = pattern.Elements[i]; + switch (element) + { + case LiteralTextPatternExpression text: + patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); + break; + case CapturePatternExpression capture when capture.Type == "*": + if (i < pattern.Elements.Count - 1) + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(patternElements[i + 1]), + capture.Name); + else + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(), // <- same as MultiLineContent + capture.Name); + break; + case CapturePatternExpression capture: + patternElements[i] = new PatternElement( + capture.Type == null ? Matchers.Token : Matchers.GetByType(capture.Type), + capture.Name); + break; + default: + throw new InvalidOperationException($"Element `{element}` not recognized."); + } + } + + return new NameValueExtractor(patternElements); + } + + // What we need to do here is: + // - for each parsed token + // - if it's literal text, map it an anonymous PatternElement with + // BuiltInPatterns.LiteralText() + // - otherwise, if it specifies no format, it's a named element with + // the BuiltInPatterns.Token parser + // - if it does specify a format, look up the parser based on the name, except + // - if the format is `$` it is BuiltInPatterns.SingleLineContent + // - if the format is `$$` it is BuiltInPatterns.MultiLineContent + // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the + // parser that follows it + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternElement.cs b/src/SeqCli/PlainText/Extraction/PatternElement.cs similarity index 73% rename from src/SeqCli/PlainText/PatternElement.cs rename to src/SeqCli/PlainText/Extraction/PatternElement.cs index 53d30eae..29670dd9 100644 --- a/src/SeqCli/PlainText/PatternElement.cs +++ b/src/SeqCli/PlainText/Extraction/PatternElement.cs @@ -1,20 +1,18 @@ using System; using Superpower; -namespace SeqCli.PlainText +namespace SeqCli.PlainText.Extraction { class PatternElement { - public PatternElement(TextParser parser, string name = null, bool isOptional = false) + public PatternElement(TextParser parser, string name = null) { Parser = parser ?? throw new ArgumentNullException(nameof(parser)); Name = name; - IsOptional = isOptional; } public TextParser Parser { get; } public string Name { get; } - public bool IsOptional { get; } public bool IsIgnored => Name == null; } } \ No newline at end of file diff --git a/src/SeqCli/PlainText/Frame.cs b/src/SeqCli/PlainText/Framing/Frame.cs similarity index 100% rename from src/SeqCli/PlainText/Frame.cs rename to src/SeqCli/PlainText/Framing/Frame.cs diff --git a/src/SeqCli/PlainText/FrameReader.cs b/src/SeqCli/PlainText/Framing/FrameReader.cs similarity index 100% rename from src/SeqCli/PlainText/FrameReader.cs rename to src/SeqCli/PlainText/Framing/FrameReader.cs diff --git a/src/SeqCli/PlainText/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs similarity index 100% rename from src/SeqCli/PlainText/LogEventBuilder.cs rename to src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs diff --git a/src/SeqCli/PlainText/TextOnlyException.cs b/src/SeqCli/PlainText/LogEvents/TextOnlyException.cs similarity index 100% rename from src/SeqCli/PlainText/TextOnlyException.cs rename to src/SeqCli/PlainText/LogEvents/TextOnlyException.cs diff --git a/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs new file mode 100644 index 00000000..52d4aacb --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/TextParserExtensions.cs @@ -0,0 +1,13 @@ +using Superpower; + +namespace SeqCli.PlainText.Parsers +{ + public static class TextParserExtensions + { + public static TextParser Cast(this TextParser parser) + where T : U + { + return parser.Select(t => (U) t); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PatternBuilder.cs b/src/SeqCli/PlainText/PatternBuilder.cs deleted file mode 100644 index 7c3c3a61..00000000 --- a/src/SeqCli/PlainText/PatternBuilder.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace SeqCli.PlainText -{ - static class PatternBuilder - { - public static Pattern DefaultPattern { get; } = new Pattern(new[] { - new PatternElement(BuiltInPatterns.MultiLineMessage, ReifiedProperties.Message) - }); - - // What we need to do here is: - // - for each parsed token - // - if it's literal text, map it an anonymous PatternElement with - // BuiltInPatterns.LiteralText() - // - otherwise, if it specifies no format, it's a named element with - // the BuiltInPatterns.Token parser - // - if it does specify a format, look up the parser based on the name, except - // - if the format is `$` it is BuiltInPatterns.SingleLineContent - // - if the format is `$$` it is BuiltInPatterns.MultiLineContent - // - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the - // parser that follows it - } -} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs new file mode 100644 index 00000000..def5648d --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs @@ -0,0 +1,16 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class CapturePatternExpression : ExtractionPatternExpression + { + public string Name { get; } + public string Type { get; } + + public CapturePatternExpression(string name, string type) + { + Name = name; + Type = type; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs new file mode 100644 index 00000000..4b810ed9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPattern.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SeqCli.PlainText.Patterns +{ + class ExtractionPattern + { + public IReadOnlyList Elements { get; } + + public ExtractionPattern(IEnumerable items) + { + if (items == null) throw new ArgumentNullException(nameof(items)); + Elements = items.ToArray(); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs new file mode 100644 index 00000000..fca4ec75 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class ExtractionPatternExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs new file mode 100644 index 00000000..1038967a --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -0,0 +1,53 @@ +using System; +using SeqCli.PlainText.Parsers; +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Patterns +{ + static class ExtractionPatternParser + { + static readonly TextParser LiteralText = + Span.EqualTo("{{").Value('{').Try() + .Or(Span.EqualTo("}}").Value('}').Try()) + .Or(Character.ExceptIn('{', '}')) + .AtLeastOnce() + .Select(ch => new LiteralTextPatternExpression(new string(ch))); + + static readonly TextParser CaptureName = + SpanEx.MatchedBy( + Character.Letter.Or(Character.In('@', '_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => s.ToStringValue()); + + static readonly TextParser CaptureType = + SpanEx.MatchedBy(Character.EqualTo('*')) + .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many()))) + .Select(s => s.ToStringValue()); + + static readonly TextParser Capture = + from _ in Character.EqualTo('{') + from name in CaptureName.OptionalOrDefault() + from type in Character.EqualTo(':') + .IgnoreThen(CaptureType) + .OptionalOrDefault() + where name != null || type != null + from __ in Character.EqualTo('}') + select new CapturePatternExpression(name, type); + + static readonly TextParser Element = + LiteralText.Cast() + .Or(Capture.Cast()); + + static readonly TextParser Pattern = + Element.AtLeastOnce().AtEnd().Select(e => new ExtractionPattern(e)); + + public static ExtractionPattern Parse(string extractionPattern) + { + if (extractionPattern == null) throw new ArgumentNullException(nameof(extractionPattern)); + if (extractionPattern == "") throw new ParseException("Zero-length extraction patterns are not allowed."); + return Pattern.Parse(extractionPattern); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs new file mode 100644 index 00000000..a8399e81 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/LiteralTextPatternExpression.cs @@ -0,0 +1,14 @@ +using System; + +namespace SeqCli.PlainText.Patterns +{ + class LiteralTextPatternExpression : ExtractionPatternExpression + { + public string Text { get; } + + public LiteralTextPatternExpression(string text) + { + Text = text ?? throw new ArgumentNullException(nameof(text)); + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs index 8f413c4b..e6e68f28 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -2,7 +2,9 @@ using System.IO; using System.Threading.Tasks; using SeqCli.Ingestion; +using SeqCli.PlainText.Extraction; using SeqCli.PlainText.Parsers; +using SeqCli.PlainText.Patterns; using Serilog.Events; namespace SeqCli.PlainText @@ -11,13 +13,16 @@ class PlainTextLogEventReader : ILogEventReader, IDisposable { static readonly TimeSpan TrailingLineArrivalDeadline = TimeSpan.FromMilliseconds(10); - readonly Pattern _pattern; + readonly NameValueExtractor _nameValueExtractor; readonly FrameReader _reader; - public PlainTextLogEventReader(TextReader input) + public PlainTextLogEventReader(TextReader input, string extractionPattern) { - _pattern = PatternBuilder.DefaultPattern; - _reader = new FrameReader(input, SpanEx.MatchedBy(_pattern.FrameStart), TrailingLineArrivalDeadline); + _nameValueExtractor = string.IsNullOrEmpty(extractionPattern) ? + PatternCompiler.MultilineMessageExtractor : + PatternCompiler.Compile(ExtractionPatternParser.Parse(extractionPattern)); + + _reader = new FrameReader(input, SpanEx.MatchedBy(_nameValueExtractor.StartMarker), TrailingLineArrivalDeadline); } public async Task TryReadAsync() @@ -29,7 +34,7 @@ public async Task TryReadAsync() if (frame.IsOrphan) throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`."); - var (properties, remainder) = _pattern.Match(frame.Value); + var (properties, remainder) = _nameValueExtractor.ExtractValues(frame.Value); return LogEventBuilder.FromProperties(properties, remainder); } diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs new file mode 100644 index 00000000..47dc3448 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -0,0 +1,59 @@ +using System; +using System.Linq; +using SeqCli.PlainText.Patterns; +using Superpower; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternParserTests + { + [Fact] + public void ARegularStringIsASingleTextLiteral() + { + var pattern = ExtractionPatternParser.Parse("Hello!"); + Assert.Single(pattern.Elements); + var tt = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("Hello!", tt.Text); + } + + [Fact] + public void CaptureNameAndTypeAreParsed() + { + var pattern = ExtractionPatternParser.Parse("{abc:def}"); + Assert.Single(pattern.Elements); + var ct = Assert.IsType(pattern.Elements.Single()); + Assert.Equal("abc", ct.Name); + Assert.Equal("def", ct.Type); + } + + [Theory] + [InlineData("", false)] + [InlineData("{}", false)] + [InlineData("{a", false)] + [InlineData("a", true)] + [InlineData("{a}", true)] + [InlineData("{@m}", true)] + [InlineData("{@m:n}", true)] + [InlineData("{@m:*}", true)] + [InlineData("{@m:n}", true)] + [InlineData("{m_N}", true)] + [InlineData("{_9}", true)] + [InlineData("{:n}", true)] + [InlineData("{:}", false)] + [InlineData("{{@m}}", true)] + [InlineData("{{a", true)] + [InlineData("a}}", true)] + [InlineData("{", false)] + [InlineData("}", false)] + [InlineData("{a} b{c} ", true)] + [InlineData("d {a}b {c}", true)] + public void OnlyValidPatternsAreAccepted(string attempt, bool isValid) + { + if (isValid) + ExtractionPatternParser.Parse(attempt); + else + Assert.Throws(() => ExtractionPatternParser.Parse(attempt)); + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/PatternTests.cs index e899cc4b..7d8a9d28 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/PatternTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; using Superpower.Model; using Xunit; @@ -12,7 +13,7 @@ public class PatternTests public void TheDefaultPatternMatchesMultilineMessages() { var frame = $"Hello,{Environment.NewLine} world!"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); Assert.Null(remainder); Assert.Single(properties, p => p.Key == ReifiedProperties.Message && ((TextSpan)p.Value).ToStringValue() == frame); @@ -22,16 +23,16 @@ public void TheDefaultPatternMatchesMultilineMessages() public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() { var frame = " world"; - var (properties, remainder) = PatternBuilder.DefaultPattern.Match(frame); + var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); Assert.Empty(properties); Assert.Equal(frame, remainder); } - static Pattern ClassMethodPattern { get; } = new Pattern(new[] + static NameValueExtractor ClassMethodPattern { get; } = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "class"), - new PatternElement(BuiltInPatterns.LiteralText(".")), - new PatternElement(BuiltInPatterns.Identifier, "method") + new PatternElement(Matchers.Identifier, "class"), + new PatternElement(Matchers.LiteralText(".")), + new PatternElement(Matchers.Identifier, "method") }); [Fact] @@ -40,7 +41,7 @@ public void PatternsExtractElements() var pattern = ClassMethodPattern; var frame = "this.that"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("this", properties["class"].ToString()); Assert.Equal("that", properties["method"].ToString()); @@ -49,24 +50,24 @@ public void PatternsExtractElements() [Fact] public void TheFirstPatternElementIsExposed() { - Assert.Same(BuiltInPatterns.Identifier, ClassMethodPattern.FrameStart); + Assert.Same(Matchers.Identifier, ClassMethodPattern.StartMarker); } [Fact] public void SingleLineContentMatchesUntilEol() { - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.SingleLineContent, "content"), - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.SingleLineContent, "content"), + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi (jkl)", properties["content"].ToString()); @@ -79,20 +80,20 @@ public void NonGreedyContentStopsMatchingWhenFollowingTokensMatch() // the "following" list, since they effectively become "mandatory" var following = new[] { - new PatternElement(BuiltInPatterns.LiteralText(" (")), - new PatternElement(BuiltInPatterns.Identifier, "last"), - new PatternElement(BuiltInPatterns.LiteralText(")")) + new PatternElement(Matchers.LiteralText(" (")), + new PatternElement(Matchers.Identifier, "last"), + new PatternElement(Matchers.LiteralText(")")) }; - var pattern = new Pattern(new[] + var pattern = new NameValueExtractor(new[] { - new PatternElement(BuiltInPatterns.Identifier, "first"), - new PatternElement(BuiltInPatterns.LiteralText(" ")), - new PatternElement(BuiltInPatterns.NonGreedyContent(following), "content"), + new PatternElement(Matchers.Identifier, "first"), + new PatternElement(Matchers.LiteralText(" ")), + new PatternElement(Matchers.NonGreedyContent(following), "content"), }.Concat(following)); var frame = "abc def ghi (jkl)"; - var (properties, remainder) = pattern.Match(frame); + var (properties, remainder) = pattern.ExtractValues(frame); Assert.Null(remainder); Assert.Equal("abc", properties["first"].ToString()); Assert.Equal("def ghi", properties["content"].ToString()); From c424b98e991225a8039916b77924ee03ee4a75ed Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Fri, 23 Feb 2018 21:32:32 +1000 Subject: [PATCH 2/5] -p already taken for properties --- src/SeqCli/Cli/Commands/IngestCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SeqCli/Cli/Commands/IngestCommand.cs b/src/SeqCli/Cli/Commands/IngestCommand.cs index ed7817ae..900cec5a 100644 --- a/src/SeqCli/Cli/Commands/IngestCommand.cs +++ b/src/SeqCli/Cli/Commands/IngestCommand.cs @@ -47,7 +47,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory) _invalidDataHandlingFeature = Enable(); _properties = Enable(); - Options.Add("p=|pattern=", + Options.Add("x=|extract=", "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)", v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim()); From 7f1703c0dcbeecfcfcbbc0702ef12de40aca0c36 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 07:32:14 +1000 Subject: [PATCH 3/5] More placeholder tests, WIP --- src/SeqCli/PlainText/Extraction/Matchers.cs | 24 +++++++++++--- src/SeqCli/PlainText/Parsers/NumericsEx.cs | 21 +++++++++++++ src/SeqCli/PlainText/Parsers/StringsEx.cs | 16 ++++++++++ .../PlainText/PatternCompilerTests.cs | 31 +++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 src/SeqCli/PlainText/Parsers/NumericsEx.cs create mode 100644 src/SeqCli/PlainText/Parsers/StringsEx.cs create mode 100644 test/SeqCli.Tests/PlainText/PatternCompilerTests.cs diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 9f5c9fcc..0107057c 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -19,12 +19,27 @@ static class Matchers [Matcher("nat")] public static TextParser Natural { get; } = - Numerics.Natural + Numerics.NaturalUInt64 .Select(span => (object) span); [Matcher("int")] public static TextParser Integer { get; } = - Numerics.Integer + Numerics.IntegerInt64 + .Select(span => (object) span); + + [Matcher("dec")] + public static TextParser Decimal { get; } = + NumericsEx.Decimal + .Select(span => (object) span); + + [Matcher("alpha")] + public static TextParser Alphabetical { get; } = + Span.WithAll(char.IsLetter) + .Select(span => (object) span); + + [Matcher("alphanum")] + public static TextParser Alphanumeric { get; } = + Span.WithAll(char.IsLetterOrDigit) .Select(span => (object) span); [Matcher("token")] @@ -74,7 +89,8 @@ public static TextParser LiteralText(string literalText) public static TextParser NonGreedyContent(params PatternElement[] following) { if (following.Length == 0) - return SpanEx.MatchedBy(Character.AnyChar.Many()).Select(span => (object) span); + return SpanEx.MatchedBy(Character.AnyChar.Many()) + .Select(span => span.Length > 0 ? (object) span : null); var rest = following[0].Parser; for (var i = 1; i < following.Length; ++i) @@ -91,7 +107,7 @@ public static TextParser NonGreedyContent(params PatternElement[] follow } var span = i.Until(remainder); - return Result.Value((object) span, i, remainder); + return Result.Value(span.Length > 0 ? (object) span : null, i, remainder); }; } } diff --git a/src/SeqCli/PlainText/Parsers/NumericsEx.cs b/src/SeqCli/PlainText/Parsers/NumericsEx.cs new file mode 100644 index 00000000..1810372b --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/NumericsEx.cs @@ -0,0 +1,21 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class NumericsEx + { + public static TextParser Decimal { get; } = + Numerics.Integer + .Then(n => Character.EqualTo('.').IgnoreThen(Numerics.Integer).OptionalOrDefault() + .Select(f => f == TextSpan.None ? n : new TextSpan(n.Source, n.Position, n.Length + f.Length + 1))); + + public static TextParser HexNatural { get; } = + SpanEx.MatchedBy(Span.EqualTo("0x") + .IgnoreThen(Character.Digit + .Or(Character.Matching(ch => ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F', "a-f")) + .Named("hex digit") + .AtLeastOnce())); + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Parsers/StringsEx.cs b/src/SeqCli/PlainText/Parsers/StringsEx.cs new file mode 100644 index 00000000..a39ffff9 --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/StringsEx.cs @@ -0,0 +1,16 @@ +using Superpower; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class StringsEx + { + static readonly TextParser SqlStringContentChar = + Span.EqualTo("''").Value('\'').Try().Or(Character.ExceptIn('\'', '\r', '\n')); + + public static TextParser SqlStyle { get; } = + Character.EqualTo('\'') + .IgnoreThen(SqlStringContentChar.Many()) + .Then(s => Character.EqualTo('\'').Value(new string(s))); + } +} diff --git a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs new file mode 100644 index 00000000..9aa4d9ab --- /dev/null +++ b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs @@ -0,0 +1,31 @@ +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class PatternCompilerTests + { + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() + { + // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting + + var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:lines}"; + } + + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() + { + // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-console#output-templates + + var pattern = "[{@t:localtime} {@l:ident}] {@m:*}{:n}{@x:lines}"; + } + + [Fact] + public void OptionalSourceContextCanBeExtracted() + { + var pattern = "[{@t} {@l:ident}] ({SourceContext:*}) {@m:*}{:n}{@x:lines}"; + } + } +} \ No newline at end of file From cbe73366b53041ab27a821b0c6813563323bebfc Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 14:01:38 +1000 Subject: [PATCH 4/5] Enough working to parse default Serilog.Sinks.File formatted events --- ...der.cs => ExtractionPatternInterpreter.cs} | 11 ++- src/SeqCli/PlainText/Extraction/Matchers.cs | 19 ++++- .../PlainText/LogEvents/LogEventBuilder.cs | 18 +++-- src/SeqCli/PlainText/Parsers/DateTimesEx.cs | 12 +++ .../PlainText/PlainTextLogEventReader.cs | 4 +- .../ExtractionPatternInterpreterTests.cs | 74 +++++++++++++++++++ .../PlainText/LogEventBuilderTests.cs | 8 ++ ...ernTests.cs => NameValueExtractorTests.cs} | 6 +- .../PlainText/PatternCompilerTests.cs | 31 -------- 9 files changed, 135 insertions(+), 48 deletions(-) rename src/SeqCli/PlainText/Extraction/{PatternBuilder.cs => ExtractionPatternInterpreter.cs} (89%) create mode 100644 src/SeqCli/PlainText/Parsers/DateTimesEx.cs create mode 100644 test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs rename test/SeqCli.Tests/PlainText/{PatternTests.cs => NameValueExtractorTests.cs} (93%) delete mode 100644 test/SeqCli.Tests/PlainText/PatternCompilerTests.cs diff --git a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs similarity index 89% rename from src/SeqCli/PlainText/Extraction/PatternBuilder.cs rename to src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs index 0ae2aae7..68e7d302 100644 --- a/src/SeqCli/PlainText/Extraction/PatternBuilder.cs +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -4,19 +4,20 @@ namespace SeqCli.PlainText.Extraction { - static class PatternCompiler + static class ExtractionPatternInterpreter { public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[] { new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message) }); - public static NameValueExtractor Compile(ExtractionPattern pattern) + public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern) { if (pattern == null) throw new ArgumentNullException(nameof(pattern)); var patternElements = new PatternElement[pattern.Elements.Count]; - for (var i = 0; i < pattern.Elements.Count; ++i) + var last = true; + for (var i = pattern.Elements.Count - 1; i >= 0; --i) { var element = pattern.Elements[i]; switch (element) @@ -25,7 +26,7 @@ public static NameValueExtractor Compile(ExtractionPattern pattern) patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); break; case CapturePatternExpression capture when capture.Type == "*": - if (i < pattern.Elements.Count - 1) + if (!last) patternElements[i] = new PatternElement( Matchers.NonGreedyContent(patternElements[i + 1]), capture.Name); @@ -42,6 +43,8 @@ public static NameValueExtractor Compile(ExtractionPattern pattern) default: throw new InvalidOperationException($"Element `{element}` not recognized."); } + + last = false; } return new NameValueExtractor(patternElements); diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 0107057c..1b8232de 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Reflection; using SeqCli.PlainText.Parsers; @@ -46,6 +47,20 @@ static class Matchers public static TextParser Token { get; } = SpanEx.NonWhiteSpace.Select(span => (object)span); + [Matcher("iso8601dt")] + // A date and time are required by this pattern, though not necessarily by the spec. + public static TextParser Iso8601DateTime { get; } = + DateTimesEx.Iso8601DateTime + .Select(span => (object) span); + + public static TextParser SerilogFileTimestamp { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?") + .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)); + + [Matcher("timestamp")] + public static TextParser Timestamp { get; } = + Iso8601DateTime.Try().Or(SerilogFileTimestamp); + // Unclear whether we need to name this public static TextParser MultiLineMessage { get; } = SpanEx.MatchedBy( @@ -101,9 +116,11 @@ public static TextParser NonGreedyContent(params PatternElement[] follow return i => { var remainder = i; - while (!rest.IsMatch(remainder)) + var attempt = rest(remainder); + while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything { remainder = remainder.ConsumeChar().Remainder; + attempt = rest(remainder); } var span = i.Until(remainder); diff --git a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs index f5b252c2..786b27cd 100644 --- a/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs +++ b/src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs @@ -87,13 +87,17 @@ static IEnumerable GetLogEventProperties(IDictionary properties) { - var timestamp = properties.TryGetValue(ReifiedProperties.Timestamp, out var t) && - t is TextSpan span && - DateTimeOffset.TryParse(span.ToStringValue(), CultureInfo.InvariantCulture, - DateTimeStyles.AssumeLocal, out var ts) - ? ts - : DateTimeOffset.Now; - return timestamp; + if (properties.TryGetValue(ReifiedProperties.Timestamp, out var t)) + { + if (t is TextSpan span && DateTimeOffset.TryParse(span.ToStringValue(), + CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var ts)) + return ts; + + if (t is DateTimeOffset dto) + return dto; + } + + return DateTimeOffset.Now; } static readonly Dictionary LevelsByName = new Dictionary(StringComparer.OrdinalIgnoreCase) diff --git a/src/SeqCli/PlainText/Parsers/DateTimesEx.cs b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs new file mode 100644 index 00000000..7653505a --- /dev/null +++ b/src/SeqCli/PlainText/Parsers/DateTimesEx.cs @@ -0,0 +1,12 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; + +namespace SeqCli.PlainText.Parsers +{ + static class DateTimesEx + { + public static TextParser Iso8601DateTime { get; } = + Span.Regex("\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?(([+-]\\d\\d:\\d\\d)|Z)?"); + } +} diff --git a/src/SeqCli/PlainText/PlainTextLogEventReader.cs b/src/SeqCli/PlainText/PlainTextLogEventReader.cs index e6e68f28..af1cac3a 100644 --- a/src/SeqCli/PlainText/PlainTextLogEventReader.cs +++ b/src/SeqCli/PlainText/PlainTextLogEventReader.cs @@ -19,8 +19,8 @@ class PlainTextLogEventReader : ILogEventReader, IDisposable public PlainTextLogEventReader(TextReader input, string extractionPattern) { _nameValueExtractor = string.IsNullOrEmpty(extractionPattern) ? - PatternCompiler.MultilineMessageExtractor : - PatternCompiler.Compile(ExtractionPatternParser.Parse(extractionPattern)); + ExtractionPatternInterpreter.MultilineMessageExtractor : + ExtractionPatternInterpreter.CreateNameValueExtractor(ExtractionPatternParser.Parse(extractionPattern)); _reader = new FrameReader(input, SpanEx.MatchedBy(_nameValueExtractor.StartMarker), TrailingLineArrivalDeadline); } diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs new file mode 100644 index 00000000..6b555a83 --- /dev/null +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Globalization; +using SeqCli.PlainText; +using SeqCli.PlainText.Extraction; +using SeqCli.PlainText.Patterns; +using Xunit; + +namespace SeqCli.Tests.PlainText +{ + public class ExtractionPatternInterpreterTests + { + [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() + { + // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting + + // {@l:ident} is required so that the default "token" pattern doesn't greedily eat up the `]`. + // "timestamp" is intended to be an aggregate timestamp parser that tries ISO 8601, RFC 2822, and various other + // popular timestamp formats. + + var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}"; + + var parsed = ExtractionPatternParser.Parse(pattern); + var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); + + var (properties, remainder) = extractor.ExtractValues( + @"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed +System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +"); + + Assert.Equal( + DateTimeOffset.ParseExact("2018-02-21 13:29:00.123 +10:00", "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture), + properties["@t"]); + Assert.Equal("ERR", properties["@l"].ToString()); + Assert.Equal("The operation failed", properties["@m"].ToString()); + Assert.Equal(@"System.DivideByZeroException: Attempt to divide by zero + at SomeClass.SomeMethod() +", properties["@x"].ToString()); + Assert.Null(remainder); + } + + // Work-in-progress... + + // [Fact] + public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() + { + // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + // See: https://github.com/serilog/serilog-sinks-console#output-templates + + // "localtime" will add the closest non-future date to the time component that is matched + // by the pattern + + // The pattern language needs to be extended here so that the brackets, timestamp, spacing and + // level are all used as the start-frame marker. The strawman syntax proposes that to the + // right of `:` will always be either an alphanumeric matcher name, or a subexpression. This + // does have the issue that `{?:foo}` would be ambiguous (optional 'foo' matcher or optional 'foo' + // literal, so some escaping would be necessary - e.g. `{?:\foo}` to indicate a literal 'foo' and + // `{?:\*}` for an optional literal asterisk, `{?:\\}` for an optional literal backslash. + + var pattern = "{:[{@t:localtime} {@l:ident}] }{@m:*}{:n}{@x:*}"; + } + + // [Fact] + public void OptionalSourceContextCanBeExtracted() + { + // The {?: optional grouping is just an anonymous optional property, e.g. if the formatting was + // not dynamic, it might be written {SourceContext?:*}; using the grouping means the surrounding + // whitespace and parens are required only if the optional group is matched. + var pattern = "{:[{@t} {@l:ident}] }{?:({SourceContext:*}) }{@m:*}{:n}{@x:*}"; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs index 50a67383..7cf38310 100644 --- a/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs +++ b/test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs @@ -45,5 +45,13 @@ public void MissingValuesAreDefaulted() Assert.Null(evt.Exception); Assert.Empty(evt.Properties); } + + [Fact] + public void DateTimeOffsetTimestampsAreAccepted() + { + var then = DateTimeOffset.Now.AddDays(-5); + var evt = LogEventBuilder.FromProperties(new Dictionary{["@t"] = then}, null); + Assert.Equal(then, evt.Timestamp); + } } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/PatternTests.cs b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs similarity index 93% rename from test/SeqCli.Tests/PlainText/PatternTests.cs rename to test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs index 7d8a9d28..785ede83 100644 --- a/test/SeqCli.Tests/PlainText/PatternTests.cs +++ b/test/SeqCli.Tests/PlainText/NameValueExtractorTests.cs @@ -7,13 +7,13 @@ namespace SeqCli.Tests.PlainText { - public class PatternTests + public class NameValueExtractorTests { [Fact] public void TheDefaultPatternMatchesMultilineMessages() { var frame = $"Hello,{Environment.NewLine} world!"; - var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Null(remainder); Assert.Single(properties, p => p.Key == ReifiedProperties.Message && ((TextSpan)p.Value).ToStringValue() == frame); @@ -23,7 +23,7 @@ public void TheDefaultPatternMatchesMultilineMessages() public void TheDefaultPatternDoesNotMatchLinesStartingWithWhitespace() { var frame = " world"; - var (properties, remainder) = PatternCompiler.MultilineMessageExtractor.ExtractValues(frame); + var (properties, remainder) = ExtractionPatternInterpreter.MultilineMessageExtractor.ExtractValues(frame); Assert.Empty(properties); Assert.Equal(frame, remainder); } diff --git a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs b/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs deleted file mode 100644 index 9aa4d9ab..00000000 --- a/test/SeqCli.Tests/PlainText/PatternCompilerTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using Xunit; - -namespace SeqCli.Tests.PlainText -{ - public class PatternCompilerTests - { - [Fact] - public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() - { - // This is the default format: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}" - // See: https://github.com/serilog/serilog-sinks-file#controlling-event-formatting - - var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:lines}"; - } - - [Fact] - public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() - { - // This is the default format: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" - // See: https://github.com/serilog/serilog-sinks-console#output-templates - - var pattern = "[{@t:localtime} {@l:ident}] {@m:*}{:n}{@x:lines}"; - } - - [Fact] - public void OptionalSourceContextCanBeExtracted() - { - var pattern = "[{@t} {@l:ident}] ({SourceContext:*}) {@m:*}{:n}{@x:lines}"; - } - } -} \ No newline at end of file From cf1647d74e6bd0cb4f55f7d8724d7611ab944929 Mon Sep 17 00:00:00 2001 From: Nicholas Blumhardt Date: Sat, 24 Feb 2018 20:15:23 +1000 Subject: [PATCH 5/5] Multiple-token non-greedy lookahead --- .../ExtractionPatternInterpreter.cs | 23 +++++-------- src/SeqCli/PlainText/Extraction/Matchers.cs | 7 +++- .../Patterns/CaptureContentExpression.cs | 6 ++++ .../Patterns/CapturePatternExpression.cs | 12 +++---- .../Patterns/ExtractionPatternParser.cs | 16 ++++----- .../Patterns/MatchTypeContentExpression.cs | 12 +++++++ .../Patterns/NonGreedyContentExpression.cs | 12 +++++++ .../ExtractionPatternInterpreterTests.cs | 33 +++++++++++++++---- .../PlainText/ExtractionPatternParserTests.cs | 7 +++- 9 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs create mode 100644 src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs diff --git a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs index 68e7d302..16834b93 100644 --- a/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs +++ b/src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using SeqCli.PlainText.Patterns; namespace SeqCli.PlainText.Extraction @@ -16,7 +17,6 @@ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern patt if (pattern == null) throw new ArgumentNullException(nameof(pattern)); var patternElements = new PatternElement[pattern.Elements.Count]; - var last = true; for (var i = pattern.Elements.Count - 1; i >= 0; --i) { var element = pattern.Elements[i]; @@ -25,26 +25,21 @@ public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern patt case LiteralTextPatternExpression text: patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text)); break; - case CapturePatternExpression capture when capture.Type == "*": - if (!last) - patternElements[i] = new PatternElement( - Matchers.NonGreedyContent(patternElements[i + 1]), - capture.Name); - else - patternElements[i] = new PatternElement( - Matchers.NonGreedyContent(), // <- same as MultiLineContent - capture.Name); + case CapturePatternExpression capture + when capture.Content is NonGreedyContentExpression ngc: + patternElements[i] = new PatternElement( + Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()), + capture.Name); break; - case CapturePatternExpression capture: + case CapturePatternExpression capture + when capture.Content is MatchTypeContentExpression mtc: patternElements[i] = new PatternElement( - capture.Type == null ? Matchers.Token : Matchers.GetByType(capture.Type), + mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type), capture.Name); break; default: throw new InvalidOperationException($"Element `{element}` not recognized."); } - - last = false; } return new NameValueExtractor(patternElements); diff --git a/src/SeqCli/PlainText/Extraction/Matchers.cs b/src/SeqCli/PlainText/Extraction/Matchers.cs index 1b8232de..ec2725bd 100644 --- a/src/SeqCli/PlainText/Extraction/Matchers.cs +++ b/src/SeqCli/PlainText/Extraction/Matchers.cs @@ -68,7 +68,7 @@ static class Matchers .IgnoreThen(Character.AnyChar.Many())) .Select(span => (object)span); - [Matcher("lines")] + // Equivalent to :* at end-of-pattern public static TextParser MultiLineContent { get; } = Span.WithAll(ch => true) .Select(span => (object)span); @@ -84,6 +84,11 @@ from _ in NewLine.OptionalOrDefault() Span.EqualTo("\r\n").Or(Span.EqualTo("\n")) .Select(span => (object)span); + [Matcher("t")] + public static TextParser Tab { get; } = + Span.EqualTo("\t") + .Select(span => (object)span); + static readonly Dictionary> ByType = new Dictionary>( from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties let attr = pi.GetCustomAttribute() diff --git a/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs new file mode 100644 index 00000000..ef4d8ec1 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/CaptureContentExpression.cs @@ -0,0 +1,6 @@ +namespace SeqCli.PlainText.Patterns +{ + abstract class CaptureContentExpression + { + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs index def5648d..4e76bca5 100644 --- a/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs +++ b/src/SeqCli/PlainText/Patterns/CapturePatternExpression.cs @@ -1,16 +1,14 @@ -using System; - -namespace SeqCli.PlainText.Patterns +namespace SeqCli.PlainText.Patterns { class CapturePatternExpression : ExtractionPatternExpression { public string Name { get; } - public string Type { get; } + public CaptureContentExpression Content { get; } - public CapturePatternExpression(string name, string type) + public CapturePatternExpression(string name, CaptureContentExpression content) { Name = name; - Type = type; + Content = content; } } -} \ No newline at end of file +} diff --git a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs index 1038967a..493edb1d 100644 --- a/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs +++ b/src/SeqCli/PlainText/Patterns/ExtractionPatternParser.cs @@ -20,21 +20,21 @@ static class ExtractionPatternParser .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) .Select(s => s.ToStringValue()); - static readonly TextParser CaptureType = - SpanEx.MatchedBy(Character.EqualTo('*')) + static readonly TextParser CaptureContent = + Character.EqualTo('*').AtLeastOnce().Select(chs => (CaptureContentExpression)new NonGreedyContentExpression(chs.Length)) .Or(SpanEx.MatchedBy(Character.Letter.Or(Character.EqualTo('_')) - .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many()))) - .Select(s => s.ToStringValue()); + .IgnoreThen(Character.LetterOrDigit.Or(Character.EqualTo('_')).Many())) + .Select(s => (CaptureContentExpression)new MatchTypeContentExpression(s.ToStringValue()))); static readonly TextParser Capture = from _ in Character.EqualTo('{') from name in CaptureName.OptionalOrDefault() - from type in Character.EqualTo(':') - .IgnoreThen(CaptureType) + from content in Character.EqualTo(':') + .IgnoreThen(CaptureContent) .OptionalOrDefault() - where name != null || type != null + where name != null || content != null from __ in Character.EqualTo('}') - select new CapturePatternExpression(name, type); + select new CapturePatternExpression(name, content); static readonly TextParser Element = LiteralText.Cast() diff --git a/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs new file mode 100644 index 00000000..1a0f9bf9 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/MatchTypeContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class MatchTypeContentExpression : CaptureContentExpression + { + public string Type { get; } + + public MatchTypeContentExpression(string type) + { + Type = type; + } + } +} \ No newline at end of file diff --git a/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs new file mode 100644 index 00000000..bb4a2969 --- /dev/null +++ b/src/SeqCli/PlainText/Patterns/NonGreedyContentExpression.cs @@ -0,0 +1,12 @@ +namespace SeqCli.PlainText.Patterns +{ + class NonGreedyContentExpression : CaptureContentExpression + { + public int Lookahead { get; } + + public NonGreedyContentExpression(int lookahead) + { + Lookahead = lookahead; + } + } +} \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs index 6b555a83..f14b1b77 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternInterpreterTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using SeqCli.PlainText; using SeqCli.PlainText.Extraction; @@ -9,6 +10,21 @@ namespace SeqCli.Tests.PlainText { public class ExtractionPatternInterpreterTests { + static (IDictionary, string) ExtractValues(string pattern, string candidate) + { + var parsed = ExtractionPatternParser.Parse(pattern); + var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); + return extractor.ExtractValues(candidate); + } + + [Fact] + public void NonGreedyMatchCanLookaheadMultipleTokens() + { + var (properties, remainder) = ExtractValues("[{test:**}]!", "[0]abc[1]!"); + Assert.Null(remainder); + Assert.Equal("0]abc[1", properties["test"].ToString()); + } + [Fact] public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() { @@ -20,15 +36,14 @@ public void TheMatchingPatternCanExtractDefaultSerilogFileOutput() // popular timestamp formats. var pattern = "{@t:timestamp} [{@l:ident}] {@m:*}{:n}{@x:*}"; - - var parsed = ExtractionPatternParser.Parse(pattern); - var extractor = ExtractionPatternInterpreter.CreateNameValueExtractor(parsed); - - var (properties, remainder) = extractor.ExtractValues( - @"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed + + var candidate = +@"2018-02-21 13:29:00.123 +10:00 [ERR] The operation failed System.DivideByZeroException: Attempt to divide by zero at SomeClass.SomeMethod() -"); +"; + + var (properties, remainder) = ExtractValues(pattern, candidate); Assert.Equal( DateTimeOffset.ParseExact("2018-02-21 13:29:00.123 +10:00", "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture), @@ -59,7 +74,9 @@ public void TheMatchingPatternCanExtractDefaultSerilogConsoleOutput() // literal, so some escaping would be necessary - e.g. `{?:\foo}` to indicate a literal 'foo' and // `{?:\*}` for an optional literal asterisk, `{?:\\}` for an optional literal backslash. +#pragma warning disable 219 var pattern = "{:[{@t:localtime} {@l:ident}] }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 } // [Fact] @@ -68,7 +85,9 @@ public void OptionalSourceContextCanBeExtracted() // The {?: optional grouping is just an anonymous optional property, e.g. if the formatting was // not dynamic, it might be written {SourceContext?:*}; using the grouping means the surrounding // whitespace and parens are required only if the optional group is matched. +#pragma warning disable 219 var pattern = "{:[{@t} {@l:ident}] }{?:({SourceContext:*}) }{@m:*}{:n}{@x:*}"; +#pragma warning restore 219 } } } \ No newline at end of file diff --git a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs index 47dc3448..85fd84fc 100644 --- a/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs +++ b/test/SeqCli.Tests/PlainText/ExtractionPatternParserTests.cs @@ -24,7 +24,7 @@ public void CaptureNameAndTypeAreParsed() Assert.Single(pattern.Elements); var ct = Assert.IsType(pattern.Elements.Single()); Assert.Equal("abc", ct.Name); - Assert.Equal("def", ct.Type); + Assert.Equal("def", ((MatchTypeContentExpression)ct.Content).Type); } [Theory] @@ -34,8 +34,13 @@ public void CaptureNameAndTypeAreParsed() [InlineData("a", true)] [InlineData("{a}", true)] [InlineData("{@m}", true)] + [InlineData("{@@m}", false)] + [InlineData("{m@}", false)] [InlineData("{@m:n}", true)] [InlineData("{@m:*}", true)] + [InlineData("{@m:***}", true)] + [InlineData("{:*}", true)] + [InlineData("{a:}", false)] [InlineData("{@m:n}", true)] [InlineData("{m_N}", true)] [InlineData("{_9}", true)]