Extraction patterns

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -37,7 +37,7 @@ class IngestCommand : Command
         readonly FileInputFeature _fileInputFeature;
         readonly PropertiesFeature _properties;
         readonly ConnectionFeature _connection;
-        string _filter;
+        string _filter, _pattern;
         bool _json;
 
         public IngestCommand(SeqConnectionFactory connectionFactory)
@@ -47,14 +47,18 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
             _invalidDataHandlingFeature = Enable<InvalidDataHandlingFeature>();
             _properties = Enable<PropertiesFeature>();
 
+            Options.Add("x=|extract=",
+                "An extraction pattern to apply to plain-text logs (ignored when `--json` is specified)",
+                v => _pattern = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
+
             Options.Add("json",
                 "Read the events as JSON (the default assumes plain text)",
                 v => _json = true);
 
             Options.Add("f=|filter=",
                 "Filter expression to select a subset of events",
                 v => _filter = string.IsNullOrWhiteSpace(v) ? null : v.Trim());
-
+            
             _connection = Enable<ConnectionFeature>();
         }
 
@@ -82,7 +86,7 @@ protected override async Task<int> Run()
 
                     var reader = _json ?
                         (ILogEventReader)new ClefLogEventReader(input) :
-                        new PlainTextLogEventReader(input);
+                        new PlainTextLogEventReader(input, _pattern);
 
                     using (reader as IDisposable)
                     {

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.PlainText.Patterns;
+
+namespace SeqCli.PlainText.Extraction
+{
+    static class ExtractionPatternInterpreter
+    {
+        public static NameValueExtractor MultilineMessageExtractor { get; } = new NameValueExtractor(new[]
+        {
+            new PatternElement(Matchers.MultiLineMessage, ReifiedProperties.Message)
+        });
+
+        public static NameValueExtractor CreateNameValueExtractor(ExtractionPattern pattern)
+        {
+            if (pattern == null) throw new ArgumentNullException(nameof(pattern));
+
+            var patternElements = new PatternElement[pattern.Elements.Count];
+            for (var i = pattern.Elements.Count - 1; i >= 0; --i)
+            {
+                var element = pattern.Elements[i];
+                switch (element)
+                {
+                   case LiteralTextPatternExpression text:
+                       patternElements[i] = new PatternElement(Matchers.LiteralText(text.Text));
+                       break;
+                   case CapturePatternExpression capture
+                       when capture.Content is NonGreedyContentExpression ngc:
+                       patternElements[i] = new PatternElement(
+                           Matchers.NonGreedyContent(patternElements.Skip(i + 1).Take(ngc.Lookahead).ToArray()),
+                           capture.Name);
+                       break;
+                   case CapturePatternExpression capture
+                       when capture.Content is MatchTypeContentExpression mtc:
+                       patternElements[i] = new PatternElement(
+                           mtc.Type == null ? Matchers.Token : Matchers.GetByType(mtc.Type),
+                           capture.Name);
+                       break;
+                   default:
+                       throw new InvalidOperationException($"Element `{element}` not recognized.");
+                }
+            }
+
+            return new NameValueExtractor(patternElements);
+        }
+
+        // What we need to do here is:
+        //  - for each parsed token
+        //    - if it's literal text, map it an anonymous PatternElement with
+        //       BuiltInPatterns.LiteralText()
+        //    - otherwise, if it specifies no format, it's a named element with
+        //       the BuiltInPatterns.Token parser
+        //    - if it does specify a format, look up the parser based on the name, except
+        //    - if the format is `$` it is BuiltInPatterns.SingleLineContent
+        //    - if the format is `$$` it is BuiltInPatterns.MultiLineContent
+        //    - if it's `*`, it's BuiltInPatterns.NonGreedyContent() passing the
+        //       parser that follows it
+    }
+}

src/SeqCli/PlainText/Extraction/MatcherAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace SeqCli.PlainText.Extraction
+{
+    [AttributeUsage(AttributeTargets.Property)]
+    class MatcherAttribute : Attribute
+    {
+        public string Name { get; }
+
+        public MatcherAttribute(string name)
+        {
+            Name = name ?? throw new ArgumentNullException(nameof(name));
+        }
+    }
+}

src/SeqCli/PlainText/Extraction/Matchers.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using SeqCli.PlainText.Parsers;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.PlainText.Extraction
+{
+    // ReSharper disable UnusedMember.Global
+    static class Matchers
+    {
+        [Matcher("ident")]
+        public static TextParser<object> Identifier { get; } =
+            IdentifierEx.CStyle
+                .Select(span => (object) span);
+
+        [Matcher("nat")]
+        public static TextParser<object> Natural { get; } =
+            Numerics.NaturalUInt64
+                .Select(span => (object) span);
+
+        [Matcher("int")]
+        public static TextParser<object> Integer { get; } =
+            Numerics.IntegerInt64
+                .Select(span => (object) span);
+
+        [Matcher("dec")]
+        public static TextParser<object> Decimal { get; } =
+            NumericsEx.Decimal
+                .Select(span => (object) span);
+
+        [Matcher("alpha")]
+        public static TextParser<object> Alphabetical { get; } =
+            Span.WithAll(char.IsLetter)
+                .Select(span => (object) span);
+
+        [Matcher("alphanum")]
+        public static TextParser<object> Alphanumeric { get; } =
+            Span.WithAll(char.IsLetterOrDigit)
+                .Select(span => (object) span);
+
+        [Matcher("token")]
+        public static TextParser<object> Token { get; } =
+            SpanEx.NonWhiteSpace.Select(span => (object)span);
+
+        [Matcher("iso8601dt")]
+        // A date and time are required by this pattern, though not necessarily by the spec.
+        public static TextParser<object> Iso8601DateTime { get; } =
+            DateTimesEx.Iso8601DateTime
+                .Select(span => (object) span);
+
+        public static TextParser<object> SerilogFileTimestamp { get; } =
+            Span.Regex("\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d(\\.\\d+)? ([+-]\\d\\d:\\d\\d)?")
+                .Select(span => (object) DateTimeOffset.ParseExact(span.ToStringValue(), "yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture));
+
+        [Matcher("timestamp")]
+        public static TextParser<object> Timestamp { get; } =
+            Iso8601DateTime.Try().Or(SerilogFileTimestamp);
+
+        // Unclear whether we need to name this
+        public static TextParser<object> MultiLineMessage { get; } =
+            SpanEx.MatchedBy(
+                    Character.Matching(ch => !char.IsWhiteSpace(ch), "non whitespace character")
+                        .IgnoreThen(Character.AnyChar.Many()))
+                .Select(span => (object)span);
+
+        // Equivalent to :* at end-of-pattern
+        public static TextParser<object> MultiLineContent { get; } =
+            Span.WithAll(ch => true)
+                .Select(span => (object)span);
+
+        [Matcher("line")]
+        public static TextParser<object> SingleLineContent { get; } =
+            from content in Span.WithoutAny(ch => ch == '\r' || ch == '\n')
+            from _ in NewLine.OptionalOrDefault()
+            select (object) content;
+
+        [Matcher("n")]
+        public static TextParser<object> NewLine { get; } =
+            Span.EqualTo("\r\n").Or(Span.EqualTo("\n"))
+                .Select(span => (object)span);
+
+        [Matcher("t")]
+        public static TextParser<object> Tab { get; } =
+            Span.EqualTo("\t")
+                .Select(span => (object)span);
+
+        static readonly Dictionary<string, TextParser<object>> ByType = new Dictionary<string, TextParser<object>>(
+            from pi in typeof(Matchers).GetTypeInfo().DeclaredProperties
+            let attr = pi.GetCustomAttribute<MatcherAttribute>()
+            where attr != null
+            select KeyValuePair.Create(attr.Name, (TextParser<object>) pi.GetValue(null)));
+
+        public static TextParser<object> GetByType(string type)
+        {
+            if (type == null) throw new ArgumentNullException(nameof(type));
+            return ByType[type];
+        }
+
+        public static TextParser<object> LiteralText(string literalText)
+        {
+            return Span.EqualTo(literalText).Select(span => (object) span);
+        }
+
+        public static TextParser<object> NonGreedyContent(params PatternElement[] following)
+        {
+            if (following.Length == 0)
+                return SpanEx.MatchedBy(Character.AnyChar.Many())
+                    .Select(span => span.Length > 0 ? (object) span : null);
+
+            var rest = following[0].Parser;
+            for (var i = 1; i < following.Length; ++i)
+            {
+                rest = rest.IgnoreThen(following[i].Parser);
+            }
+
+            return i =>
+            {
+                var remainder = i;
+                var attempt = rest(remainder);
+                while (!attempt.HasValue || attempt.Remainder == remainder) // A zero-length match doesn't tell us anything
+                {
+                    remainder = remainder.ConsumeChar().Remainder;
+                    attempt = rest(remainder);
+                }
+
+                var span = i.Until(remainder);
+                return Result.Value(span.Length > 0 ? (object) span : null, i, remainder);
+            };
+        }
+    }
+}

src/SeqCli/PlainText/Pattern.cs → src/SeqCli/PlainText/Extraction/NameValueExtractor.cs

@@ -1,29 +1,28 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using Superpower;
 using Superpower.Model;
 using Superpower.Parsers;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
-    class Pattern
+    class NameValueExtractor
     {
         readonly PatternElement[] _elements;
 
-        public Pattern(IEnumerable<PatternElement> elements)
+        public NameValueExtractor(IEnumerable<PatternElement> elements)
         {
             _elements = elements?.ToArray() ?? throw new ArgumentNullException(nameof(elements));
             if (_elements.Length == 0)
-                throw new ArgumentException("A match pattern must contain at least one element.");            
+                throw new ArgumentException("An extraction pattern must contain at least one element.");            
         }
 
-        public TextParser<object> FrameStart => _elements[0].Parser;
+        public TextParser<object> StartMarker => _elements[0].Parser;
 
-        public (IDictionary<string, object>, string) Match(string frame)
+        public (IDictionary<string, object>, string) ExtractValues(string plainText)
         {
-            var input = new TextSpan(frame);
+            var input = new TextSpan(plainText);
             var result = new Dictionary<string, object>();
 
             var remainder = input;
@@ -42,8 +41,7 @@ public Pattern(IEnumerable<PatternElement> elements)
 
                 if (!element.IsIgnored)
                 {
-                    if (match.Value != null || !element.IsOptional)
-                        result.Add(element.Name, match.Value);                
+                    result.Add(element.Name, match.Value);                
                 }
             }
 

src/SeqCli/PlainText/PatternElement.cs → src/SeqCli/PlainText/Extraction/PatternElement.cs

@@ -1,20 +1,18 @@
 using System;
 using Superpower;
 
-namespace SeqCli.PlainText
+namespace SeqCli.PlainText.Extraction
 {
     class PatternElement
     {
-        public PatternElement(TextParser<object> parser, string name = null, bool isOptional = false)
+        public PatternElement(TextParser<object> parser, string name = null)
         {
             Parser = parser ?? throw new ArgumentNullException(nameof(parser));
             Name = name;
-            IsOptional = isOptional;
         }
 
         public TextParser<object> Parser { get; }
         public string Name { get; }
-        public bool IsOptional { get; }
         public bool IsIgnored => Name == null;
     }
 }