diff --git a/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs b/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs index 19488703f07f7e..23fb622ac705eb 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs +++ b/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs @@ -735,11 +735,15 @@ private static void EmitTryFindNextPossibleStartingPosition(IndentedTextWriter w case FindNextStartingPositionMode.LeadingString_LeftToRight: case FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight: case FindNextStartingPositionMode.FixedDistanceString_LeftToRight: - EmitIndexOf_LeftToRight(); + EmitIndexOfString_LeftToRight(); break; case FindNextStartingPositionMode.LeadingString_RightToLeft: - EmitIndexOf_RightToLeft(); + EmitIndexOfString_RightToLeft(); + break; + + case FindNextStartingPositionMode.LeadingStrings_LeftToRight: + EmitIndexOfStrings_LeftToRight(); break; case FindNextStartingPositionMode.LeadingSet_LeftToRight: @@ -964,7 +968,7 @@ bool EmitAnchors() } // Emits a case-sensitive left-to-right search for a substring. - void EmitIndexOf_LeftToRight() + void EmitIndexOfString_LeftToRight() { RegexFindOptimizations opts = regexTree.FindOptimizations; @@ -1010,8 +1014,43 @@ void EmitIndexOf_LeftToRight() } } + // Emits a case-sensitive left-to-right search for any one of multiple leading prefixes. + void EmitIndexOfStrings_LeftToRight() + { + RegexFindOptimizations opts = regexTree.FindOptimizations; + Debug.Assert(opts.FindMode == FindNextStartingPositionMode.LeadingStrings_LeftToRight); + + string prefixes = string.Join(", ", opts.LeadingPrefixes.Select(prefix => Literal(prefix))); + + string fieldName = "s_indexOfAnyStrings_"; + using (SHA256 sha = SHA256.Create()) + { +#pragma warning disable CA1850 // SHA256.HashData isn't available on netstandard2.0 + fieldName += $"{BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(prefixes))).Replace("-", "")}"; +#pragma warning restore CA1850 + } + + if (!requiredHelpers.ContainsKey(fieldName)) + { + requiredHelpers.Add(fieldName, new string[] + { + $"/// Supports searching for any of the strings {EscapeXmlComment(prefixes)}.", + $"internal static readonly SearchValues {fieldName} = SearchValues.Create(new[] {{ {prefixes} }}, StringComparison.Ordinal);", + }); + } + + writer.WriteLine($"// The pattern has multiple strings that could begin the match. Search for any of them."); + writer.WriteLine($"// If none can be found, there's no match."); + writer.WriteLine($"int i = inputSpan.Slice(pos).IndexOfAny({HelpersTypeName}.{fieldName});"); + using (EmitBlock(writer, "if (i >= 0)")) + { + writer.WriteLine("base.runtextpos = pos + i;"); + writer.WriteLine("return true;"); + } + } + // Emits a case-sensitive right-to-left search for a substring. - void EmitIndexOf_RightToLeft() + void EmitIndexOfString_RightToLeft() { string prefix = regexTree.FindOptimizations.LeadingPrefix; diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunner.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunner.cs index b58f584662ed9a..827c6d5384b5d6 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunner.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunner.cs @@ -1,34 +1,35 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Globalization; +#pragma warning disable CA1823, CS0169, IDE0044 // Fields used via reflection + namespace System.Text.RegularExpressions { internal sealed class CompiledRegexRunner : RegexRunner { private readonly ScanDelegate _scanMethod; - private readonly SearchValues[]? _searchValues; + /// Set if the regex uses any SearchValues instances. Accessed via reflection. + /// If the array is non-null, this contains instances of SearchValues{char} or SearchValues{string}. + private readonly object[]? _searchValues; - /// This field will only be set if the pattern contains backreferences and has RegexOptions.IgnoreCase + /// Set if the pattern contains backreferences and has RegexOptions.IgnoreCase. Accessed via reflection. private readonly CultureInfo? _culture; -#pragma warning disable CA1823, CS0169, IDE0044 // Used via reflection to cache the Case behavior if needed. + /// Caches a RegexCaseBehavior. Accessed via reflection. private RegexCaseBehavior _caseBehavior; -#pragma warning restore CA1823, CS0169, IDE0044 internal delegate void ScanDelegate(RegexRunner runner, ReadOnlySpan text); - public CompiledRegexRunner(ScanDelegate scan, SearchValues[]? searchValues, CultureInfo? culture) + public CompiledRegexRunner(ScanDelegate scan, object[]? searchValues, CultureInfo? culture) { _scanMethod = scan; _searchValues = searchValues; _culture = culture; } - protected internal override void Scan(ReadOnlySpan text) - => _scanMethod(this, text); + protected internal override void Scan(ReadOnlySpan text) => _scanMethod(this, text); } } diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunnerFactory.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunnerFactory.cs index 6784f068d1c559..83eb957327847e 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunnerFactory.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/CompiledRegexRunnerFactory.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Globalization; using System.Reflection.Emit; @@ -10,14 +9,14 @@ namespace System.Text.RegularExpressions internal sealed class CompiledRegexRunnerFactory : RegexRunnerFactory { private readonly DynamicMethod _scanMethod; - private readonly SearchValues[]? _searchValues; + private readonly object[]? _searchValues; /// This field will only be set if the pattern has backreferences and uses RegexOptions.IgnoreCase private readonly CultureInfo? _culture; // Delegate is lazily created to avoid forcing JIT'ing until the regex is actually executed. private CompiledRegexRunner.ScanDelegate? _scan; - public CompiledRegexRunnerFactory(DynamicMethod scanMethod, SearchValues[]? searchValues, CultureInfo? culture) + public CompiledRegexRunnerFactory(DynamicMethod scanMethod, object[]? searchValues, CultureInfo? culture) { _scanMethod = scanMethod; _searchValues = searchValues; diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs index 20ccc3afefcca6..8491cd831ed144 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs @@ -71,6 +71,7 @@ internal abstract class RegexCompiler private static readonly MethodInfo s_spanIndexOfAnyCharCharChar = typeof(MemoryExtensions).GetMethod("IndexOfAny", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), Type.MakeGenericMethodParameter(0), Type.MakeGenericMethodParameter(0), Type.MakeGenericMethodParameter(0) })!.MakeGenericMethod(typeof(char)); private static readonly MethodInfo s_spanIndexOfAnySpan = typeof(MemoryExtensions).GetMethod("IndexOfAny", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)) })!.MakeGenericMethod(typeof(char)); private static readonly MethodInfo s_spanIndexOfAnySearchValues = typeof(MemoryExtensions).GetMethod("IndexOfAny", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), typeof(SearchValues<>).MakeGenericType(Type.MakeGenericMethodParameter(0)) })!.MakeGenericMethod(typeof(char)); + private static readonly MethodInfo s_spanIndexOfAnySearchValuesString = typeof(MemoryExtensions).GetMethod("IndexOfAny", new Type[] { typeof(ReadOnlySpan), typeof(SearchValues) })!; private static readonly MethodInfo s_spanIndexOfAnyExceptChar = typeof(MemoryExtensions).GetMethod("IndexOfAnyExcept", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), Type.MakeGenericMethodParameter(0) })!.MakeGenericMethod(typeof(char)); private static readonly MethodInfo s_spanIndexOfAnyExceptCharChar = typeof(MemoryExtensions).GetMethod("IndexOfAnyExcept", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), Type.MakeGenericMethodParameter(0), Type.MakeGenericMethodParameter(0) })!.MakeGenericMethod(typeof(char)); private static readonly MethodInfo s_spanIndexOfAnyExceptCharCharChar = typeof(MemoryExtensions).GetMethod("IndexOfAnyExcept", new Type[] { typeof(ReadOnlySpan<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), Type.MakeGenericMethodParameter(0), Type.MakeGenericMethodParameter(0), Type.MakeGenericMethodParameter(0) })!.MakeGenericMethod(typeof(char)); @@ -114,8 +115,8 @@ internal abstract class RegexCompiler /// Whether this expression has a non-infinite timeout. protected bool _hasTimeout; - /// instances used by the expression. For now these are only ASCII sets. - protected List>? _searchValues; + /// instances used by the expression. + protected List? _searchValues; /// Pool of Int32 LocalBuilders. private Stack? _int32LocalsPool; @@ -459,6 +460,7 @@ protected void EmitTryFindNextPossibleStartingPosition() { case FindNextStartingPositionMode.LeadingString_LeftToRight: case FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight: + case FindNextStartingPositionMode.LeadingStrings_LeftToRight: case FindNextStartingPositionMode.FixedDistanceString_LeftToRight: EmitIndexOf_LeftToRight(); break; @@ -744,15 +746,15 @@ bool EmitAnchors() return false; } - // Emits a case-sensitive left-to-right search for a substring. + // Emits a case-sensitive left-to-right search for a substring or substrings. void EmitIndexOf_LeftToRight() { RegexFindOptimizations opts = _regexTree.FindOptimizations; - Debug.Assert(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight or FindNextStartingPositionMode.FixedDistanceString_LeftToRight); + Debug.Assert(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight or FindNextStartingPositionMode.FixedDistanceString_LeftToRight or FindNextStartingPositionMode.LeadingStrings_LeftToRight); using RentedLocalBuilder i = RentInt32Local(); - // int i = inputSpan.Slice(pos).IndexOf(prefix); + // int i = inputSpan.Slice(pos)... Ldloca(inputSpan); Ldloc(pos); if (opts.FindMode is FindNextStartingPositionMode.FixedDistanceString_LeftToRight && @@ -762,18 +764,28 @@ void EmitIndexOf_LeftToRight() Add(); } Call(s_spanSliceIntMethod); - Ldstr(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? - opts.LeadingPrefix : - opts.FixedDistanceLiteral.String!); - Call(s_stringAsSpanMethod); - if (opts.FindMode is FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight) + + // ...IndexOf(prefix); + if (opts.FindMode == FindNextStartingPositionMode.LeadingStrings_LeftToRight) { - Ldc((int)StringComparison.OrdinalIgnoreCase); - Call(s_spanIndexOfSpanStringComparison); + LoadSearchValues(opts.LeadingPrefixes); + Call(s_spanIndexOfAnySearchValuesString); } else { - Call(s_spanIndexOfSpan); + Ldstr(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? + opts.LeadingPrefix : + opts.FixedDistanceLiteral.String!); + Call(s_stringAsSpanMethod); + if (opts.FindMode is FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight) + { + Ldc((int)StringComparison.OrdinalIgnoreCase); + Call(s_spanIndexOfSpanStringComparison); + } + else + { + Call(s_spanIndexOfSpan); + } } Stloc(i); @@ -967,12 +979,12 @@ void EmitFixedSet_LeftToRight() // a sequential walk). In order to do that search, we actually build up a set for all of the ASCII // characters _not_ contained in the set, and then do a search for the inverse of that, which will be // all of the target ASCII characters and all of non-ASCII. - var asciiChars = new List(); - for (int i = 0; i <= 0x7f; i++) + using var asciiChars = new ValueListBuilder(stackalloc char[128]); + for (int i = 0; i < 128; i++) { if (!RegexCharClass.CharInClass((char)i, primarySet.Set)) { - asciiChars.Add((char)i); + asciiChars.Append((char)i); } } @@ -984,7 +996,7 @@ void EmitFixedSet_LeftToRight() // int i = span. Ldloc(span); - if (asciiChars.Count == 128) + if (asciiChars.Length == 128) { // IndexOfAnyExceptInRange('\0', '\u007f'); Ldc(0); @@ -994,7 +1006,7 @@ void EmitFixedSet_LeftToRight() else { // IndexOfAnyExcept(searchValuesArray[...]); - LoadSearchValues(CollectionsMarshal.AsSpan(asciiChars)); + LoadSearchValues(asciiChars.AsSpan().ToArray()); Call(s_spanIndexOfAnyExceptSearchValues); } Stloc(i); @@ -6112,13 +6124,16 @@ private void EmitTimeoutCheckIfNeeded() } /// - /// Adds an entry in for the given and emits a load of that initialized value. + /// Adds an entry in for the given and emits a load of that initialized value. /// - private void LoadSearchValues(ReadOnlySpan chars) + private void LoadSearchValues(T[] values) { - List> list = _searchValues ??= new(); + List list = _searchValues ??= new(); int index = list.Count; - list.Add(SearchValues.Create(chars)); + list.Add( + typeof(T) == typeof(char) ? SearchValues.Create((char[])(object)values) : + typeof(T) == typeof(string) ? SearchValues.Create((string[])(object)values, StringComparison.Ordinal) : + throw new UnreachableException()); // Logically do _searchValues[index], but avoid the bounds check on accessing the array, // and cast to the known derived sealed type to enable devirtualization. diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs index 517c9da6b4270c..72d3f19aef4044 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs @@ -139,6 +139,17 @@ public RegexFindOptimizations(RegexNode root, RegexOptions options) // We're now left-to-right only and looking for sets. + // If there are multiple leading strings, we can search for any of them. + if (compiled) + { + if (RegexPrefixAnalyzer.FindPrefixes(root) is { Length: > 1 } prefixes) + { + LeadingPrefixes = prefixes; + FindMode = FindNextStartingPositionMode.LeadingStrings_LeftToRight; + return; + } + } + // Build up a list of all of the sets that are a fixed distance from the start of the expression. List? fixedDistanceSets = RegexPrefixAnalyzer.FindFixedDistanceSets(root, thorough: !interpreter); Debug.Assert(fixedDistanceSets is null || fixedDistanceSets.Count != 0); @@ -244,6 +255,9 @@ public RegexFindOptimizations(RegexNode root, RegexOptions options) /// Gets the leading prefix. May be an empty string. public string LeadingPrefix { get; } = string.Empty; + /// Gets the leading prefixes. May be an empty array. + public string[] LeadingPrefixes { get; } = Array.Empty(); + /// When in fixed distance literal mode, gets the literal and how far it is from the start of the pattern. public (char Char, string? String, int Distance) FixedDistanceLiteral { get; } @@ -773,10 +787,15 @@ public bool TryFindNextStartingPositionLeftToRight(ReadOnlySpan textSpan, return false; } + // Not supported in the interpreter, but we could end up here for patterns so complex the compiler gave up on them. + + case FindNextStartingPositionMode.LeadingStrings_LeftToRight: + return true; + // Nothing special to look for. Just return true indicating this is a valid position to try to match. default: - Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch); + Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch, $"Unexpected FindMode {FindMode}"); return true; } } @@ -816,6 +835,9 @@ internal enum FindNextStartingPositionMode /// A multi-character ordinal case-insensitive substring at the beginning of the pattern. LeadingString_OrdinalIgnoreCase_LeftToRight, + /// Multiple leading prefix strings + LeadingStrings_LeftToRight, + /// A set starting the pattern. LeadingSet_LeftToRight, /// A set starting the right-to-left pattern. diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs index bf3c03acb517d8..227448dd83b778 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs @@ -11,6 +11,329 @@ namespace System.Text.RegularExpressions /// Detects various forms of prefixes in the regular expression that can help FindFirstChars optimize its search. internal static class RegexPrefixAnalyzer { + /// Cache of ToString() strings for the ASCII chars. + /// The strings are lazily created on first use. + private static string[]? s_asciiCharStrings; + + /// Gets the ToString() string for the specified char. + private static string GetCharString(char ch) + { + // If the character isn't ASCII, just ToString it. + if (ch >= 128) + { + return ch.ToString(); + } + + // Use a lazily-initialized cache of strings for ASCII characters. The overall cache is initialized + // with Interlocked.CompareExchange simply to avoid accidentally throwing out a lot of strings accidentally. + + string[] asciiCharString = + s_asciiCharStrings ?? + Interlocked.CompareExchange(ref s_asciiCharStrings, new string[128], null) ?? + s_asciiCharStrings; + + return asciiCharString[ch] ??= ch.ToString(); + } + + /// Finds an array of multiple prefixes that a node can begin with. + /// The node to search. + /// + /// If a fixed set of prefixes is found, such that a match for this node is guaranteed to begin + /// with one of those prefixes, an array of those prefixes is returned. Otherwise, null. + /// + public static string[]? FindPrefixes(RegexNode node) + { + // Minimum string length for prefixes to be useful. If any prefix has length 1, + // then we're generally better off just using IndexOfAny with chars. + const int MinPrefixLength = 2; + + // Arbitrary string length limit to avoid creating strings that are longer than is useful and consuming too much memory. + // Some of the algorithms used also create strings by appending a single character at a time, which is O(N^2) in the + // final length of the string, so keeping this small helps avoid problematic complexity. + const int MaxPrefixLength = 8; + + // Limit based on what IndexOfAny is able to handle most efficiently. This can be updated in future if/when IndexOfAny improves further. + const int MaxPrefixes = 64; + + // Analyze the node to find prefixes. + string[]? results = FindPrefixesCore(node); + if (results is not null) + { + // If the number found is larger than our limit, we didn't find a valid set, so return null. + if (results.Length > MaxPrefixes) + { + return null; + } + + // If any of the prefixes is shorter than the required minimum, we didn't find a valid set, so + // return null. + foreach (string result in results) + { + if (result.Length < MinPrefixLength) + { + return null; + } + } + } + + // Return the prefixes if we got any, or null if we didn't. + return results; + + static string[]? FindPrefixesCore(RegexNode node) + { + // If we're too deep to analyze further, we can't trust what we've already computed, so give up. + if (!StackHelper.TryEnsureSufficientExecutionStack()) + { + return null; + } + + // These limits are approximations. We'll stop trying to make strings longer once we exceed the max length, + // and if we exceed the max number of prefixes by a non-trivial amount, we'll fail the operation. + Span setChars = stackalloc char[MaxPrefixes]; // limit how many chars we get from a set based on the max prefixes we care about + Span scratch = stackalloc char[32]; // arbitrary limit + + // Loop down the left side of the tree, looking for a starting node we can handle. We only loop through + // atomic and capture nodes, as the child is guaranteed to execute once, as well as loops with a positive + // minimum and thus at least one guaranteed iteration. + while (true) + { + switch (node.Kind) + { + case RegexNodeKind.Atomic: + case RegexNodeKind.Capture: + case RegexNodeKind.Loop or RegexNodeKind.Lazyloop when node.M > 0: + // These nodes are all guaranteed to execute at least once, so we can just + // skip through them to their child. + node = node.Child(0); + break; + + case RegexNodeKind.Alternate: + // For alternations, we need to find a prefix for every branch; if we can't compute a + // prefix for any one branch, we can't trust the results and need to give up, since we don't + // know if our set of prefixes is complete. + { + // If there are more children than our maximum, just give up immediately, as we + // won't be able to get a prefix for every branch and have it be within our max. + int childCount = node.ChildCount(); + if (childCount > MaxPrefixes) + { + return null; + } + + List? prefixes = null; + for (int i = 0; i < childCount; i++) + { + // Get the prefix for the branch. + string[]? childPrefixes = FindPrefixesCore(node.Child(i)); + + // If we couldn't get one, or if its results puts us over the max, give up. + if (childPrefixes is null || + (childPrefixes.Length + (prefixes?.Count ?? 0) > MaxPrefixes)) + { + return null; + } + + // Otherwise, add the prefixes to our list. + (prefixes ??= new()).AddRange(childPrefixes); + } + + Debug.Assert(prefixes is not null && prefixes.Count is > 0 and <= MaxPrefixes); + return prefixes.ToArray(); + } + + case RegexNodeKind.One: + // If we hit a single character, we can just return that character. + return new string[] { GetCharString(node.Ch) }; + + case RegexNodeKind.Multi: + // If we hit a string, we can just return that string. + return new string[] { node.Str! }; + + case RegexNodeKind.Set when !RegexCharClass.IsNegated(node.Str!): + // If we hit a non-negated set (negated sets are too complex to analyze), + // we can try to extract the characters that comprise it, and if there are + // any and there aren't more than the max number of prefixes, we can return + // them each as a prefix. Effectively, this is an alternation of the characters + // that comprise the set. + { + string[]? results = null; + int charCount = RegexCharClass.GetSetChars(node.Str!, setChars); + if (charCount != 0) + { + results = new string[charCount]; + for (int i = 0; i < charCount; i++) + { + results[i] = GetCharString(setChars[i]); + } + } + return results; + } + + case RegexNodeKind.Concatenate: + { + // For concatenations, we specially recognize certain kinds of children and fail on others. + // Concatenations can get complex in the face of multiple children that branch. To simplify + // and handle the 95% case, we only process up a single branch within the concatenation. + // Once we've seen such a branch (a set or an alternation), the next time we see one we stop trying + // to lengthen the prefixes and just return what we have. For example, given the pattern + // `ab[cd]ef[gh]`, we'll create the prefixes "abcef" and "abdef". Thus, we maintain two pieces of state: a + // ValueStringBuilder for the current prefix in the concatenation if we haven't set branched, or a + // list of strings for the prefixes if we have branched. Once we switch over to using concatStrings, + // the ValueStringBuilder is no longer used. + var vsb = new ValueStringBuilder(scratch); + try + { + string[]? concatStrings = null; + + int childCount = node.ChildCount(); + for (int i = 0; i < childCount && vsb.Length < MaxPrefixLength; i++) + { + // Atomic and Capture nodes don't impact prefixes, so skip through them. + // Unlike earlier, however, we can't skip through loops, as a loop with + // more than one iteration impacts the matched sequence for the concatenation, + // and since we need a minimum of one, we'd only be able to skip a loop with + // both a min and max of 1, which in general is removed as superfluous during + // tree optimization. + RegexNode child = SkipThroughAtomicAndCapture(node.Child(i)); + + switch (child.Kind) + { + case RegexNodeKind.One: + // For a single char, if we haven't branched yet, just append it to the ValueStringBuilder. + // Otherwise, append it to every string in our branched list. + if (concatStrings is null) + { + vsb.Append(child.Ch); + } + else + { + for (int s = 0; s < concatStrings.Length; s++) + { + concatStrings[s] += GetCharString(child.Ch); + } + } + break; + + case RegexNodeKind.Multi: + // For a string, if we haven't branched yet, just append it to the ValueStringBuilder. + // Otherwise, append it to every string in our branched list. + if (concatStrings is null) + { + vsb.Append(child.Str); + } + else + { + for (int s = 0; s < concatStrings.Length; s++) + { + concatStrings[s] += child.Str; + } + } + break; + + case RegexNodeKind.Set when concatStrings is null && !RegexCharClass.IsNegated(child.Str!): // if we've already branched for a set, don't again + // For a set when we haven't branched yet, get the characters that comprise it, and if we + // were able to get any and it's under our maximum, upgrade from our ValueStringBuilder to + // the concatStrings list. This means + { + int charCount = RegexCharClass.GetSetChars(child.Str!, setChars); + if (charCount == 0) + { + // Couldn't process the set, so we can't continue to process the concatenation. + goto default; + } + + ReadOnlySpan sliced = setChars.Slice(0, charCount); + concatStrings = new string[sliced.Length]; + if (vsb.Length == 0) + { + // We don't have any prefix yet, so just add each set's char as its own string. + for (int c = 0; c < sliced.Length; c++) + { + concatStrings[c] = GetCharString(sliced[c]); + } + } + else + { + // We have a prefix, so for each char in the set, add a string comprised of + // the current prefix plus that char. + for (int c = 0; c < sliced.Length; c++) + { + vsb.Append(sliced[c]); + concatStrings[c] = vsb.AsSpan().ToString(); + vsb.Length--; + } + } + } + break; + + case RegexNodeKind.Alternate when concatStrings is null: // if we've already branched, don't again + // Process the alternation. Since we haven't yet branched, if we don't have any prefix yet + // for this concatenation, we can just use the alternation's prefixes as our prefixes. + // If we do have a prefix, we need to prepend that prefix onto each of the alternation's. + { + concatStrings = FindPrefixesCore(child); + if (concatStrings is not null) + { + if (vsb.Length != 0) + { + int currentLength = vsb.Length; + for (int c = 0; c < concatStrings.Length; c++) + { + vsb.Append(concatStrings[c]); + concatStrings[c] = vsb.AsSpan().ToString(); + vsb.Length = currentLength; + } + } + } + + // If we failed to find any prefixes, we need to stop processing the concatenation. + // But even if we found prefixes, we still need to stop processing the concatenation, + // as the prefixes aren't guaranteed to be the full width of the child, in which case + // the node coming after the altneration in the concatenation might not be contiguous + // after every prefix. + goto default; + } + + case RegexNodeKind.Bol: + case RegexNodeKind.Eol: + case RegexNodeKind.Boundary: + case RegexNodeKind.ECMABoundary: + case RegexNodeKind.NonBoundary: + case RegexNodeKind.NonECMABoundary: + case RegexNodeKind.Beginning: + case RegexNodeKind.Start: + case RegexNodeKind.EndZ: + case RegexNodeKind.End: + case RegexNodeKind.Empty: + case RegexNodeKind.UpdateBumpalong: + case RegexNodeKind.PositiveLookaround: + case RegexNodeKind.NegativeLookaround: + // Zero-width anchors and assertions don't impact the prefix. + break; + + default: + i = childCount; // exit the loop + break; + } + } + + // Return multiple prefixes if we branched, or a single prefix if we didn't, + // or null if we didn't find any prefix. + return concatStrings ?? (vsb.Length != 0 ? new[] { vsb.ToString() } : null); + } + finally + { + vsb.Dispose(); + } + } + + default: + return null; + } + } + } + } + /// Computes the leading substring in ; may be empty. public static string FindPrefix(RegexNode node) { @@ -787,10 +1110,7 @@ public static (RegexNode LoopNode, (char Char, string? String, char[]? Chars) Li // Find the first concatenation. We traverse through atomic and capture nodes as they don't effect flow control. (We don't // want to explore loops, even if they have a guaranteed iteration, because we may use information about the node to then // skip the node's execution in the matching algorithm, and we would need to special-case only skipping the first iteration.) - while (node.Kind is RegexNodeKind.Atomic or RegexNodeKind.Capture) - { - node = node.Child(0); - } + node = SkipThroughAtomicAndCapture(node); if (node.Kind != RegexNodeKind.Concatenate) { return null; @@ -963,6 +1283,16 @@ private static RegexNodeKind FindLeadingOrTrailingAnchor(RegexNode node, bool le } } + /// Walk through a node's children as long as the nodes are atomic or capture. + private static RegexNode SkipThroughAtomicAndCapture(RegexNode node) + { + while (node.Kind is RegexNodeKind.Atomic or RegexNodeKind.Capture) + { + node = node.Child(0); + } + return node; + } + /// Percent occurrences in source text (100 * char count / total count). private static ReadOnlySpan Frequency => new float[] { diff --git a/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs b/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs index d4520ba19746cd..aff4874dfaa1f0 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs @@ -70,6 +70,40 @@ public void FindFirstCharClass_StressDeep() FindFirstCharClass(string.Concat(Enumerable.Repeat($"(a?", nesting).Concat(Enumerable.Repeat(")*", nesting))), 0, null); } + [Theory] + [InlineData("abc|def", new[] { "abc", "def" })] + [InlineData("abc|def|(ghi|jklm)", new[] { "abc", "def", "ghi", "jklm" })] + [InlineData("abc[def]ghi", new[] { "abcdghi", "abceghi", "abcfghi" })] + [InlineData("abc[def]ghi|[jkl]m", new[] { "abcdghi", "abceghi", "abcfghi", "jm", "km", "lm" })] + [InlineData("agggtaaa|tttaccct", new[] { "agggtaaa", "tttaccct" })] + [InlineData("[cgt]gggtaaa|tttaccc[acg]", new[] { "cgggtaaa", "ggggtaaa", "tgggtaaa", "tttaccca", "tttacccc", "tttacccg" })] + [InlineData("a[act]ggtaaa|tttacc[agt]t", new[] { "aaggtaaa", "acggtaaa", "atggtaaa", "tttaccat", "tttaccgt", "tttacctt" })] + [InlineData("ag[act]gtaaa|tttac[agt]ct", new[] { "agagtaaa", "agcgtaaa", "agtgtaaa", "tttacact", "tttacgct", "tttactct" })] + [InlineData("agg[act]taaa|ttta[agt]cct", new[] { "aggataaa", "aggctaaa", "aggttaaa", "tttaacct", "tttagcct", "tttatcct" })] + [InlineData("\b(abc|def)\b", new[] { "abc", "def" })] + [InlineData("^(abc|def)$", new[] { "abc", "def" })] + [InlineData("abcdefg|h", null)] + [InlineData("abc[def]ghi|[jkl]", null)] + public void FindPrefixes(string pattern, string[] expectedSet) + { + RegexTree tree = RegexParser.Parse(pattern, RegexOptions.None, CultureInfo.InvariantCulture); + string[] actual = RegexPrefixAnalyzer.FindPrefixes(tree.Root); + + if (expectedSet is null) + { + Assert.Null(actual); + } + else + { + Assert.NotNull(actual); + + Array.Sort(actual, StringComparer.Ordinal); + Array.Sort(expectedSet, StringComparer.Ordinal); + + Assert.Equal(expectedSet, actual); + } + } + private static string FormatSet(string set) { if (set is null)