diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs index b4673ca55..9a0abc7fc 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisLanguageRule.cs @@ -10,13 +10,13 @@ internal class AnalysisLanguageRule : IRule { private readonly Morpher _morpher; private readonly List _strata; - private readonly List> _rules; + private readonly List _rules; public AnalysisLanguageRule(Morpher morpher, Language language) { _morpher = morpher; _strata = language.Strata.Reverse().ToList(); - _rules = _strata.Select(stratum => stratum.CompileAnalysisRule(morpher)).ToList(); + _rules = _strata.Select(stratum => new AnalysisStratumRule(morpher, stratum)).ToList(); } public IEnumerable Apply(Word input) @@ -32,9 +32,13 @@ public IEnumerable Apply(Word input) HashSet outputSet = tempSet; outputSet.Clear(); + // Limit alternatives accross all invocations of _rules[i].Apply. + int alternativeCount = 0; + _rules[i].SetMaxAlternatives(_morpher.MaxAlternatives); + foreach (Word inData in inputSet) { - foreach (Word outData in _rules[i].Apply(inData)) + foreach (Word outData in _rules[i].Apply(inData, ref alternativeCount)) { outputSet.Add(outData); results.Add(outData); diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index 36d9557ad..c7176ec1f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -10,11 +10,12 @@ namespace SIL.Machine.Morphology.HermitCrab { internal class AnalysisStratumRule : IRule { - private readonly IRule _mrulesRule; + private readonly RuleCascade _mrulesRule; private readonly IRule _prulesRule; - private readonly IRule _templatesRule; + private readonly RuleBatch _templatesRule; private readonly Stratum _stratum; private readonly Morpher _morpher; + private int _maxAlternatives; public AnalysisStratumRule(Morpher morpher, Stratum stratum) { @@ -99,7 +100,20 @@ private IRule CompilePhonologicalRule(IPhonologicalRule prule, } } + public void SetMaxAlternatives(int maxAlternatives) + { + _maxAlternatives = maxAlternatives; + _mrulesRule.SetMaxAlternatives(maxAlternatives); + _templatesRule.SetMaxAlternatives(maxAlternatives); + } + public IEnumerable Apply(Word input) + { + int alternativeCount = 0; + return Apply(input, ref alternativeCount); + } + + internal IEnumerable Apply(Word input, ref int alternativeCount) { if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.BeginUnapplyStratum(_stratum, input); @@ -125,6 +139,14 @@ public IEnumerable Apply(Word input) _morpher.TraceManager.EndUnapplyStratum(_stratum, input); foreach (Word mruleOutWord in mruleOutWords) { + alternativeCount++; + if (_maxAlternatives > 0 && alternativeCount >= _maxAlternatives) + { + // Not literally a timeout, but serves the same purpose. + // (A literal timeout would produce different results on different machines.) + // Stops before full enumeration because ApplyTemplates and ApplyMorphologicalRules use yield return. + throw new TimeoutException("MaxAlternatives exceeded"); + } // Skip intermediate sources from phonological rules, templates, and morphological rules. mruleOutWord.Source = origInput; if (mergeEquivalentAnalyses) @@ -141,8 +163,6 @@ public IEnumerable Apply(Word input) output.Add(mruleOutWord); if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord); - if (_morpher.MaxUnapplications > 0 && output.Count >= _morpher.MaxUnapplications) - break; } return output; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index e4fd1879d..c452c460a 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -54,7 +54,7 @@ public Morpher(ITraceManager traceManager, Language lang) _analysisRule = lang.CompileAnalysisRule(this); _synthesisRule = lang.CompileSynthesisRule(this); MaxStemCount = 2; - MaxUnapplications = 0; + MaxAlternatives = 0; MergeEquivalentAnalyses = true; LexEntrySelector = entry => true; RuleSelector = rule => true; @@ -76,7 +76,7 @@ public ITraceManager TraceManager /// to make it possible to debug words that take 30 minutes to parse /// because there are too many unapplications. /// - public int MaxUnapplications { get; set; } + public int MaxAlternatives { get; set; } /// /// Merge analyses that have equivalent shapes. @@ -303,6 +303,7 @@ private IEnumerable Synthesize(string word, IEnumerable analyses) private IEnumerable Synthesize(string word, ConcurrentQueue analyses) { var matches = new ConcurrentBag(); + int alternativeCount = 0; Exception exception = null; Parallel.ForEach( Partitioner.Create(0, analyses.Count), @@ -318,6 +319,9 @@ private IEnumerable Synthesize(string word, ConcurrentQueue analyses { foreach (Word alternative in synthesisWord.ExpandAlternatives()) { + alternativeCount++; + if (MaxAlternatives > 0 && alternativeCount >= MaxAlternatives) + throw new TimeoutException("MaxAlternatives exceeded"); foreach (Word validWord in _synthesisRule.Apply(alternative).Where(IsWordValid)) { if (IsMatch(word, validWord)) diff --git a/src/SIL.Machine/Rules/CombinationRuleCascade.cs b/src/SIL.Machine/Rules/CombinationRuleCascade.cs index 25fdfb8c6..1b8909b49 100644 --- a/src/SIL.Machine/Rules/CombinationRuleCascade.cs +++ b/src/SIL.Machine/Rules/CombinationRuleCascade.cs @@ -48,6 +48,7 @@ private void ApplyRules(TData input, HashSet rulesApplied, HashSet o } output.Add(result); + CheckMaxAlternatives(output.Count); } } } diff --git a/src/SIL.Machine/Rules/LinearRuleCascade.cs b/src/SIL.Machine/Rules/LinearRuleCascade.cs index b4e985a33..295450514 100644 --- a/src/SIL.Machine/Rules/LinearRuleCascade.cs +++ b/src/SIL.Machine/Rules/LinearRuleCascade.cs @@ -46,6 +46,7 @@ private bool ApplyRules(TData input, int ruleIndex, HashSet output) { output.Add(result); } + CheckMaxAlternatives(output.Count); applied = true; } diff --git a/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs b/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs index 9a68ea244..981660918 100644 --- a/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs +++ b/src/SIL.Machine/Rules/ParallelCombinationRuleCascade.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using SIL.Machine.Annotations; @@ -35,38 +36,52 @@ public override IEnumerable Apply(TData input) var from = new ConcurrentStack>>(); from.Push(Tuple.Create(input, !MultipleApplication ? new HashSet() : null)); var to = new ConcurrentStack>>(); + Exception exception = null; + int alternativeCount = 0; while (!from.IsEmpty) { to.Clear(); Parallel.ForEach( from, - work => + (work, state) => { - for (int i = 0; i < Rules.Count; i++) + try { - if ((work.Item2 == null || !work.Item2.Contains(i))) + for (int i = 0; i < Rules.Count; i++) { - TData[] results = ApplyRule(Rules[i], i, work.Item1).ToArray(); - if (results.Length > 0) + if ((work.Item2 == null || !work.Item2.Contains(i))) { - output.PushRange(results); + TData[] results = ApplyRule(Rules[i], i, work.Item1).ToArray(); + if (results.Length > 0) + { + output.PushRange(results); + Interlocked.Add(ref alternativeCount, results.Length); + CheckMaxAlternatives(alternativeCount); - Tuple>[] workItems = results - .Where(res => !Comparer.Equals(work.Item1, res)) - .Select(res => - Tuple.Create( - res, - work.Item2 == null ? null : new HashSet(work.Item2) { i } + Tuple>[] workItems = results + .Where(res => !Comparer.Equals(work.Item1, res)) + .Select(res => + Tuple.Create( + res, + work.Item2 == null ? null : new HashSet(work.Item2) { i } + ) ) - ) - .ToArray(); - if (workItems.Length > 0) - to.PushRange(workItems); + .ToArray(); + if (workItems.Length > 0) + to.PushRange(workItems); + } } } } + catch (Exception ex) + { + state.Stop(); + exception = ex; + } } ); + if (exception != null) + throw exception; ConcurrentStack>> temp = from; from = to; to = temp; diff --git a/src/SIL.Machine/Rules/ParallelPermutationRuleCascade.cs b/src/SIL.Machine/Rules/ParallelPermutationRuleCascade.cs index 34961739a..bbd39fca3 100644 --- a/src/SIL.Machine/Rules/ParallelPermutationRuleCascade.cs +++ b/src/SIL.Machine/Rules/ParallelPermutationRuleCascade.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using SIL.Machine.Annotations; @@ -35,30 +36,44 @@ public override IEnumerable Apply(TData input) var from = new ConcurrentStack>(); from.Push(Tuple.Create(input, 0)); var to = new ConcurrentStack>(); + Exception exception = null; + int alternativeCount = 0; while (!from.IsEmpty) { to.Clear(); Parallel.ForEach( from, - work => + (work, state) => { - for (int i = work.Item2; i < Rules.Count; i++) + try { - TData[] results = ApplyRule(Rules[i], i, work.Item1).ToArray(); - if (results.Length > 0) + for (int i = work.Item2; i < Rules.Count; i++) { - output.PushRange(results); + TData[] results = ApplyRule(Rules[i], i, work.Item1).ToArray(); + if (results.Length > 0) + { + output.PushRange(results); + Interlocked.Add(ref alternativeCount, results.Length); + CheckMaxAlternatives(alternativeCount); - Tuple[] workItems = results - .Where(res => !MultipleApplication || !Comparer.Equals(work.Item1, res)) - .Select(res => Tuple.Create(res, MultipleApplication ? i : i + 1)) - .ToArray(); - if (workItems.Length > 0) - to.PushRange(workItems); + Tuple[] workItems = results + .Where(res => !MultipleApplication || !Comparer.Equals(work.Item1, res)) + .Select(res => Tuple.Create(res, MultipleApplication ? i : i + 1)) + .ToArray(); + if (workItems.Length > 0) + to.PushRange(workItems); + } } } + catch (Exception ex) + { + state.Stop(); + exception = ex; + } } ); + if (exception != null) + throw exception; ConcurrentStack> temp = from; from = to; to = temp; diff --git a/src/SIL.Machine/Rules/PermutationRuleCascade.cs b/src/SIL.Machine/Rules/PermutationRuleCascade.cs index b16671f44..2c7768308 100644 --- a/src/SIL.Machine/Rules/PermutationRuleCascade.cs +++ b/src/SIL.Machine/Rules/PermutationRuleCascade.cs @@ -39,6 +39,7 @@ private void ApplyRules(TData input, int ruleIndex, HashSet output) if (!MultipleApplication || !Comparer.Equals(input, result)) ApplyRules(result, MultipleApplication ? i : i + 1, output); output.Add(result); + CheckMaxAlternatives(output.Count); } } } diff --git a/src/SIL.Machine/Rules/PipelineRuleCascade.cs b/src/SIL.Machine/Rules/PipelineRuleCascade.cs index 524300883..899dd34f8 100644 --- a/src/SIL.Machine/Rules/PipelineRuleCascade.cs +++ b/src/SIL.Machine/Rules/PipelineRuleCascade.cs @@ -23,7 +23,10 @@ public override IEnumerable Apply(TData input) outputSet.Clear(); foreach (TData inData in inputSet) + { outputSet.UnionWith(ApplyRule(Rules[i], i, inData)); + CheckMaxAlternatives(outputSet.Count); + } tempSet = inputSet; inputSet = outputSet; diff --git a/src/SIL.Machine/Rules/RuleBatch.cs b/src/SIL.Machine/Rules/RuleBatch.cs index 61249068b..36a264dce 100644 --- a/src/SIL.Machine/Rules/RuleBatch.cs +++ b/src/SIL.Machine/Rules/RuleBatch.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using SIL.Extensions; using SIL.Machine.Annotations; @@ -10,6 +11,7 @@ public class RuleBatch : IRule private readonly List> _rules; private readonly bool _disjunctive; private readonly IEqualityComparer _comparer; + private int _maxAlternatives; public RuleBatch(IEnumerable> rules) : this(rules, EqualityComparer.Default) { } @@ -42,12 +44,19 @@ public bool IsDisjunctive get { return _disjunctive; } } + public void SetMaxAlternatives(int maxAlternatives) + { + _maxAlternatives = maxAlternatives; + } + public virtual IEnumerable Apply(TData input) { var output = new HashSet(_comparer); foreach (IRule rule in _rules) { output.UnionWith(rule.Apply(input)); + if (_maxAlternatives > 0 && output.Count >= _maxAlternatives) + throw new TimeoutException("MaxAlternatives exceeded"); if (_disjunctive && output.Count > 0) return output; } diff --git a/src/SIL.Machine/Rules/RuleCascade.cs b/src/SIL.Machine/Rules/RuleCascade.cs index a139e8ced..e1eed5965 100644 --- a/src/SIL.Machine/Rules/RuleCascade.cs +++ b/src/SIL.Machine/Rules/RuleCascade.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using SIL.Machine.Annotations; using SIL.ObjectModel; @@ -11,6 +12,7 @@ public abstract class RuleCascade : IRule private readonly ReadOnlyList> _rules; private readonly bool _multiApp; private readonly IEqualityComparer _comparer; + protected int _maxAlternatives; protected RuleCascade(IEnumerable> rules) : this(rules, false) { } @@ -47,6 +49,17 @@ public IReadOnlyList> Rules get { return _rules; } } + public void SetMaxAlternatives(int maxApplications) + { + _maxAlternatives = maxApplications; + } + + public void CheckMaxAlternatives(int alternativeCount) + { + if (_maxAlternatives > 0 && alternativeCount >= _maxAlternatives) + throw new TimeoutException("MaxAlternatives exceeded"); + } + public abstract IEnumerable Apply(TData input); protected virtual IEnumerable ApplyRule(IRule rule, int index, TData input) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs index eb8944ad0..905663e0a 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorpherTests.cs @@ -39,6 +39,33 @@ public void AnalyzeWord_CanAnalyze_ReturnsCorrectAnalysis() ); } + [Test] + public void AnalyzeWord_MaxAlternatives() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + var edSuffix = new AffixProcessRule + { + Id = "PAST", + Name = "ed_suffix", + Gloss = "PAST", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "+d") }, + } + ); + Morphophonemic.MorphologicalRules.Add(edSuffix); + + var morpher = new Morpher(TraceManager, Language); + morpher.MaxAlternatives = 1; + + Assert.Throws(() => morpher.AnalyzeWord("sagd")); + } + [Test] public void AnalyzeWord_CanAnalyzeLinear_ReturnsCorrectAnalysis() {