diff --git a/bench/src/main/scala/cats/parse/bench/StringInBench.scala b/bench/src/main/scala/cats/parse/bench/StringInBench.scala index f4307a26..0c6a73ce 100644 --- a/bench/src/main/scala/cats/parse/bench/StringInBench.scala +++ b/bench/src/main/scala/cats/parse/bench/StringInBench.scala @@ -39,7 +39,9 @@ private[parse] class StringInBenchmarks { var radixNode: RadixNode = _ - var stringIn: Parser[Unit] = _ + var stringInV: Parser[Unit] = _ + + var stringInS: Parser[String] = _ var oneOf: Parser[Unit] = _ @@ -60,13 +62,18 @@ private[parse] class StringInBenchmarks { } radixNode = RadixNode.fromStrings(stringsToMatch) - stringIn = Parser.stringIn(stringsToMatch).void + stringInS = Parser.stringIn(stringsToMatch) + stringInV = stringInS.void oneOf = Parser.oneOf(stringsToMatch.map(Parser.string(_))) } @Benchmark - def stringInParse(): Unit = - inputs.foreach(stringIn.parseAll(_)) + def stringInVParse(): Unit = + inputs.foreach(stringInV.parseAll(_)) + + @Benchmark + def stringInSParse(): Unit = + inputs.foreach(stringInS.parseAll(_)) @Benchmark def oneOfParse(): Unit = diff --git a/build.sbt b/build.sbt index 5696119a..dab99733 100644 --- a/build.sbt +++ b/build.sbt @@ -135,18 +135,33 @@ lazy val core = crossProject(JSPlatform, JVMPlatform) if (isScala211) Set.empty else mimaPreviousArtifacts.value }, mimaBinaryIssueFilters ++= { + /* + * It is okay to filter anything in Impl or RadixNode which are private + */ if (tlIsScala3.value) List( - ProblemFilters.exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#State.error"), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("cats.parse.Parser#State.error_="), - ProblemFilters.exclude[IncompatibleMethTypeProblem]("cats.parse.RadixNode.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.Parser#Impl.mergeCharIn"), + ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.Parser#Impl.mergeStrIn"), + ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.RadixNode.children"), ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.RadixNode.fsts"), ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.RadixNode.prefixes"), - ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.RadixNode.children"), ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.RadixNode.word"), - ProblemFilters.exclude[FinalClassProblem]("cats.parse.RadixNode") + ProblemFilters.exclude[FinalClassProblem]("cats.parse.RadixNode"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("cats.parse.Parser#State.error_="), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("cats.parse.RadixNode.this"), + ProblemFilters + .exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#Impl#StringIn.parseMut"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#Impl.stringIn"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#State.error") + ) + else + List( + ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.Parser#Impl.mergeCharIn"), + ProblemFilters.exclude[DirectMissingMethodProblem]("cats.parse.Parser#Impl.mergeStrIn"), + ProblemFilters + .exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#Impl#StringIn.parseMut"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("cats.parse.Parser#Impl.stringIn") ) - else Nil } ) .jsSettings( diff --git a/core/shared/src/main/scala/cats/parse/Parser.scala b/core/shared/src/main/scala/cats/parse/Parser.scala index 70656240..1cef9c21 100644 --- a/core/shared/src/main/scala/cats/parse/Parser.scala +++ b/core/shared/src/main/scala/cats/parse/Parser.scala @@ -27,6 +27,7 @@ import cats.data.{AndThen, Chain, NonEmptyList} import cats.implicits._ import scala.collection.immutable.SortedSet import scala.collection.mutable.ListBuffer +import scala.annotation.tailrec import java.util.Arrays import scala.collection.immutable.NumericRange @@ -742,7 +743,7 @@ object Parser { } private def mergeInRange(irs: List[InRange]): List[InRange] = { - @annotation.tailrec + @tailrec def merge(rs: List[InRange], aux: Chain[InRange] = Chain.empty): Chain[InRange] = rs match { case x :: y :: rest => @@ -762,14 +763,14 @@ object Parser { Some(OneOfStr(ooss.head.offset, ssb.result().toList)) } - @annotation.tailrec + @tailrec private def stripContext(ex: Expectation): Expectation = ex match { case WithContext(_, inner) => stripContext(inner) case _ => ex } - @annotation.tailrec + @tailrec private def addContext(revCtx: List[String], ex: Expectation): Expectation = revCtx match { case Nil => ex @@ -991,31 +992,56 @@ object Parser { * Note: order matters here, since we don't backtrack by default. */ def oneOf[A](parsers: List[Parser[A]]): Parser[A] = { - @annotation.tailrec - def flatten(ls: List[Parser[A]], acc: ListBuffer[Parser[A]]): List[Parser[A]] = - ls match { - case Nil => acc.toList.distinct - case Impl.OneOf(ps) :: rest => - flatten(ps ::: rest, acc) - case Impl.Fail() :: rest => - flatten(rest, acc) - case notOneOf :: rest => - flatten(rest, acc += notOneOf) - } - - val flattened = flatten(parsers, new ListBuffer) - // we unmap if we can to make merging work better - val isStr = flattened.forall(Impl.matchesString) - val maybeUnmap = if (isStr) flattened.map(Impl.unmap) else flattened - - val cs = Impl.mergeCharIn[Any, Parser[Any]](maybeUnmap) - val res = Impl.mergeStrIn[Any, Parser[Any]](cs) match { - case Nil => fail - case p :: Nil => p - case two => Impl.OneOf(two) - } + @tailrec + def loop(ps: List[Parser[A]], acc: List[Parser[A]]): Parser[A] = + ps match { + case Nil => + /* + * we can still have inner oneof if the head items + * were not oneof and couldn't be merged + * but the last items did have oneof + */ + val flat = acc.reverse.flatMap { + case Impl.OneOf(ps) => ps + case one => one :: Nil + } - (if (isStr) string(res) else res).asInstanceOf[Parser[A]] + flat match { + case Nil => Impl.Fail() + case one :: Nil => one + case many => + many.traverse[Option, Parser[Any]] { + case Impl.StringP(p) => Some(p) + case _ => None + } match { + case Some(m0) => + Impl + .StringP(Impl.OneOf(m0)) + .asInstanceOf[Parser[A]] + case None => + many.traverse[Option, Parser[Any]] { + case Impl.Void(p) => Some(p) + case _ => None + } match { + case Some(m0) => + Impl + .Void(Impl.OneOf(m0)) + .asInstanceOf[Parser[A]] + case None => + Impl.OneOf(many) + } + } + } + case h :: Nil => loop(Nil, h :: acc) + case h1 :: (t1 @ (h2 :: tail2)) => + Impl.merge(h1, h2) match { + case Impl.OneOf(a :: b :: Nil) if (a eq h1) && (b eq h2) => + loop(t1, h1 :: acc) + case h => + loop(h :: tail2, acc) + } + } + loop(parsers, Nil) } /** go through the list of parsers trying each as long as they are epsilon failures (don't @@ -1027,40 +1053,42 @@ object Parser { * * Note: order matters here, since we don't backtrack by default. */ - def oneOf0[A](ps: List[Parser0[A]]): Parser0[A] = { - @annotation.tailrec - def flatten(ls: List[Parser0[A]], acc: ListBuffer[Parser0[A]]): List[Parser0[A]] = - ls match { - case Nil => acc.toList.distinct - case Impl.OneOf0(ps) :: rest => - flatten(ps ::: rest, acc) - case Impl.OneOf(ps) :: rest => - flatten(ps ::: rest, acc) - case Impl.Fail() :: rest => - flatten(rest, acc) - case notOneOf :: rest => - if (Impl.alwaysSucceeds(notOneOf)) { - (acc += notOneOf).toList.distinct - } else { - flatten(rest, acc += notOneOf) - } - } - - val flat0 = flatten(ps, new ListBuffer) - // we unmap if we can to make merging work better - val isStr = flat0.forall(Impl.matchesString) - val flat = if (isStr) flat0.map(Impl.unmap0) else flat0 - - val cs = Impl.mergeCharIn[Any, Parser0[Any]](flat) - val res = Impl.mergeStrIn[Any, Parser0[Any]](cs) match { - case Nil => fail - case p :: Nil => p - case two => Impl.OneOf0(two) + def oneOf0[A](ps: List[Parser0[A]]): Parser0[A] = + if (ps.forall(_.isInstanceOf[Parser[_]])) oneOf(ps.asInstanceOf[List[Parser[A]]]) + else { + @tailrec + def loop(ps: List[Parser0[A]], acc: List[Parser0[A]]): Parser0[A] = + ps match { + case Nil => + /* + * we can still have inner oneof if the head items + * were not oneof and couldn't be merged + * but the last items did have oneof + */ + val flat = acc.reverse.flatMap { + case Impl.OneOf(ps) => ps + case Impl.OneOf0(ps) => ps + case one => one :: Nil + } + flat match { + case Nil => Impl.Fail() + case one :: Nil => one + case many => Impl.OneOf0(many) + } + case h :: Nil => loop(Nil, h :: acc) + case h1 :: (t1 @ (h2 :: tail2)) => + Impl.merge0(h1, h2) match { + case Impl.OneOf0(a :: b :: Nil) if (a eq h1) && (b eq h2) => + loop(t1, h1 :: acc) + case Impl.OneOf(a :: b :: Nil) if (a eq h1) && (b eq h2) => + loop(t1, h1 :: acc) + case h => + loop(h :: tail2, acc) + } + } + loop(ps, Nil) } - (if (isStr) string0(res) else res).asInstanceOf[Parser0[A]] - } - /** Parse the longest matching string between alternatives. The order of the strings does not * matter. * @@ -1077,7 +1105,6 @@ object Parser { .StringIn( SortedSet(two: _*) ) // sadly scala 2.12 doesn't have the `SortedSet.from` constructor function - .string } /** Version of stringIn that allows the empty string @@ -1589,9 +1616,10 @@ object Parser { */ def charWhere(fn: Char => Boolean): Parser[Char] = fn match { - case s: Set[Char] => - // Set extends function, but it is also iterable - charIn(s) + case s: Set[_] => + // Set extends function, so if the fn is a Set it has to be a Set[Char] + // but it is also iterable + charIn(s.asInstanceOf[Set[Char]]) case _ => charIn(Impl.allChars.filter(fn)) } @@ -1647,17 +1675,13 @@ object Parser { */ def void0(pa: Parser0[Any]): Parser0[Unit] = pa match { - case v @ Impl.Void0(_) => v case p1: Parser[_] => void(p1) case s if Impl.alwaysSucceeds(s) => unit + case v @ Impl.Void0(_) => v case _ => - Impl.unmap0(pa) match { - case Impl.StartParser => Impl.StartParser - case Impl.EndParser => Impl.EndParser - case n @ Impl.Not(_) => n - case p @ Impl.Peek(_) => p - case other => Impl.Void0(other) - } + val unmapped = Impl.unmap0(pa) + if (Impl.isVoided(unmapped)) unmapped.asInstanceOf[Parser0[Unit]] + else Impl.Void0(unmapped) } /** discard the value in a Parser. This is an optimization because we remove trailing map @@ -1671,10 +1695,9 @@ object Parser { Impl.unmap(pa) match { case f @ Impl.Fail() => f.widen case f @ Impl.FailWith(_) => f.widen - case p: Impl.Str => p - case p: Impl.StringIn => p - case p: Impl.IgnoreCase => p - case notVoid => Impl.Void(notVoid) + case notVoid => + if (Impl.isVoided(notVoid)) notVoid.asInstanceOf[Parser[Unit]] + else Impl.Void(notVoid) } } @@ -1700,11 +1723,12 @@ object Parser { case str if Impl.matchesString(str) => str.asInstanceOf[Parser[String]] case _ => Impl.unmap(pa) match { + case si @ Impl.StringIn(_) => si case len @ Impl.Length(_) => len case strP @ Impl.Str(expect) => strP.as(expect) - case ci @ Impl.CharIn(min, bs, _) if BitSetUtil.isSingleton(bs) => + case ci @ Impl.SingleChar(c) => // we can allocate the returned string once here - val minStr = min.toChar.toString + val minStr = c.toString Impl.Map(ci, Impl.ConstFn(minStr)) case f @ Impl.Fail() => f.widen case f @ Impl.FailWith(_) => f.widen @@ -1759,6 +1783,7 @@ object Parser { pa match { case p1: Parser[A] => backtrack(p1) case pa if Impl.doesBacktrack(pa) => pa + case Impl.Void0(b) => Impl.Void0(Impl.Backtrack0(b)) case nbt => Impl.Backtrack0(nbt) } @@ -1768,6 +1793,7 @@ object Parser { def backtrack[A](pa: Parser[A]): Parser[A] = pa match { case pa if Impl.doesBacktrack(pa) => pa + case Impl.Void(b) => Impl.Void(Impl.Backtrack(b)) case nbt => Impl.Backtrack(nbt) } @@ -1781,7 +1807,7 @@ object Parser { // void cannot make a Parser0 a Parser // If b is (), such as foo.as(()) // we can just return v - if (b.equals(())) voided.asInstanceOf[Parser0[B]] + if (Impl.isUnit(b)) voided.asInstanceOf[Parser0[B]] else if (Impl.alwaysSucceeds(voided)) pure(b) else Impl.Map0(voided, Impl.ConstFn(b)) } @@ -1792,19 +1818,21 @@ object Parser { val v = pa.void // If b is (), such as foo.as(()) // we can just return v - if (b.equals(())) v.asInstanceOf[Parser[B]] + if (Impl.isUnit(b)) v.asInstanceOf[Parser[B]] else v match { - case Impl.Void(ci @ Impl.CharIn(min, bs, _)) => + case Impl.Void(ci @ Impl.SingleChar(c)) => // CharIn is common and cheap, no need to wrap // with Void since CharIn always returns the char // even when voided b match { - case bc: Char if BitSetUtil.isSingleton(bs) && (min.toChar == bc) => + case bc: Char if bc == c => ci.asInstanceOf[Parser[B]] case _ => Impl.Map(ci, Impl.ConstFn(b)) } + case f @ Impl.Fail() => f.widen + case f @ Impl.FailWith(_) => f.widen case voided => Impl.Map(voided, Impl.ConstFn(b)) } @@ -1813,12 +1841,19 @@ object Parser { /** Add a context string to Errors to aid debugging */ def withContext0[A](p0: Parser0[A], ctx: String): Parser0[A] = - Impl.WithContextP0(ctx, p0) + p0 match { + case Impl.Void0(p) => Impl.Void0(withContext0(p, ctx)).asInstanceOf[Parser0[A]] + case _ if Impl.alwaysSucceeds(p0) => p0 + case _ => Impl.WithContextP0(ctx, p0) + } /** Add a context string to Errors to aid debugging */ def withContext[A](p: Parser[A], ctx: String): Parser[A] = - Impl.WithContextP(ctx, p) + p match { + case Impl.Void(p) => Impl.Void(withContext(p, ctx)) + case _ => Impl.WithContextP(ctx, p) + } implicit val catsInstancesParser : FlatMap[Parser] with Defer[Parser] with MonoidK[Parser] with FunctorFilter[Parser] = @@ -1934,6 +1969,9 @@ object Parser { val allChars = Char.MinValue to Char.MaxValue + def isUnit(a: Any): Boolean = + a.equals(()) + case class ConstFn[A](result: A) extends Function[Any, A] { def apply(any: Any) = result @@ -1962,7 +2000,7 @@ object Parser { final def doesBacktrackCheat(p: Parser0[Any]): Boolean = doesBacktrack(p) - @annotation.tailrec + @tailrec final def doesBacktrack(p: Parser0[Any]): Boolean = p match { case Backtrack0(_) | Backtrack(_) | AnyChar | CharIn(_, _, _) | Str(_) | IgnoreCase(_) | @@ -1974,18 +2012,42 @@ object Parser { case Map(p, _) => doesBacktrack(p) case SoftProd0(a, b) => doesBacktrackCheat(a) && doesBacktrack(b) case SoftProd(a, b) => doesBacktrackCheat(a) && doesBacktrack(b) - case WithContextP(_, p) => doesBacktrack(p) case WithContextP0(_, p) => doesBacktrack(p) + case WithContextP(_, p) => doesBacktrack(p) + case OneOf0(ps) => ps.forall(doesBacktrackCheat(_)) + case OneOf(ps) => ps.forall(doesBacktrackCheat(_)) + case Void0(p) => doesBacktrack(p) + case Void(p) => doesBacktrack(p) case _ => false } + object SingleChar { + def unapply(p: Parser0[Any]): Option[Char] = + p match { + case CharIn(min, bs, _) if BitSetUtil.isSingleton(bs) => Some(min.toChar) + case _ => None + } + } + + object DefiniteString { + def unapply(p: Parser0[Any]): Option[String] = + p match { + case Pure("") => Some("") + case Map(left, ConstFn(res: String)) => + left match { + case Str(s0) if s0 == res => Some(s0) + case SingleChar(c) if (res.length == 1) && (res.charAt(0) == c) => Some(res) + case _ => None + } + case _ => None + } + } // does this parser return the string it matches def matchesString(p: Parser0[Any]): Boolean = p match { - case StringP0(_) | StringP(_) | Pure("") | Length(_) | Fail() | FailWith(_) => true - case Map(Str(e1), ConstFn(e2)) => e1 == e2 - case Map(CharIn(min, bs, _), ConstFn(e)) if BitSetUtil.isSingleton(bs) => - e == min.toChar.toString + case StringP0(_) | StringP(_) | StringIn(_) | Length(_) | Fail() | FailWith(_) | + DefiniteString(_) => + true case OneOf(ss) => ss.forall(matchesString) case OneOf0(ss) => ss.forall(matchesString) case WithContextP(_, p) => matchesString(p) @@ -1993,7 +2055,7 @@ object Parser { case _ => false } - // does this parser always succeed? + // does this parser always succeed without consuming input // note: a parser1 does not always succeed // and by construction, a oneOf0 never always succeeds final def alwaysSucceeds(p: Parser0[Any]): Boolean = @@ -2009,14 +2071,30 @@ object Parser { case _ => false } + // does this parser always eventually succeed (maybe consuming input) + // note, Parser1 has to consume, but may get an empty string, so can't + // always succeed + final def eventuallySucceeds(p: Parser0[Any]): Boolean = + p match { + case Index | GetCaret | Pure(_) => true + case Map0(p, _) => eventuallySucceeds(p) + case SoftProd0(a, b) => eventuallySucceeds(a) && eventuallySucceeds(b) + case Prod0(a, b) => eventuallySucceeds(a) && eventuallySucceeds(b) + case WithContextP0(_, p) => eventuallySucceeds(p) + case OneOf0(ps) => eventuallySucceeds(ps.last) + // by construction we never build a Not(Fail()) since + // it would just be the same as unit + // case Not(Fail() | FailWith(_)) => true + case _ => false + } + val someUnit: Some[Unit] = Some(()) // *if* the parser succeeds, do we know the result? // it may not always suceed final def hasKnownResult[A](p: Parser0[A]): Option[A] = p match { case Pure(a) => Some(a) - case Impl.CharIn(min, bs, _) if BitSetUtil.isSingleton(bs) => - Some(min.toChar.asInstanceOf[A]) + case SingleChar(c) => Some(c.asInstanceOf[A]) case Map0(_, fn) => // scala 3.0.2 seems to fail if we inline // this match above @@ -2071,22 +2149,49 @@ object Parser { case WithContextP0(_, p) => hasKnownResult(p) case Backtrack(p) => hasKnownResult(p) case Backtrack0(p) => hasKnownResult(p) - case Not(_) | Peek(_) | Void(_) | Void0(_) | StartParser | EndParser | Str(_) | StringIn( - _ - ) | IgnoreCase( + case Not(_) | Peek(_) | Void(_) | Void0(_) | StartParser | EndParser | Str(_) | IgnoreCase( _ ) => // these are always unit someUnit.asInstanceOf[Option[A]] case Rep0(_, _, _) | Rep(_, _, _, _) | FlatMap0(_, _) | FlatMap(_, _) | TailRecM(_, _) | - TailRecM0(_, _) | Defer(_) | Defer0(_) | GetCaret | Index | OneOf(_) | OneOf0(_) | - Length(_) | Fail() | FailWith(_) | CharIn(_, _, _) | AnyChar | StringP( + TailRecM0(_, _) | Defer(_) | Defer0(_) | GetCaret | Index | Length(_) | Fail() | + FailWith(_) | CharIn(_, _, _) | AnyChar | StringP( + _ + ) | OneOf(Nil) | OneOf0(Nil) | StringP0(_) | Select(_, _) | Select0(_, _) | StringIn( _ - ) | StringP0(_) | Select(_, _) | Select0(_, _) => + ) => // these we don't know the value fundamentally or by construction None } + /** return true if this is already the same as void + * + * @param p + * the Parser to check + * @return + * true if this parser does not capture + */ + def isVoided(p: Parser0[Any]): Boolean = + p match { + case Pure(a) => isUnit(a) + case StartParser | EndParser | Void(_) | Void0(_) | IgnoreCase(_) | Str(_) | Fail() | + FailWith(_) | Not(_) | Peek(_) => + true + case OneOf(ps) => ps.forall(isVoided(_)) + case OneOf0(ps) => ps.forall(isVoided(_)) + case WithContextP(_, p) => isVoided(p) + case WithContextP0(_, p) => isVoided(p) + case Backtrack(p) => isVoided(p) + case Backtrack0(p) => isVoided(p) + case Length(_) | StringP(_) | StringIn(_) | Prod(_, _) | SoftProd(_, _) | Map(_, _) | + Select(_, _) | FlatMap(_, _) | TailRecM(_, _) | Defer(_) | Rep(_, _, _, _) | AnyChar | + CharIn(_, _, _) | StringP0(_) | Index | GetCaret | Prod0(_, _) | SoftProd0(_, _) | + Map0(_, _) | Select0(_, _) | FlatMap0(_, _) | TailRecM0(_, _) | Defer0(_) | + Rep0(_, _, _) => + false + } + /** This removes any trailing map functions which can cause wasted allocations if we are later * going to void or return strings. This stops at StringP or VoidP since those are markers that * anything below has already been transformed @@ -2117,7 +2222,11 @@ object Parser { // unmap0 may simplify enough // to remove the backtrack wrapper Parser.backtrack0(unmap0(p)) - case OneOf0(ps) => Parser.oneOf0(ps.map(unmap0)) + case OneOf0(ps) => + // Find the fixed point here + val next = Parser.oneOf0(ps.map(unmap0)) + if (next == pa) pa + else unmap0(next) case Prod0(p1, p2) => unmap0(p1) match { case Prod0(p11, p12) => @@ -2190,7 +2299,10 @@ object Parser { // unmap may simplify enough // to remove the backtrack wrapper Parser.backtrack(unmap(p)) - case OneOf(ps) => Parser.oneOf(ps.map(unmap)) + case OneOf(ps) => + val next = Parser.oneOf(ps.map(unmap)) + if (next == pa) pa + else unmap(next) case Prod(p1, p2) => unmap0(p1) match { case Prod0(p11, p12) => @@ -2293,9 +2405,9 @@ object Parser { state.capture = false val init = state.offset pa.parseMut(state) - val str = state.str.substring(init, state.offset) state.capture = s0 - str + if (state.error eq null) state.str.substring(init, state.offset) + else null } case class StringP0[A](parser: Parser0[A]) extends Parser0[String] { @@ -2449,16 +2561,16 @@ object Parser { null.asInstanceOf[A] } - final def stringIn[A](radix: RadixNode, all: SortedSet[String], state: State): Unit = { - val str = state.str + final def stringIn(radix: RadixNode, all: SortedSet[String], state: State): String = { val startOffset = state.offset - val lastMatch = radix.matchAt(str, startOffset) + val matched = radix.matchAtOrNull(state.str, startOffset) - if (lastMatch < 0) { + if (matched eq null) { state.error = Eval.later(Chain.one(Expectation.OneOfStr(startOffset, all.toList))) - state.offset = startOffset + null } else { - state.offset = lastMatch + state.offset = startOffset + matched.length + matched } } @@ -2476,13 +2588,13 @@ object Parser { override def parseMut(state: State): A = oneOf(ary, state) } - case class StringIn(sorted: SortedSet[String]) extends Parser[Unit] { + case class StringIn(sorted: SortedSet[String]) extends Parser[String] { require(sorted.size >= 2, s"expected more than two items, found: ${sorted.size}") require(!sorted.contains(""), "empty string is not allowed in alternatives") private[this] val tree = RadixNode.fromSortedStrings(NonEmptyList.fromListUnsafe(sorted.toList)) - override def parseMut(state: State): Unit = stringIn(tree, sorted, state) + override def parseMut(state: State): String = stringIn(tree, sorted, state) } final def prod[A, B](pa: Parser0[A], pb: Parser0[B], state: State): (A, B) = { @@ -2640,14 +2752,14 @@ object Parser { override def parseMut(state: State): B = Impl.tailRecM(p1, fn, state) } - @annotation.tailrec + @tailrec final def compute0[A](fn: () => Parser0[A]): Parser0[A] = fn() match { case Defer(f) => compute(f) case Defer0(f) => compute0(f) case notDefer0 => notDefer0 } - @annotation.tailrec + @tailrec final def compute[A](fn: () => Parser[A]): Parser[A] = fn() match { case Defer(f) => compute(f) @@ -2790,93 +2902,224 @@ object Parser { } } - /* - * Merge CharIn bitsets - */ - def mergeCharIn[A, P0 <: Parser0[A]](ps: List[P0]): List[P0] = { - @annotation.tailrec - def loop(ps: List[P0], front: List[CharIn], result: Chain[P0]): Chain[P0] = { - @inline - def frontRes: Chain[P0] = - front match { - case Nil => Chain.nil - case one :: Nil => Chain.one(one.asInstanceOf[P0]) - case many => - // we need to union - val minBs = many.iterator.map { case CharIn(m, bs, _) => (m, bs) } - Chain.one(Parser.charIn(BitSetUtil.union(minBs)).asInstanceOf[P0]) - } + def allCharsIn(ci: CharIn): List[String] = + BitSetUtil + .union((ci.min, ci.bitSet) :: Nil) + .iterator + .map(_.toString) + .toList - ps match { - case Nil => result ++ frontRes - case AnyChar :: tail => - // AnyChar is bigger than all subsequent CharIn: - // and any direct prefix CharIns - val tail1 = tail.filterNot(_.isInstanceOf[CharIn]) - (result :+ AnyChar.asInstanceOf[P0]) ++ Chain.fromSeq(tail1) - case (ci: CharIn) :: tail => - loop(tail, ci :: front, result) - case h :: tail => - // h is not an AnyChar or CharIn - // we make our prefix frontRes - // and resume working on the tail - loop(tail, Nil, (result ++ frontRes) :+ h) - } + def merge0[A](left: Parser0[A], right: Parser0[A]): Parser0[A] = + (left, right) match { + case (l1: Parser[A], r1: Parser[A]) => merge(l1, r1) + case (_, _) if eventuallySucceeds(left) => left + case (Fail(), _) => right + case (_, Fail()) => left + case (OneOf0(_), OneOf(rs)) => + merge0(left, OneOf0(rs)) + case (OneOf(ls), OneOf0(_)) => + merge0(OneOf0(ls), right) + case (OneOf0(ls), OneOf0(rights @ (h :: t))) => + merge0(ls.last, h) match { + case OneOf(_) | OneOf0(_) => + // just concat + OneOf0(ls ::: rights) + case l1 => + val newLeft = OneOf0(ls.init :+ l1) + t match { + case rlast :: Nil => + merge0(newLeft, rlast) + case twoOrMore => + merge0(newLeft, OneOf0(twoOrMore)) + } + } + case (left, OneOf0(rs @ (h :: t))) => + merge0(left, h) match { + case OneOf(_) | OneOf0(_) => + OneOf0(left :: rs) + case h1 => + OneOf0(h1 :: t) + } + case (left, OneOf(rs @ (h :: t))) => + merge0(left, h) match { + case OneOf(_) | OneOf0(_) => + OneOf0(left :: rs) + case h1: Parser[A] => + OneOf(h1 :: t) + case h1 => + OneOf0(h1 :: t) + } + case (OneOf0(ls), right) => + merge0(ls.last, right) match { + case OneOf(_) | OneOf0(_) => + OneOf0(ls :+ right) + case l1 => + OneOf0(ls.init :+ l1) + } + case (OneOf(ls), right) => + merge0(ls.last, right) match { + case OneOf(_) | OneOf0(_) => + OneOf0(ls :+ right) + case l: Parser[A] => OneOf(ls.init :+ l) + case l => OneOf0(ls.init :+ l) + } + case (Void0(vl), Void0(vr)) => + merge0(vl, vr).void + case (Void0(vl), right) if isVoided(right) => + merge0(vl, right).void + case (Void(vl), right) if isVoided(right) => + merge0(vl, right).void + case (left, Void0(vr)) if isVoided(left) => + merge0(left, vr).void + case (left, Void(vr)) if isVoided(left) => + merge0(left, vr).void + case _ => OneOf0(left :: right :: Nil) } - loop(ps, Nil, Chain.nil).toList - } - - /* - * Merge Str and StringIn - * - * the semantic issue is this: - * oneOf matches the first, left to right - * StringIn matches the longest. - * we can only merge into the left if - * there are no prefixes of the right inside the left - */ - def mergeStrIn[A, P0 <: Parser0[A]](ps: List[P0]): List[P0] = { - @annotation.tailrec - def loop(ps: List[P0], front: SortedSet[String], result: Chain[P0]): Chain[P0] = { - @inline - def res(front: SortedSet[String]): Chain[P0] = - if (front.isEmpty) Chain.nil - else if (front.size == 1) Chain.one(Str(front.head).asInstanceOf[P0]) - else Chain.one(StringIn(front).asInstanceOf[P0]) - - // returns if there is a strict prefix (equality does not count) - def frontHasPrefixOf(s: String): Boolean = - front.exists { f => s.startsWith(f) && (f.length != s.length) } - - ps match { - case Nil => result ++ res(front) - case Str(s) :: tail => - if (frontHasPrefixOf(s)) { - // there is an overlap, so we need to match what we have first, then come here - loop(tail, SortedSet(s), result ++ res(front)) - } else { - // there is no overlap in the tree, just merge it in: - loop(tail, front + s, result) - } - case StringIn(ss) :: tail => - val (rights, lefts) = ss.partition(frontHasPrefixOf(_)) - val front1 = front | lefts - if (rights.nonEmpty) { - // there are some that can't be merged in - loop(tail, rights, result ++ res(front1)) - } else { - // everything can be merged in - loop(tail, front1, result) - } - case h :: tail => - loop(tail, SortedSet.empty, (result ++ res(front)) :+ h) - } + def merge[A](left: Parser[A], right: Parser[A]): Parser[A] = + (left, right) match { + case (Fail(), _) => right + case (_, Fail()) => left + case (OneOf(ls), OneOf(rights @ (h :: t))) => + merge(ls.last, h) match { + case OneOf(_) => + // just concat + OneOf(ls ::: rights) + case l1 => + val newLeft = OneOf(ls.init :+ l1) + t match { + case rlast :: Nil => + merge(newLeft, rlast) + case twoOrMore => + merge(newLeft, OneOf(twoOrMore)) + } + } + case (left, OneOf(rs @ (h :: t))) => + merge(left, h) match { + case OneOf(_) => + OneOf(left :: rs) + case h1 => + // maybe we can progess on t + if (t.lengthCompare(2) >= 0) + merge(h1, OneOf(t)) + else + merge(h1, t.head) + } + case (OneOf(ls), right) => + merge(ls.last, right) match { + case OneOf(_) => + OneOf(ls :+ right) + case l1 => + val li = ls.init + if (li.lengthCompare(2) >= 0) + merge(OneOf(li), l1) + else + merge(li.head, l1) + } + case (CharIn(_, _, _), AnyChar) => AnyChar + case (AnyChar, CharIn(_, _, _) | Str(_) | StringIn(_)) => AnyChar + case (CharIn(m1, b1, _), CharIn(m2, b2, _)) => + Parser.charIn(BitSetUtil.union((m1, b1) :: (m2, b2) :: Nil)) + case (Void(ci @ CharIn(_, _, _)), Str(_)) => + Parser.oneOf(allCharsIn(ci).map(Str(_)) ::: (right :: Nil)).asInstanceOf[Parser[A]] + case (StringP(ci @ CharIn(_, _, _)), DefiniteString(_) | StringIn(_)) => + // make sure we make progress... + val strs = StringIn(SortedSet(allCharsIn(ci): _*)) + merge(strs.asInstanceOf[Parser[A]], right) + case (Str(l), Void(ci @ CharIn(_, _, _))) => + Parser.oneOf(Str(l) :: allCharsIn(ci).map(Str(_))).asInstanceOf[Parser[A]] + case (DefiniteString(_) | StringIn(_), StringP(ci @ CharIn(_, _, _))) => + // make sure we make progress... + val strs = StringIn(SortedSet(allCharsIn(ci): _*)) + merge(left, strs.asInstanceOf[Parser[A]]) + case (Str(l), Str(r)) => + // if l is a prefix of r, it matches first + // if not, then we can make a StringIn(_).void + if (r.startsWith(l)) left + else Void(StringIn(SortedSet(l, r))) + case (DefiniteString(l), DefiniteString(r)) => + // if l is a prefix of r, it matches first + // if not, then we can make a StringIn(_).void + if (r.startsWith(l)) left + else { + val res = + if (l.length == 1 && r.length == 1) { + charIn(l.head :: r.head :: Nil).string + } else { + StringIn(SortedSet(l, r)) + } + + res.asInstanceOf[Parser[A]] + } + case (StringIn(ls), DefiniteString(s1)) => + if (ls.exists { l => s1.startsWith(l) && (l.length <= s1.length) }) { + // if left didn't match, then s1 can't match + left + } else StringIn(ls + s1) + case (Void(StringIn(ls)), Str(s1)) => + if (ls.exists { l => s1.startsWith(l) && (l.length <= s1.length) }) { + // if left didn't match, then s1 can't match + left + } else Void(StringIn(ls + s1)) + case (DefiniteString(l), StringIn(rs)) => + // We know if we go to rs that l did + // not match so nothing in rs can have l as a prefix + val good = rs.filterNot(_.startsWith(l)) + if (good.isEmpty) left + else { + StringIn(good + l) + } + case (Str(l), Void(StringIn(rs))) => + // We know if we go to rs that l did + // not match so nothing in rs can have l as a prefix + val good = rs.filterNot(_.startsWith(l)) + if (good.isEmpty) left + else { + Void(StringIn(good + l)) + } + case (StringIn(ls), StringIn(rs)) => + // any string in rs that doesn't have a substring in ls can be moved + // over, since substrings would match first in oneOf but not StringIn + val canMatch = rs.filterNot { s => + ls.exists { l => s.startsWith(l) && (l.length <= s.length) } + } + if (canMatch.isEmpty) left + else { + StringIn(ls | canMatch) + } + case (Void(StringIn(ls)), Void(ci @ CharIn(_, _, _))) => + val rs = SortedSet(allCharsIn(ci): _*) + // any string in rs that doesn't have a substring in ls can be moved + // over, since substrings would match first in oneOf but not StringIn + val canMatch = rs.filterNot { s => + ls.exists { l => s.startsWith(l) && (l.length <= s.length) } + } + if (canMatch.isEmpty) left + else { + Void(StringIn(ls | canMatch)) + } + case (Void(ci @ CharIn(_, _, _)), Void(StringIn(rs))) => + val ls = SortedSet(allCharsIn(ci): _*) + // any string in rs that doesn't have a substring in ls can be moved + // over, since substrings would match first in oneOf but not StringIn + val canMatch = rs.filterNot { s => + ls.exists { l => s.startsWith(l) && (l.length <= s.length) } + } + if (canMatch.isEmpty) left + else { + Void(StringIn(ls | canMatch)) + } + case (Void(vl), Void(vr)) => + merge(vl, vr).void + case (StringP(l1), StringP(r1)) => + merge(l1, r1).string + case (Void(vl), right) if isVoided(right) => + merge(vl, right).void + case (left, Void(vr)) if isVoided(left) => + merge(left, vr).void + case _ => OneOf(left :: right :: Nil) } - loop(ps, SortedSet.empty, Chain.nil).toList - } - case object AnyChar extends Parser[Char] { override def parseMut(state: State): Char = { val offset = state.offset @@ -2935,7 +3178,10 @@ object Parser { case class Not(under: Parser0[Unit]) extends Parser0[Unit] { override def parseMut(state: State): Unit = { val offset = state.offset + val cap = state.capture + state.capture = false under.parseMut(state) + state.capture = cap if (state.error ne null) { // under failed, so we succeed state.error = null @@ -2966,7 +3212,10 @@ object Parser { case class Peek(under: Parser0[Unit]) extends Parser0[Unit] { override def parseMut(state: State): Unit = { val offset = state.offset + val cap = state.capture + state.capture = false under.parseMut(state) + state.capture = cap if (state.error eq null) { // under passed, so we succeed state.offset = offset diff --git a/core/shared/src/test/scala/cats/parse/ParserTest.scala b/core/shared/src/test/scala/cats/parse/ParserTest.scala index 8e3e77a8..d0fcd1e8 100644 --- a/core/shared/src/test/scala/cats/parse/ParserTest.scala +++ b/core/shared/src/test/scala/cats/parse/ParserTest.scala @@ -21,15 +21,15 @@ package cats.parse -import cats.{Eq, Id, FlatMap, Functor, Defer, MonoidK, Monad, Eval} import cats.arrow.FunctionK -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, NonEmptyVector} +import cats.implicits._ +import cats.{Eq, Id, FlatMap, Functor, Defer, MonoidK, Monad, Eval} import org.scalacheck.Prop.forAll import org.scalacheck.{Arbitrary, Gen, Cogen} - -import cats.implicits._ import scala.util.Random -import cats.data.NonEmptyVector + +import Arbitrary.arbitrary sealed abstract class GenT[F[_]] { self => type A @@ -71,7 +71,7 @@ object ParserGen { } implicit val cogenCaret: Cogen[Caret] = - Cogen { caret: Caret => + Cogen { (caret: Caret) => (caret.offset.toLong << 32) | (caret.col.toLong << 16) | (caret.line.toLong) } @@ -612,6 +612,8 @@ class ParserTest extends munit.ScalaCheckSuite { // "bUQvEfxFZ73bxtVjauK8tJDrEKOFbxUfk6WrGiy3bkH=" // "PPsKExr4HRlyCXkMrC6Rki5u59V88vwSeVTiGWJFS3G=" // "Ova1uT18mkE4uTX4RdgQza6z70fxyv6micl4hIZvywP=" + // "YcGRsiTHa791rV5CIL4wYhWDofanqbYbvO418dbZnOK=" + // "6YoSspuNxqEoMosfi5J6wHgo4I4rD48Zg21XAnZtMcA=" def parseTest[A: Eq](p: Parser0[A], str: String, a: A) = p.parse(str) match { @@ -757,12 +759,23 @@ class ParserTest extends munit.ScalaCheckSuite { } } + test("string(x).void == string(x) and withContext") { + assertEquals(Parser.string("foo").void, Parser.string("foo")) + assertEquals( + Parser.string("foo").withContext("ctx").void, + Parser.string("foo").withContext("ctx") + ) + } + property("voided only changes the result") { forAll(ParserGen.gen0, Arbitrary.arbitrary[String]) { (genP, str) => - val r1 = genP.fa.parse(str) - val r2 = genP.fa.void.parse(str) - val r3 = FlatMap[Parser0].void(genP.fa).parse(str) - val r4 = genP.fa.as(()).parse(str) + def go[A](p: Parser0[A]) = + p.parse(str).leftMap(_.offsets) + + val r1 = go(genP.fa) + val r2 = go(genP.fa.void) + val r3 = go(FlatMap[Parser0].void(genP.fa)) + val r4 = go(genP.fa.as(())) assertEquals(r2, r1.map { case (off, _) => (off, ()) }) assertEquals(r2, r3) @@ -772,11 +785,14 @@ class ParserTest extends munit.ScalaCheckSuite { property("voided only changes the result Parser") { forAll(ParserGen.gen, Arbitrary.arbitrary[String]) { (genP, str) => - val r1 = genP.fa.parse(str) - val r2 = genP.fa.void.parse(str) - val r3 = FlatMap[Parser].void(genP.fa).parse(str) - val r4 = genP.fa.as(()).parse(str) - val r5 = ((genP.fa.void: Parser0[Unit]) <* Monad[Parser0].unit).parse(str) + def go[A](p: Parser0[A]) = + p.parse(str).leftMap(_.offsets) + + val r1 = go(genP.fa) + val r2 = go(genP.fa.void) + val r3 = go(FlatMap[Parser].void(genP.fa)) + val r4 = go(genP.fa.as(())) + val r5 = go((genP.fa.void: Parser0[Unit]) <* Monad[Parser0].unit) assertEquals(r2, r1.map { case (off, _) => (off, ()) }) assertEquals(r2, r3) @@ -804,7 +820,7 @@ class ParserTest extends munit.ScalaCheckSuite { val oneOf = Parser.oneOf0((genP1 ::: genP2).map(_.fa)) val oneOf2 = Parser.oneOf0(genP1.map(_.fa)).orElse(Parser.oneOf0(genP2.map(_.fa))) - assertEquals(oneOf.parse(str), oneOf2.parse(str)) + assertEquals(oneOf.parse(str).leftMap(_.offsets), oneOf2.parse(str).leftMap(_.offsets)) } } @@ -818,7 +834,7 @@ class ParserTest extends munit.ScalaCheckSuite { Parser.oneOf(genP2.map(_.fa)) ) - assertEquals(oneOf.parse(str), oneOf2.parse(str)) + assertEquals(oneOf.parse(str).leftMap(_.offsets), oneOf2.parse(str).leftMap(_.offsets)) } } @@ -845,15 +861,45 @@ class ParserTest extends munit.ScalaCheckSuite { case right => right } + def oneOfLaw(left: Parser0[Any], right: Parser0[Any], str: String) = { + assertEquals( + left.orElse(right).parse(str).leftMap(_.offsets), + orElse(left, right, str).leftMap(_.offsets) + ) + } + property("oneOf0 composes as expected") { forAll(ParserGen.gen0, ParserGen.gen0, Arbitrary.arbitrary[String]) { (genP1, genP2, str) => - assertEquals(genP1.fa.orElse(genP2.fa).parse(str), orElse(genP1.fa, genP2.fa, str)) + oneOfLaw(genP1.fa, genP2.fa, str) } } property("oneOf composes as expected") { forAll(ParserGen.gen, ParserGen.gen, Arbitrary.arbitrary[String]) { (genP1, genP2, str) => - assertEquals(genP1.fa.orElse(genP2.fa).parse(str), orElse(genP1.fa, genP2.fa, str)) + oneOfLaw(genP1.fa, genP2.fa, str) + } + } + + property("check some specific oneOf compositions") { + val pairs: List[(Parser0[Any], Parser0[Any])] = + (Parser.string("foo").string, Parser.stringIn("foo" :: "bar" :: "foobar" :: Nil)) :: + (Parser.stringIn("foo" :: "quux" :: Nil), Parser.string("foobar").string) :: + (Parser.stringIn("foo" :: "quux" :: Nil), Parser.char('f').string) :: + (Parser.stringIn("foo" :: "quux" :: Nil), Parser.stringIn("foo" :: "quux" :: Nil)) :: + ( + Parser.stringIn("foo" :: "quux" :: "bar" :: Nil), + Parser.stringIn("foo" :: "quux" :: Nil) + ) :: + ( + Parser.stringIn("foo" :: "quux" :: Nil), + Parser.stringIn("foo" :: "quux" :: "bar" :: Nil) + ) :: + Nil + + forAll { (str: String) => + pairs.foreach { case (p1, p2) => + oneOfLaw(p1, p2, str) + } } } @@ -863,7 +909,10 @@ class ParserTest extends munit.ScalaCheckSuite { leftp.orElse(p.fa) } - assertEquals(oneOfImpl.parse(str), Parser.oneOf0(genP1.map(_.fa)).parse(str)) + assertEquals( + oneOfImpl.parse(str).leftMap(_.offsets), + Parser.oneOf0(genP1.map(_.fa)).parse(str).leftMap(_.offsets) + ) } } @@ -871,7 +920,10 @@ class ParserTest extends munit.ScalaCheckSuite { forAll(Gen.listOf(ParserGen.gen), Arbitrary.arbitrary[String]) { (genP1, str) => val oneOfImpl = genP1.foldLeft(Parser.fail[Any]) { (leftp, p) => leftp.orElse(p.fa) } - assertEquals(oneOfImpl.parse(str), Parser.oneOf(genP1.map(_.fa)).parse(str)) + assertEquals( + oneOfImpl.parse(str).leftMap(_.offsets), + Parser.oneOf(genP1.map(_.fa)).parse(str).leftMap(_.offsets) + ) } } @@ -1364,35 +1416,40 @@ class ParserTest extends munit.ScalaCheckSuite { property("repSep0 with unit sep is the same as rep0") { + case class MinMax(min: Int, max: Int) val minMax = for { min <- biasSmall(0) max <- biasSmall(Integer.max(min, 1)) - } yield (min, max) + } yield MinMax(min, max) - forAll(ParserGen.gen, biasSmall(0), Arbitrary.arbitrary[String]) { (genP, min, str) => + forAll(ParserGen.gen, biasSmall(0), Arbitrary.arbitrary[String]) { (genP, min0, str) => + val min = min0 & Int.MaxValue // make sure it is positive + // repSep0 internally uses | which triggers rewriting optimizations val p1a = genP.fa.repSep0(min = min, sep = Parser.unit) val p1b = genP.fa.rep0(min = min) - assertEquals(p1a.parse(str), p1b.parse(str)) + assertEquals(p1a.parse(str).leftMap(_.offsets), p1b.parse(str).leftMap(_.offsets)) val min1 = if (min < 1) 1 else min val p2a = genP.fa.repSep(min = min1, sep = Parser.unit) val p2b = genP.fa.rep(min = min1) - assertEquals(p2a.parse(str), p2b.parse(str)) + assertEquals(p2a.parse(str).leftMap(_.offsets), p2b.parse(str).leftMap(_.offsets)) } && - forAll(ParserGen.gen, minMax, Arbitrary.arbitrary[String]) { case (genP, (min, max), str) => - val p1a = genP.fa.repSep0(min = min, max = max, sep = Parser.unit) - val p1b = genP.fa.rep0(min = min, max = max) + forAll(ParserGen.gen, minMax, Arbitrary.arbitrary[String]) { + case (genP, MinMax(min, max), str) => + // repSep0 internally uses | which triggers rewriting optimizations + val p1a = genP.fa.repSep0(min = min, max = max, sep = Parser.unit) + val p1b = genP.fa.rep0(min = min, max = max) - assertEquals(p1a.parse(str), p1b.parse(str)) + assertEquals(p1a.parse(str).leftMap(_.offsets), p1b.parse(str).leftMap(_.offsets)) - val min1 = if (min < 1) 1 else min - val p2a = genP.fa.repSep(min = min1, max = max, sep = Parser.unit) - val p2b = genP.fa.rep(min = min1, max = max) + val min1 = if (min < 1) 1 else min + val p2a = genP.fa.repSep(min = min1, max = max, sep = Parser.unit) + val p2b = genP.fa.rep(min = min1, max = max) - assertEquals(p2a.parse(str), p2b.parse(str)) + assertEquals(p2a.parse(str).leftMap(_.offsets), p2b.parse(str).leftMap(_.offsets)) } } @@ -1454,8 +1511,8 @@ class ParserTest extends munit.ScalaCheckSuite { } } - property("p orElse p == p") { - forAll(ParserGen.gen, Arbitrary.arbitrary[String]) { (genP, str) => + property("p orElse p == p (0)") { + forAll(ParserGen.gen0, Arbitrary.arbitrary[String]) { (genP, str) => val res0 = genP.fa.parse(str) val res1 = genP.fa.orElse(genP.fa).parse(str) assertEquals(res1, res0) @@ -2001,8 +2058,8 @@ class ParserTest extends munit.ScalaCheckSuite { val left = pa.repAs0[Unit](Accumulator0.unitAccumulator0) val right = pa.rep0.void - val leftRes = left.parse(str) - val rightRes = right.parse(str) + val leftRes = left.parse(str).leftMap(_.offsets) + val rightRes = right.parse(str).leftMap(_.offsets) assertEquals(leftRes, rightRes) } } @@ -2297,6 +2354,26 @@ class ParserTest extends munit.ScalaCheckSuite { } property("p.as(a).map(fn) == p.as(fn(a))") { + val regressions = + (Parser.defer(Parser.string("foo")).void.backtrack) :: + (Parser.defer(Parser.string("foo")).void.withContext("ctx").backtrack) :: + Nil + + regressions.foreach { p => + assertEquals(p.void.as(1), p.as(1)) + assertEquals(p.as(1).void, p.void) + assertEquals(p.as(1).as(1), p.as(1)) + + val a = 42 + val fn = { (x: Int) => x + 1 } + assertEquals(p.as(a).map(fn), p.as(fn(a))) + } + assertEquals(Parser.string("foo").void, Parser.string("foo")) + assertEquals( + Parser.string("foo").withContext("bar").void, + Parser.string("foo").withContext("bar") + ) + forAll(ParserGen.gen, Gen.choose(0, 128), Gen.function1[Int, Int](Gen.choose(0, 128))) { (p, a, fn) => assertEquals(p.fa.as(a).map(fn), p.fa.as(fn(a))) @@ -2442,7 +2519,10 @@ class ParserTest extends munit.ScalaCheckSuite { val left = Parser.oneOf0(as.map(_.fa.string)) val right = Parser.oneOf0[Any](as.map(_.fa)).string - assertEquals(left.parse(toParse), right.parse(toParse)) + assertEquals( + left.parse(toParse).leftMap(_.offsets), + right.parse(toParse).leftMap(_.offsets) + ) } } @@ -2452,7 +2532,10 @@ class ParserTest extends munit.ScalaCheckSuite { val left = Parser.oneOf(as.map(_.fa.string)) val right = Parser.oneOf[Any](as.map(_.fa)).string - assertEquals(left.parse(toParse), right.parse(toParse)) + assertEquals( + left.parse(toParse).leftMap(_.offsets), + right.parse(toParse).leftMap(_.offsets) + ) } } @@ -2561,6 +2644,17 @@ class ParserTest extends munit.ScalaCheckSuite { } property("P.void is idempotent") { + val regressions = + ((Parser.string("aa").map(_ => 1) | Parser.string("bb").map(_ => 2)).withContext("ctx")) :: + (Parser.defer(Parser.string("foo")).void.backtrack) :: + (Parser.defer(Parser.string("foo")).void.withContext("ctx").backtrack) :: + Nil + + regressions.foreach { p => + val v = p.void + assertEquals(v.void, v) + } + forAll(ParserGen.gen) { p => val v1 = p.fa.void assertEquals(v1.void, v1) @@ -2615,4 +2709,111 @@ class ParserTest extends munit.ScalaCheckSuite { } } } + + property("stringIn(s).void.string == stringIn(s)") { + forAll { (ss0: List[String]) => + val ss = ss0.filter(_.nonEmpty) + val si = Parser.stringIn(ss) + + assertEquals(si.void.string, si) + } + } + + property("test that some parsers unify with |") { + forAll { (s10: Set[String], s20: Set[String]) => + val s1 = s10.filter(_.length > 1) + val s2 = s20.filterNot { s => (s.length < 2) || s1.exists(s.startsWith(_)) } + assertEquals(Parser.stringIn(s1) | Parser.stringIn(s2), Parser.stringIn(s1 | s2)) + assertEquals( + Parser.stringIn(s1).void | Parser.stringIn(s2).void, + Parser.stringIn(s1 | s2).void + ) + } && + forAll { (s1: Set[Char], s2: Set[Char]) => + assertEquals(Parser.charIn(s1) | Parser.charIn(s2), Parser.charIn(s1 | s2)) + assertEquals(Parser.charIn(s1).void | Parser.charIn(s2).void, Parser.charIn(s1 | s2).void) + // TODO: make this law pass. Currently the left is StringIn, but the right is StringP(CharIn(_, _, _)) + // assertEquals(Parser.charIn(s1).string | Parser.charIn(s2).string, Parser.charIn(s1 | s2).string) + } && + forAll { (s1: String, s2: String) => + if (!s2.startsWith(s1) && (s1.nonEmpty && s2.nonEmpty)) { + if ((s1.length > 1) || (s2.length > 1)) { + assertEquals( + Parser.stringIn(s1 :: s2 :: Nil).void, + Parser.string(s1) | Parser.string(s2) + ) + + assertEquals( + Parser.stringIn(s1 :: s2 :: Nil), + (Parser.string(s1) | Parser.string(s2)).string + ) + + assertEquals( + Parser.stringIn(s1 :: s2 :: Nil), + Parser.string(s1).string | Parser.string(s2).string + ) + } else () + } else () + } + } + + property("a | b is associative") { + def strict(a: Parser0[Any], b: Parser0[Any], c: Parser0[Any]) = + assertEquals((a | b) | c, a | (b | c)) + + val regressions: List[(Parser0[Any], Parser0[Any], Parser0[Any])] = + (Parser.char('c'), Parser.unit, Parser.char('b')) :: + (Parser.string("foo"), Parser.string("bar"), Parser.string("cow")) :: + ( + Parser.stringIn("foo" :: "bar" :: Nil), + Parser.string("x").string, + Parser.string("y").string + ) :: + ( + Parser.stringIn("foo" :: "bar" :: Nil), + Parser.char('x').string, + Parser.char('y').string + ) :: + (Parser.stringIn("foo" :: "bar" :: Nil).void, Parser.string("x"), Parser.string("y")) :: + (Parser.stringIn("foo" :: "bar" :: Nil).void, Parser.char('x'), Parser.char('y')) :: + (Parser.string("foo"), Parser.string("bar"), Parser.string("cow").withContext("ctx")) :: + // (Parser.string("foo"), Parser.string("bar"), Parser.unit.withContext("ctx")) :: + (Parser.char('a'), Parser.char('b'), Parser.char('c')) :: + // (Parser.end, Parser.string("foo"), Parser.char('c')) :: + (Parser.char('c'), Parser.string("foo"), Parser.ignoreCase("select")) :: + (Parser.string("aa"), Parser.string("bb"), Parser.char('c')) :: + (Parser.string("aa").string, Parser.string("bb").string, Parser.char('c').string) :: + ( + Parser.string("aa").string, + Parser.string("bb").string, + Parser.charIn('c' :: 'd' :: Nil).string + ) :: + (Parser.ignoreCase("select").string, Parser.char('a').string, Parser.char('b').string) :: + (Parser.anyChar.void, Parser.char('a'), Parser.string("foo")) :: + (Parser.anyChar, Parser.string("foo"), Parser.stringIn("bar" :: "baz" :: Nil)) :: + ( + Parser.stringIn("foo" :: "bar" :: Nil).void, + Parser.anyChar.void, + Parser.charIn('a' :: 'b' :: Nil) + ) :: + Nil + + regressions.foreach { case (a, b, c) => strict(a, b, c) } + + def law(a: Parser0[Any], b: Parser0[Any], c: Parser0[Any], str: String) = + // We only compare the offsets of errors here because occassionally + // merges produce equivalent parsers, but different errors kinds + // since there is some overlap in how things can be expressed + assertEquals( + ((a | b) | c).parse(str).leftMap(_.offsets), + (a | (b | c)).parse(str).leftMap(_.offsets) + ) + + forAll(ParserGen.gen, ParserGen.gen, ParserGen.gen, arbitrary[String]) { (a, b, c, str) => + law(a.fa, b.fa, c.fa, str) + } && + forAll(ParserGen.gen0, ParserGen.gen0, ParserGen.gen0, arbitrary[String]) { (a, b, c, str) => + law(a.fa, b.fa, c.fa, str) + } + } }