Verbatim string ast

src/fsharp/CheckComputationExpressions.fs

@@ -1917,7 +1917,7 @@ let TcArrayOrListSequenceExpression (cenv: cenv) env overallTy tpenv (isArray, c
                 if nelems > 0 && List.forall (function SynExpr.Const (SynConst.UInt16 _, _) -> true | _ -> false) elems 
                 then SynExpr.Const (SynConst.UInt16s (Array.ofList (List.map (function SynExpr.Const (SynConst.UInt16 x, _) -> x | _ -> failwith "unreachable") elems)), m)
                 elif nelems > 0 && List.forall (function SynExpr.Const (SynConst.Byte _, _) -> true | _ -> false) elems 
-                then SynExpr.Const (SynConst.Bytes (Array.ofList (List.map (function SynExpr.Const (SynConst.Byte x, _) -> x | _ -> failwith "unreachable") elems), m), m)
+                then SynExpr.Const (SynConst.Bytes (Array.ofList (List.map (function SynExpr.Const (SynConst.Byte x, _) -> x | _ -> failwith "unreachable") elems), SynByteStringKind.Regular, m), m)
                 else SynExpr.ArrayOrList (isArray, elems, m)
             else 
                 if elems.Length > 500 then 

src/fsharp/CheckExpressions.fs

@@ -810,7 +810,7 @@ let TcConst cenv ty m env c =
     | SynConst.Measure(SynConst.UInt64 i, _) when expandedMeasurablesEnabled -> unifyMeasureArg (i=0UL) cenv.g.puint64_tcr c; Const.UInt64 i
     | SynConst.Measure(SynConst.UIntPtr i, _) when expandedMeasurablesEnabled -> unifyMeasureArg (i=0UL) cenv.g.punativeint_tcr c; Const.UIntPtr i
     | SynConst.Char c -> unif cenv.g.char_ty; Const.Char c
-    | SynConst.String (s, _) -> unif cenv.g.string_ty; Const.String s
+    | SynConst.String (s, _, _) -> unif cenv.g.string_ty; Const.String s
     | SynConst.UserNum _ -> error (InternalError(FSComp.SR.tcUnexpectedBigRationalConstant(), m))
     | SynConst.Measure _ -> error (Error(FSComp.SR.tcInvalidTypeForUnitsOfMeasure(), m))
     | SynConst.UInt16s _ -> error (InternalError(FSComp.SR.tcUnexpectedConstUint16Array(), m))
@@ -4414,7 +4414,7 @@ and TcStaticConstantParameter cenv (env: TcEnv) tpenv kind (StripParenTypes v) i
             | SynConst.Single n when typeEquiv g g.float32_ty kind -> record(g.float32_ty); box (n: single)
             | SynConst.Double n when typeEquiv g g.float_ty kind -> record(g.float_ty); box (n: double)
             | SynConst.Char n when typeEquiv g g.char_ty kind -> record(g.char_ty); box (n: char)
-            | SynConst.String (s, _) when s <> null && typeEquiv g g.string_ty kind -> record(g.string_ty); box (s: string)
+            | SynConst.String (s, _, _) when s <> null && typeEquiv g g.string_ty kind -> record(g.string_ty); box (s: string)
             | SynConst.Bool b when typeEquiv g g.bool_ty kind -> record(g.bool_ty); box (b: bool)
             | _ -> fail()
         v, tpenv
@@ -4794,7 +4794,7 @@ and TcPat warnOnUpper cenv env topValInfo vFlags (tpenv, names, takenNames) ty p
     match pat with 
     | SynPat.Const (c, m) -> 
         match c with 
-        | SynConst.Bytes (bytes, m) -> 
+        | SynConst.Bytes (bytes, _, m) -> 
             UnifyTypes cenv env m ty (mkByteArrayTy cenv.g) 
             TcPat warnOnUpper cenv env None vFlags (tpenv, names, takenNames) ty (SynPat.ArrayOrList (true, [ for b in bytes -> SynPat.Const(SynConst.Byte b, m) ], m))
 
@@ -5425,11 +5425,11 @@ and TcExprUndelayed cenv overallTy env tpenv (synExpr: SynExpr) =
     | SynExpr.DotIndexedGet _ | SynExpr.DotIndexedSet _
     | SynExpr.TypeApp _ | SynExpr.Ident _ | SynExpr.LongIdent _ | SynExpr.App _ | SynExpr.DotGet _ -> error(Error(FSComp.SR.tcExprUndelayed(), synExpr.Range))
 
-    | SynExpr.Const (SynConst.String (s, m), _) -> 
+    | SynExpr.Const (SynConst.String (s, _, m), _) -> 
         CallExprHasTypeSink cenv.tcSink (m, env.NameEnv, overallTy, env.AccessRights)
         TcConstStringExpr cenv overallTy env m tpenv s
 
-    | SynExpr.InterpolatedString (parts, m) -> 
+    | SynExpr.InterpolatedString (parts, _, m) -> 
         checkLanguageFeatureError cenv.g.langVersion LanguageFeature.StringInterpolation m
 
         CallExprHasTypeSink cenv.tcSink (m, env.NameEnv, overallTy, env.AccessRights)
@@ -6784,7 +6784,7 @@ and TcConstExpr cenv overallTy env m tpenv c =
     match c with 
 
     // NOTE: these aren't "really" constants 
-    | SynConst.Bytes (bytes, m) -> 
+    | SynConst.Bytes (bytes, _, m) -> 
        UnifyTypes cenv env m overallTy (mkByteArrayTy cenv.g) 
        Expr.Op (TOp.Bytes bytes, [], [], m), tpenv
 
@@ -6810,7 +6810,7 @@ and TcConstExpr cenv overallTy env m tpenv c =
                             let i64 = int64 s  
                             SynExpr.App (ExprAtomicFlag.Atomic, false, mkSynLidGet m [modName] "FromInt64", SynExpr.Const (SynConst.Int64 i64, m), m)
                         with _ ->             
-                            SynExpr.App (ExprAtomicFlag.Atomic, false, mkSynLidGet m [modName] "FromString", SynExpr.Const (SynConst.String (s, m), m), m) 
+                            SynExpr.App (ExprAtomicFlag.Atomic, false, mkSynLidGet m [modName] "FromString", SynExpr.Const (SynConst.String (s, SynStringKind.Regular, m), m), m) 
 
                 if suffix <> "I" then
                     expr

src/fsharp/SyntaxTree.fs

@@ -81,6 +81,19 @@ type SynTypar =
         | Typar(id, _, _) ->
             id.idRange
 
+/// Indicate if the string had a special format
+[<Struct; RequireQualifiedAccess>]
+type SynStringKind =
+    | Regular
+    | Verbatim
+    | TripleQuote
+
+/// Indicate if the byte string had a special format
+[<Struct; RequireQualifiedAccess>]
+type SynByteStringKind =
+    | Regular
+    | Verbatim
+
 /// The unchecked abstract syntax tree of constants in F# types and expressions.
 [<NoEquality; NoComparison; RequireQualifiedAccess>]
 type SynConst =
@@ -139,13 +152,13 @@ type SynConst =
     | UserNum of value: string * suffix: string
 
     /// F# syntax: verbatim or regular string, e.g. "abc"
-    | String of text: string * range: range
+    | String of text: string * synStringKind :SynStringKind * range: range
 
     /// F# syntax: verbatim or regular byte string, e.g. "abc"B.
     ///
     /// Also used internally in the typechecker once an array of unit16 constants
     /// is detected, to allow more efficient processing of large arrays of uint16 constants.
-    | Bytes of bytes: byte[] * range: range
+    | Bytes of bytes: byte[] * synByteStringKind: SynByteStringKind * range: range
 
     /// Used internally in the typechecker once an array of unit16 constants
     /// is detected, to allow more efficient processing of large arrays of uint16 constants.
@@ -157,7 +170,7 @@ type SynConst =
     /// Gets the syntax range of this construct
     member c.Range dflt =
         match c with
-        | SynConst.String (_, m0) | SynConst.Bytes (_, m0) -> m0
+        | SynConst.String (_, _, m0) | SynConst.Bytes (_, _, m0) -> m0
         | _ -> dflt
 
 /// Represents an unchecked syntax tree of F# unit of measure annotations.
@@ -1015,6 +1028,7 @@ type SynExpr =
     /// Note the string ranges include the quotes, verbatim markers, dollar sign and braces
     | InterpolatedString of
         contents: SynInterpolatedStringPart list *
+        synStringKind :SynStringKind *
         range: range
 
     /// Gets the syntax range of this construct

src/fsharp/SyntaxTreeOps.fs

@@ -735,7 +735,7 @@ let rec synExprContainsError inpExpr =
           | SynExpr.LetOrUseBang  (rhs=e1;body=e2;andBangs=es) ->
               walkExpr e1 || walkExprs [ for (_,_,_,_,e,_) in es do yield e ] || walkExpr e2
 
-          | SynExpr.InterpolatedString (parts, _m) ->
+          | SynExpr.InterpolatedString (parts, _, _m) ->
               walkExprs 
                   (parts |> List.choose (function 
                       | SynInterpolatedStringPart.String _ -> None

src/fsharp/lex.fsl

@@ -136,33 +136,50 @@ let startString args (lexbuf: UnicodeLexing.Lexbuf) =
     let m = lexbuf.LexemeRange 
     let startp = lexbuf.StartPos 
     let fin =
-        LexerStringFinisher (fun buf kind isPart cont -> 
+        LexerStringFinisher (fun buf kind context cont -> 
             // Adjust the start-of-token mark back to the true start of the token 
             lexbuf.StartPos <- startp
-            if kind.IsByteString then
+            let isPart = context.HasFlag(LexerStringFinisherContext.InterpolatedPart)
+            let isVerbatim = context.HasFlag(LexerStringFinisherContext.Verbatim)
+            let isTripleQuote = context.HasFlag(LexerStringFinisherContext.TripleQuote)
+
+            if kind.IsByteString then          
+                let synByteStringKind = if isVerbatim then SynByteStringKind.Verbatim else SynByteStringKind.Regular
                 if kind.IsInterpolated then 
                     fail args lexbuf (FSComp.SR.lexByteStringMayNotBeInterpolated()) ()
-                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, cont)
+                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
                 elif Lexhelp.stringBufferIsBytes buf then
-                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, cont)
+                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
                 else
                     fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode()) ()
-                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, cont)
+                    BYTEARRAY (Lexhelp.stringBufferAsBytes buf, synByteStringKind, cont)
             elif kind.IsInterpolated then 
                 let s = Lexhelp.stringBufferAsString buf
-                if kind.IsInterpolatedFirst then 
+                if kind.IsInterpolatedFirst then
+                    let synStringKind =
+                        if isTripleQuote then
+                            SynStringKind.TripleQuote
+                        else
+                            SynStringKind.Regular
                     if isPart then 
-                        INTERP_STRING_BEGIN_PART (s, cont)
+                        INTERP_STRING_BEGIN_PART (s, synStringKind, cont)
                     else
-                        INTERP_STRING_BEGIN_END (s, cont)
+                        INTERP_STRING_BEGIN_END (s, synStringKind, cont)
                 else
                     if isPart then
                         INTERP_STRING_PART (s, cont)
                     else
                         INTERP_STRING_END (s, cont)
             else
                 let s = Lexhelp.stringBufferAsString buf
-                STRING (s, cont))  
+                let synStringKind =
+                    if isVerbatim then
+                        SynStringKind.Verbatim
+                    elif isTripleQuote then
+                        SynStringKind.TripleQuote
+                    else
+                        SynStringKind.Regular
+                STRING (s, synStringKind, cont))  
     buf,fin,m
 
 
@@ -1124,13 +1141,13 @@ and singleQuoteString sargs skip = parse
  |  '"' 
     { let (buf, fin, _m, kind, args) = sargs 
       let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-      fin.Finish buf kind false cont
+      fin.Finish buf kind (enum<LexerStringFinisherContext>(0)) cont
     }
 
  |  '"''B' 
     { let (buf, fin, _m, kind, args) = sargs 
       let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-      fin.Finish buf { kind with IsByteString = true } false cont
+      fin.Finish buf { kind with IsByteString = true } (enum<LexerStringFinisherContext>(0)) cont
     }
 
  | ("{{" | "}}")
@@ -1147,7 +1164,7 @@ and singleQuoteString sargs skip = parse
           let m2 = lexbuf.LexemeRange
           args.stringNest <- (1, LexerStringStyle.SingleQuote, m2) :: args.stringNest
           let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-          fin.Finish buf kind true cont
+          fin.Finish buf kind LexerStringFinisherContext.InterpolatedPart cont
       else
           addUnicodeString buf (lexeme lexbuf) 
           if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.SingleQuote, kind, m))
@@ -1214,13 +1231,13 @@ and verbatimString sargs skip = parse
  |  '"' 
     { let (buf, fin, _m, kind, args) = sargs 
       let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-      fin.Finish buf kind false cont
+      fin.Finish buf kind LexerStringFinisherContext.Verbatim cont
     }
 
  |  '"''B' 
     { let (buf, fin, _m, kind, args) = sargs 
       let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-      fin.Finish buf { kind with IsByteString = true } false cont
+      fin.Finish buf { kind with IsByteString = true } LexerStringFinisherContext.Verbatim cont
     }
 
  | newline 
@@ -1244,7 +1261,7 @@ and verbatimString sargs skip = parse
           let m2 = lexbuf.LexemeRange
           args.stringNest <- (1, LexerStringStyle.Verbatim, m2) :: args.stringNest
           let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-          fin.Finish buf kind true cont
+          fin.Finish buf kind (enum<LexerStringFinisherContext>(3)) cont
       else
           addUnicodeString buf (lexeme lexbuf) 
           if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.Verbatim, kind, m))
@@ -1297,7 +1314,7 @@ and tripleQuoteString sargs skip = parse
  |  '"' '"' '"' 
     { let (buf, fin, _m, kind, args) = sargs 
       let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-      fin.Finish buf kind false cont }
+      fin.Finish buf kind (enum<LexerStringFinisherContext>(4)) cont }
 
  | newline 
     { let (buf, _fin, m, kind, args) = sargs 
@@ -1340,7 +1357,7 @@ and tripleQuoteString sargs skip = parse
           let m2 = lexbuf.LexemeRange
           args.stringNest <- (1, LexerStringStyle.TripleQuote, m2) :: args.stringNest
           let cont = LexCont.Token(args.ifdefStack, args.stringNest)
-          fin.Finish buf kind true cont
+          fin.Finish buf kind (enum<LexerStringFinisherContext>(5)) cont
       else
           addUnicodeString buf (lexeme lexbuf) 
           if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.TripleQuote, kind, m))

src/fsharp/lexhelp.fs

@@ -18,6 +18,7 @@ open FSharp.Compiler.SourceCodeServices
 open FSharp.Compiler.SourceCodeServices.PrettyNaming
 open FSharp.Compiler.Text
 open FSharp.Compiler.Text.Range
+open FSharp.Compiler.SyntaxTree
 
 /// The "mock" filename used by fsi.exe when reading from stdin.
 /// Has special treatment by the lexer, i.e. __SOURCE_DIRECTORY__ becomes GetCurrentDirectory()
@@ -119,31 +120,54 @@ let stringBufferAsBytes (buf: ByteBuffer) =
     let bytes = buf.Close()
     Array.init (bytes.Length / 2) (fun i -> bytes.[i*2]) 
 
+[<Flags>]
+type LexerStringFinisherContext = 
+    | InterpolatedPart = 1
+    | Verbatim = 2
+    | TripleQuote = 4
+
 type LexerStringFinisher =
-    | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> bool -> LexerContinuation -> token)
+    | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> LexerStringFinisherContext -> LexerContinuation -> token)
 
-    member fin.Finish (buf: ByteBuffer) kind isInterpolatedStringPart cont =
+    member fin.Finish (buf: ByteBuffer) kind context cont =
         let (LexerStringFinisher f)  = fin
-        f buf kind isInterpolatedStringPart cont
+        f buf kind context cont
 
     static member Default =
-        LexerStringFinisher (fun buf kind isPart cont ->
+        LexerStringFinisher (fun buf kind context cont ->
+            let isPart = context.HasFlag(LexerStringFinisherContext.InterpolatedPart)
+            let isVerbatim = context.HasFlag(LexerStringFinisherContext.Verbatim)
+            let isTripleQuote = context.HasFlag(LexerStringFinisherContext.TripleQuote)
+
             if kind.IsInterpolated then 
                 let s = stringBufferAsString buf
-                if kind.IsInterpolatedFirst then 
+                if kind.IsInterpolatedFirst then
+                    let synStringKind =
+                        if isTripleQuote then
+                            SynStringKind.TripleQuote
+                        else
+                            SynStringKind.Regular
                     if isPart then 
-                        INTERP_STRING_BEGIN_PART (s, cont)
+                        INTERP_STRING_BEGIN_PART (s, synStringKind, cont)
                     else
-                        INTERP_STRING_BEGIN_END (s, cont)
+                        INTERP_STRING_BEGIN_END (s, synStringKind, cont)
                 else
                     if isPart then
                         INTERP_STRING_PART (s, cont)
                     else
                         INTERP_STRING_END (s, cont)
-            elif kind.IsByteString then 
-                BYTEARRAY (stringBufferAsBytes buf, cont)
+            elif kind.IsByteString then
+                let synByteStringKind = if isVerbatim then SynByteStringKind.Verbatim else SynByteStringKind.Regular
+                BYTEARRAY (stringBufferAsBytes buf, synByteStringKind, cont)
             else
-                STRING (stringBufferAsString buf, cont)
+                let synStringKind =
+                    if isVerbatim then
+                        SynStringKind.Verbatim
+                    elif isTripleQuote then
+                        SynStringKind.TripleQuote
+                    else
+                        SynStringKind.Regular
+                STRING (stringBufferAsString buf, synStringKind, cont)
         ) 
 
 let addUnicodeString (buf: ByteBuffer) (x:string) =

src/fsharp/lexhelp.fsi

@@ -54,10 +54,15 @@ val reusingLexbufForParsing: UnicodeLexing.Lexbuf -> (unit -> 'a) -> 'a
 
 val usingLexbufForParsing: UnicodeLexing.Lexbuf * string -> (UnicodeLexing.Lexbuf -> 'a) -> 'a
 
+type LexerStringFinisherContext = 
+    | InterpolatedPart = 1
+    | Verbatim = 2
+    | TripleQuote = 4
+
 type LexerStringFinisher =
-    | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> bool -> LexerContinuation -> token)
+    | LexerStringFinisher of (ByteBuffer -> LexerStringKind -> LexerStringFinisherContext -> LexerContinuation -> token)
 
-    member Finish: buf: ByteBuffer -> kind: LexerStringKind -> isInterpolatedStringPart: bool -> cont: LexerContinuation -> token
+    member Finish: buf: ByteBuffer -> kind: LexerStringKind -> context: LexerStringFinisherContext -> cont: LexerContinuation -> token
 
     static member Default: LexerStringFinisher