Distinguish raw and non-raw lifetime/label

Author: mattheww

src/combination.rs

@@ -57,6 +57,9 @@ pub enum CoarseTokenData {
     LifetimeOrLabel {
         name: Charseq,
     },
+    RawLifetimeOrLabel {
+        name: Charseq,
+    },
     ByteLiteral {
         represented_byte: u8,
         suffix: Charseq,
@@ -327,6 +330,9 @@ impl TryFrom<FineTokenData> for CoarseTokenData {
             FineTokenData::LifetimeOrLabel { name } => {
                 Ok(CoarseTokenData::LifetimeOrLabel { name })
             }
+            FineTokenData::RawLifetimeOrLabel { name } => {
+                Ok(CoarseTokenData::RawLifetimeOrLabel { name })
+            }
             FineTokenData::ByteLiteral {
                 represented_byte,
                 suffix,

src/lex_via_rustc.rs

@@ -78,6 +78,7 @@ pub enum RustcTokenData {
         identifier: String,
     },
     Lifetime {
+        style: RustcIdentIsRaw,
         /// This includes the leading '
         symbol: String,
     },
@@ -133,7 +134,7 @@ pub enum RustcDocCommentStyle {
     Outer,
 }
 
-/// Whether an identifier was written in raw form.
+/// Whether an identifier or lifetime/label was written in raw form.
 pub enum RustcIdentIsRaw {
     No,
     Yes,
@@ -398,7 +399,8 @@ fn token_from_ast_token(
             style: style.into(),
             identifier: symbol.to_string(),
         },
-        TokenKind::Lifetime(symbol, _is_raw) => RustcTokenData::Lifetime {
+        TokenKind::Lifetime(symbol, style) => RustcTokenData::Lifetime {
+            style: style.into(),
             symbol: symbol.to_string(),
         },
         TokenKind::Literal(rustc_ast::token::Lit {

src/lexlucid/pretokenisation.rs

@@ -58,6 +58,9 @@ pub enum PretokenData {
     LifetimeOrLabel {
         name: Charseq,
     },
+    RawLifetimeOrLabel {
+        name: Charseq,
+    },
     SingleQuoteLiteral {
         prefix: Charseq,
         literal_content: Charseq,

src/lexlucid/pretokenisation/pretokenisation_rules.rs

@@ -189,7 +189,7 @@ fn make_named_rules() -> BTreeMap<RuleName, Rule> {
        // Lifetime or label
        (RuleName::RawLifetimeOrLabel2021,
         Rule::new_regex(
-            |cp| PretokenData::LifetimeOrLabel {
+            |cp| PretokenData::RawLifetimeOrLabel {
                 name: cp["name"].into(),
             }, r##"\A
                 ' r \#

src/lexlucid/reprocessing.rs

@@ -53,6 +53,9 @@ pub enum FineTokenData {
     LifetimeOrLabel {
         name: Charseq,
     },
+    RawLifetimeOrLabel {
+        name: Charseq,
+    },
     CharacterLiteral {
         represented_character: char,
         suffix: Charseq,
@@ -158,6 +161,9 @@ pub fn reprocess(pretoken: &Pretoken) -> Result<FineToken, Error> {
         PretokenData::LifetimeOrLabel { name } => {
             FineTokenData::LifetimeOrLabel { name: name.clone() }
         }
+        PretokenData::RawLifetimeOrLabel { name } => {
+            FineTokenData::RawLifetimeOrLabel { name: name.clone() }
+        }
         PretokenData::SingleQuoteLiteral {
             prefix,
             literal_content,

src/regular_tokens.rs

@@ -67,6 +67,7 @@ pub enum RegularTokenData {
     LifetimeOrLabel {
         /// This includes the leading '
         symbol: Charseq,
+        style: IdentifierStyle,
     },
     ByteLiteral {
         represented_byte: u8,
@@ -158,8 +159,12 @@ pub fn regularise_from_rustc(tokens: impl IntoIterator<Item = RustcToken>) -> Ve
                     represented_identifier: identifier.into(),
                     style: style.into(),
                 },
-                RustcTokenData::Lifetime { symbol: name } => RegularTokenData::LifetimeOrLabel {
+                RustcTokenData::Lifetime {
+                    style,
+                    symbol: name,
+                } => RegularTokenData::LifetimeOrLabel {
                     symbol: name.into(),
+                    style: style.into(),
                 },
                 RustcTokenData::Lit { literal_data } => regularise_rustc_literal(literal_data)
                     .expect("rustc token represented an error"),
@@ -296,6 +301,11 @@ fn from_coarse_token(token: CoarseToken) -> RegularTokenData {
         },
         CoarseTokenData::LifetimeOrLabel { name } => RegularTokenData::LifetimeOrLabel {
             symbol: once('\'').chain(name.iter().copied()).collect(),
+            style: IdentifierStyle::NonRaw,
+        },
+        CoarseTokenData::RawLifetimeOrLabel { name } => RegularTokenData::LifetimeOrLabel {
+            symbol: once('\'').chain(name.iter().copied()).collect(),
+            style: IdentifierStyle::Raw,
         },
         CoarseTokenData::CharacterLiteral {
             represented_character,