Rework the description of the top-level nonterminals

Change the name "token-kind nonterminal" to "tokenisation nonterminal".

"token nonterminal" now refers to a tokenisation nonterminal which isn't a
reserved form, rather than to the TOKEN_yyy nonterminals.

Define them explicitly using the fine-grained tokens table and a list of
reserved forms, rather than saying they are the nonterminals which appear in
the top-level choice expressions.

Author: mattheww

src/framework/simple_reports.rs

@@ -20,7 +20,7 @@ use crate::datatypes::trees::Forest;
 use crate::reimplementation::cleaning::{self, CleaningOutcome};
 use crate::reimplementation::doc_lowering::lower_doc_comments;
 use crate::reimplementation::fine_tokens::FineToken;
-use crate::reimplementation::tokenisation::{self, TokenKindMatch};
+use crate::reimplementation::tokenisation::{self, TokenisationMatch};
 use crate::rustc_harness::lex_via_rustc;
 use crate::tokens_common::Origin;
 use crate::{CleaningMode, Edition, Lowering};
@@ -128,7 +128,7 @@ pub enum DetailsMode {
     Always,
 }
 
-fn describe_match(match_data: &TokenKindMatch) -> impl Iterator<Item = String> + use<'_> {
+fn describe_match(match_data: &TokenisationMatch) -> impl Iterator<Item = String> + use<'_> {
     once(format!(
         "{:?}, {:?}",
         match_data.matched_nonterminal, match_data.consumed
@@ -249,7 +249,7 @@ fn show_inspect(input: &str, edition: Edition, cleaning: CleaningMode, lowering:
                     println!("  error: {message}");
                 }
             }
-            println!("  -- token-kind nonterminal matches --");
+            println!("  -- tokenisation nonterminal matches --");
             for match_data in matches {
                 for s in describe_match(&match_data) {
                     println!("  {s}",);
@@ -274,7 +274,7 @@ fn show_inspect(input: &str, edition: Edition, cleaning: CleaningMode, lowering:
                 }
                 tokenisation::Reason::Processing(message, rejected, matches, tokens) => {
                     println!(
-                        "lex_via_peg: {failure_label} when processing a match of a token-kind nonterminal"
+                        "lex_via_peg: {failure_label} when processing a match of a tokenisation nonterminal"
                     );
                     println!("  error: {message}");
                     println!("  -- when considering match --");
@@ -284,7 +284,7 @@ fn show_inspect(input: &str, edition: Edition, cleaning: CleaningMode, lowering:
                     (matches, tokens)
                 }
             };
-            println!("  -- previous token-kind nonterminal matches --");
+            println!("  -- previous tokenisation nonterminal matches --");
             for match_data in matches {
                 for s in describe_match(&match_data) {
                     println!("  {s}");

src/reimplementation/tokenisation.rs

@@ -8,7 +8,7 @@ use super::fine_tokens::FineToken;
 mod processing;
 mod tokens_matching;
 
-pub use tokens_matching::TokenKindMatch;
+pub use tokens_matching::TokenisationMatch;
 use tokens_matching::TokensMatchData;
 
 const MAX_INPUT_LENGTH: usize = 0x100_0000;
@@ -39,7 +39,7 @@ pub fn analyse(input: &Charseq, edition: Edition) -> Analysis {
     }
 
     let TokensMatchData {
-        token_kind_matches,
+        tokenisation_matches,
         consumed_entire_input: matched_entire_input,
     } = match tokens_matching::match_tokens(edition, input.chars()) {
         Ok(tokens_match_data) => tokens_match_data,
@@ -48,11 +48,11 @@ pub fn analyse(input: &Charseq, edition: Edition) -> Analysis {
         }
     };
 
-    // Note that if there's a processing error we only report the token-kind matches up to the match
-    // that failed processing.
+    // Note that if there's a processing error we only report the tokenisation matches up to the
+    // match that failed processing.
     let mut tokens = Vec::new();
     let mut reported_matches = Vec::new();
-    for match_data in token_kind_matches {
+    for match_data in tokenisation_matches {
         match processing::process(&match_data) {
             Ok(token) => {
                 reported_matches.push(match_data);
@@ -91,7 +91,7 @@ pub fn analyse(input: &Charseq, edition: Edition) -> Analysis {
 /// Result of running lexical analysis on a string.
 pub enum Analysis {
     /// Lexical analysis accepted the input.
-    Accepts(Vec<TokenKindMatch>, Vec<FineToken>),
+    Accepts(Vec<TokenisationMatch>, Vec<FineToken>),
 
     /// Lexical analysis rejected the input.
     Rejects(Reason),
@@ -102,23 +102,28 @@ pub enum Analysis {
 
 /// Explanation of why and where input was rejected.
 pub enum Reason {
-    /// Rejected when trying to match the edition's token nonterminal.
+    /// Rejected when trying to match the edition's tokens nonterminal.
     ///
     /// The string describes the reason for rejection (or a model error).
     ///
-    /// The lists of matches and tokens represent what was lexed successfully before the token
+    /// The lists of matches and tokens represent what was lexed successfully before the tokens
     /// nonterminal ceased to match.
-    Matching(String, Vec<TokenKindMatch>, Vec<FineToken>),
+    Matching(String, Vec<TokenisationMatch>, Vec<FineToken>),
 
-    /// Rejected when processing a match of a token-kind nonterminal.
+    /// Rejected when processing a match of a tokenisation nonterminal.
     ///
     /// The string describes the reason for rejection (or a model error).
     ///
     /// The single MatchData describes the match which was rejected (or which was being processed
     /// when we encountered a problem with the model).
     ///
     /// The lists of matches and tokens represent what was lexed successfully first.
-    Processing(String, TokenKindMatch, Vec<TokenKindMatch>, Vec<FineToken>),
+    Processing(
+        String,
+        TokenisationMatch,
+        Vec<TokenisationMatch>,
+        Vec<FineToken>,
+    ),
 }
 
 impl Reason {
@@ -159,13 +164,13 @@ impl Reason {
 /// Otherwise returns None.
 pub fn lex_as_single_token(input: &[char], edition: Edition) -> Option<FineToken> {
     let Ok(TokensMatchData {
-        token_kind_matches,
+        tokenisation_matches,
         consumed_entire_input: true,
     }) = tokens_matching::match_tokens(edition, input)
     else {
         return None;
     };
-    let [match_data] = &token_kind_matches[..] else {
+    let [match_data] = &tokenisation_matches[..] else {
         return None;
     };
     processing::process(match_data).ok()
@@ -186,14 +191,15 @@ pub fn first_nonwhitespace_token(input: &[char], edition: Edition) -> Option<Fin
 
     use crate::reimplementation::fine_tokens::{CommentStyle, FineTokenData::*};
 
-    let token_kind_matches = match tokens_matching::match_tokens(edition, input) {
+    let tokenisation_matches = match tokens_matching::match_tokens(edition, input) {
         Ok(TokensMatchData {
-            token_kind_matches, ..
-        }) => token_kind_matches,
+            tokenisation_matches,
+            ..
+        }) => tokenisation_matches,
         Err(_) => return None,
     };
 
-    for match_data in token_kind_matches {
+    for match_data in tokenisation_matches {
         let Ok(token) = processing::process(&match_data) else {
             return None;
         };

src/reimplementation/tokenisation/processing.rs

@@ -8,18 +8,18 @@ use crate::reimplementation::tokenisation::processing::escape_processing::{
 };
 use crate::tokens_common::{NumericBase, Origin};
 
-use super::tokens_matching::{Nonterminal, TokenKindMatch};
+use super::tokens_matching::{Nonterminal, TokenisationMatch};
 
 mod escape_processing;
 
 /// Converts a match to a fine-grained token, or rejects the match.
 ///
-/// This is the "Processing a match" stage of extracting a fine-grained token.
+/// This implements "Processing a tokenisation nonterminal match".
 ///
 /// If the match is accepted, returns a fine-grained token.
 ///
-/// If the match is rejected, distinguishes rejection from "model error".
-pub fn process(match_data: &TokenKindMatch) -> Result<FineToken, Error> {
+/// If the match is not accepted, distinguishes rejection from "model error".
+pub fn process(match_data: &TokenisationMatch) -> Result<FineToken, Error> {
     let token_data = match match_data.matched_nonterminal {
         Nonterminal::Whitespace => process_whitespace(match_data)?,
         Nonterminal::Line_comment => process_line_comment(match_data)?,
@@ -54,7 +54,7 @@ pub fn process(match_data: &TokenKindMatch) -> Result<FineToken, Error> {
                 match_data.matched_nonterminal
             )));
         }
-        _ => return model_error("unhandled token-kind nonterminal"),
+        _ => return model_error("unhandled tokenisation nonterminal"),
     };
     Ok(FineToken {
         data: token_data,
@@ -97,7 +97,7 @@ impl From<escape_processing::Error> for Error {
     }
 }
 
-impl TokenKindMatch {
+impl TokenisationMatch {
     /// Returns the characters consumed by the specified subsidiary nonterminal, or None if that
     /// nonterminal did not participate in the match.
     ///
@@ -138,11 +138,11 @@ impl TokenKindMatch {
     }
 }
 
-fn process_whitespace(_m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_whitespace(_m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     Ok(FineTokenData::Whitespace)
 }
 
-fn process_line_comment(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_line_comment(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let comment_content = m.consumed(Nonterminal::LINE_COMMENT_CONTENT)?;
     let (style, body) = match comment_content.chars() {
         ['/', '/', ..] => (CommentStyle::NonDoc, &[] as &[char]),
@@ -159,7 +159,7 @@ fn process_line_comment(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_block_comment(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_block_comment(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let comment_content = m.consumed_by_first_participating(Nonterminal::BLOCK_COMMENT_CONTENT)?;
     let (style, body) = match comment_content.chars() {
         ['*', '*', ..] => (CommentStyle::NonDoc, &[] as &[char]),
@@ -176,7 +176,7 @@ fn process_block_comment(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_character_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_character_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     use LiteralComponent::*;
     let single_quoted_content = m.consumed(Nonterminal::SINGLE_QUOTED_CONTENT)?;
     let single_escape_interpretation =
@@ -209,7 +209,7 @@ fn process_character_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error>
     })
 }
 
-fn process_byte_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_byte_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     use LiteralComponent::*;
     let single_quoted_content = m.consumed(Nonterminal::SINGLE_QUOTED_CONTENT)?;
     let single_escape_interpretation =
@@ -246,7 +246,7 @@ fn process_byte_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     use LiteralComponent::*;
     let double_quoted_content = m.consumed(Nonterminal::DOUBLE_QUOTED_CONTENT)?;
     let escape_interpretation = match try_escape_interpretation(double_quoted_content)? {
@@ -282,7 +282,7 @@ fn process_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_byte_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_byte_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     use LiteralComponent::*;
     let double_quoted_content = m.consumed(Nonterminal::DOUBLE_QUOTED_CONTENT)?;
     let escape_interpretation = match try_escape_interpretation(double_quoted_content)? {
@@ -321,7 +321,7 @@ fn process_byte_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Erro
     })
 }
 
-fn process_c_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_c_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     use LiteralComponent::*;
     let double_quoted_content = m.consumed(Nonterminal::DOUBLE_QUOTED_CONTENT)?;
     let escape_interpretation = match try_escape_interpretation(double_quoted_content)? {
@@ -377,7 +377,7 @@ fn process_c_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error>
     })
 }
 
-fn process_raw_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_raw_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let raw_double_quoted_content = m.consumed(Nonterminal::RAW_DOUBLE_QUOTED_CONTENT)?.clone();
     if raw_double_quoted_content.contains('\u{000d}') {
         return rejected("CR non-escape");
@@ -392,7 +392,7 @@ fn process_raw_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error
     })
 }
 
-fn process_raw_byte_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_raw_byte_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let raw_double_quoted_content = m.consumed(Nonterminal::RAW_DOUBLE_QUOTED_CONTENT)?;
     if raw_double_quoted_content.scalar_values().any(|n| n > 127) {
         return rejected("non-ASCII character");
@@ -414,7 +414,7 @@ fn process_raw_byte_string_literal(m: &TokenKindMatch) -> Result<FineTokenData,
     })
 }
 
-fn process_raw_c_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_raw_c_string_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let raw_double_quoted_content = m.consumed(Nonterminal::RAW_DOUBLE_QUOTED_CONTENT)?;
     if raw_double_quoted_content.contains('\u{000d}') {
         return rejected("CR in raw content");
@@ -433,7 +433,7 @@ fn process_raw_c_string_literal(m: &TokenKindMatch) -> Result<FineTokenData, Err
     })
 }
 
-fn process_float_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_float_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let body = match (
         m.maybe_consumed(Nonterminal::FLOAT_BODY_WITH_EXPONENT)?,
         m.maybe_consumed(Nonterminal::FLOAT_BODY_WITHOUT_EXPONENT)?,
@@ -453,7 +453,7 @@ fn process_float_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_integer_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_integer_literal(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let base = match (
         m.maybe_consumed(Nonterminal::INTEGER_BINARY_LITERAL)?,
         m.maybe_consumed(Nonterminal::INTEGER_OCTAL_LITERAL)?,
@@ -504,7 +504,7 @@ fn process_integer_literal(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     })
 }
 
-fn process_raw_lifetime_or_label(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_raw_lifetime_or_label(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let name = m.consumed(Nonterminal::IDENT)?.clone();
     let s = name.to_string();
     if s == "_" || s == "crate" || s == "self" || s == "super" || s == "Self" {
@@ -513,12 +513,12 @@ fn process_raw_lifetime_or_label(m: &TokenKindMatch) -> Result<FineTokenData, Er
     Ok(FineTokenData::RawLifetimeOrLabel { name })
 }
 
-fn process_lifetime_or_label(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_lifetime_or_label(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let name = m.consumed(Nonterminal::IDENT)?.clone();
     Ok(FineTokenData::LifetimeOrLabel { name })
 }
 
-fn process_raw_ident(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_raw_ident(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let represented_ident = m.consumed(Nonterminal::IDENT)?.nfc();
     let s = represented_ident.to_string();
     if s == "_" || s == "crate" || s == "self" || s == "super" || s == "Self" {
@@ -527,13 +527,13 @@ fn process_raw_ident(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
     Ok(FineTokenData::RawIdent { represented_ident })
 }
 
-fn process_ident(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_ident(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     Ok(FineTokenData::Ident {
         represented_ident: m.consumed(Nonterminal::IDENT)?.nfc(),
     })
 }
 
-fn process_punctuation(m: &TokenKindMatch) -> Result<FineTokenData, Error> {
+fn process_punctuation(m: &TokenisationMatch) -> Result<FineTokenData, Error> {
     let mark = match m.consumed.chars() {
         [c] => *c,
         _ => return rejected("impossible Punctuation match"),