Feature: Add colorization to js doc comments

common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/compiler",
+      "comment": "**IDE** Add coloring for doc comment",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/compiler"
+}

common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "typespec-vscode",
+      "comment": "Add colors definition for `param`",
+      "type": "none"
+    }
+  ],
+  "packageName": "typespec-vscode"
+}

packages/compiler/src/server/serverlib.ts

@@ -70,6 +70,7 @@ import {
 import { Program, compile as compileProgram } from "../core/program.js";
 import {
   Token,
+  TokenFlags,
   createScanner,
   isKeyword,
   isPunctuation,
@@ -177,6 +178,8 @@ export enum SemanticTokenKind {
   Number,
   Regexp,
   Operator,
+
+  DocCommentTag,
 }
 
 export interface SemanticToken {
@@ -935,17 +938,37 @@ export function createServer(host: ServerHost): Server {
       const scanner = createScanner(file, () => {});
 
       while (scanner.scan() !== Token.EndOfFile) {
-        const kind = classifyToken(scanner.token);
-        if (kind === ignore) {
-          continue;
+        if (scanner.tokenFlags & TokenFlags.DocComment) {
+          classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position });
+        } else {
+          const kind = classifyToken(scanner.token);
+          if (kind === ignore) {
+            continue;
+          }
+          tokens.set(scanner.tokenPosition, {
+            kind: kind === defer ? undefined! : kind,
+            pos: scanner.tokenPosition,
+            end: scanner.position,
+          });
         }
-        tokens.set(scanner.tokenPosition, {
-          kind: kind === defer ? undefined! : kind,
-          pos: scanner.tokenPosition,
-          end: scanner.position,
-        });
       }
       return tokens;
+
+      function classifyDocComment(range: TextRange) {
+        scanner.scanRange(range, () => {
+          while (scanner.scanDoc() !== Token.EndOfFile) {
+            const kind = classifyDocToken(scanner.token);
+            if (kind === ignore) {
+              continue;
+            }
+            tokens.set(scanner.tokenPosition, {
+              kind: kind === defer ? undefined! : kind,
+              pos: scanner.tokenPosition,
+              end: scanner.position,
+            });
+          }
+        });
+      }
     }
 
     function classifyToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore {
@@ -970,6 +993,23 @@ export function createServer(host: ServerHost): Server {
       }
     }
 
+    /** Classify tokens when scanning doc comment. */
+    function classifyDocToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore {
+      switch (token) {
+        case Token.NewLine:
+        case Token.Whitespace:
+          return ignore;
+        case Token.DocText:
+        case Token.Star:
+        case Token.Identifier:
+          return SemanticTokenKind.Comment;
+        case Token.At:
+          return defer;
+        default:
+          return ignore;
+      }
+    }
+
     function classifyNode(node: Node) {
       switch (node.kind) {
         case SyntaxKind.DirectiveExpression:
@@ -1053,16 +1093,43 @@ export function createServer(host: ServerHost): Server {
         case SyntaxKind.ProjectionMemberExpression:
           classifyReference(node.id);
           break;
+        case SyntaxKind.DocParamTag:
+        case SyntaxKind.DocTemplateTag:
+          classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag);
+          classifyOverride(node.paramName, SemanticTokenKind.Variable);
+          break;
+        case SyntaxKind.DocReturnsTag:
+          classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag);
+          break;
+        case SyntaxKind.DocUnknownTag:
+          classifyDocTag(node.tagName, SemanticTokenKind.Macro);
+          break;
+        default:
+          break;
       }
       visitChildren(node, classifyNode);
     }
 
+    function classifyDocTag(node: IdentifierNode, kind: SemanticTokenKind) {
+      classifyOverride(node, kind);
+      const token = tokens.get(node.pos - 1); // Get the `@` token
+      if (token) {
+        token.kind = kind;
+      }
+    }
+
     function classify(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) {
       const token = tokens.get(node.pos);
       if (token && token.kind === undefined) {
         token.kind = kind;
       }
     }
+    function classifyOverride(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) {
+      const token = tokens.get(node.pos);
+      if (token) {
+        token.kind = kind;
+      }
+    }
 
     function classifyReference(node: Node, kind = SemanticTokenKind.Type) {
       switch (node.kind) {

packages/compiler/src/server/tmlanguage.ts

@@ -11,18 +11,26 @@ type MatchRule = tm.MatchRule<TypeSpecScope>;
 type Grammar = tm.Grammar<TypeSpecScope>;
 
 export type TypeSpecScope =
+  // Comments
   | "comment.block.tsp"
   | "comment.line.double-slash.tsp"
+  // Constants
   | "constant.character.escape.tsp"
   | "constant.numeric.tsp"
   | "constant.language.tsp"
+  // Keywords
   | "keyword.directive.name.tsp"
+  | "keyword.other.tsp"
+  | "keyword.tag.tspdoc"
+  // Entities
   | "entity.name.type.tsp"
   | "entity.name.function.tsp"
   | "entity.name.tag.tsp"
-  | "keyword.other.tsp"
+  | "entity.name.function.macro.tsp"
+  // Strings
   | "string.quoted.double.tsp"
   | "string.quoted.triple.tsp"
+  // Variables
   | "variable.name.tsp"
   // Operators
   | "keyword.operator.type.annotation.tsp"
@@ -160,12 +168,62 @@ const blockComment: BeginEndRule = {
   end: "\\*/",
 };
 
+const docCommentParam: MatchRule = {
+  key: "doc-comment-param",
+  scope: "comment.block.tsp",
+  match: `(?x)((@)(?:param|template))\\s+(${identifier})\\b`,
+  captures: {
+    "1": { scope: "keyword.tag.tspdoc" },
+    "2": { scope: "keyword.tag.tspdoc" },
+    "3": { scope: "variable.name.tsp" },
+  },
+};
+const docCommentReturn: MatchRule = {
+  key: "doc-comment-return-tag",
+  scope: "comment.block.tsp",
+  match: `(?x)((@)(?:returns))\\b`,
+  captures: {
+    "1": { scope: "keyword.tag.tspdoc" },
+    "2": { scope: "keyword.tag.tspdoc" },
+  },
+};
+const docCommentUnknownTag: MatchRule = {
+  key: "doc-comment-unknown-tag",
+  scope: "comment.block.tsp",
+  match: `(?x)((@)(?:${identifier}))\\b`,
+  captures: {
+    "1": { scope: "entity.name.tag.tsp" },
+    "2": { scope: "entity.name.tag.tsp" },
+  },
+};
+
+const docCommentBlock: IncludeRule = {
+  key: "doc-comment-block",
+  patterns: [docCommentParam, docCommentReturn, docCommentUnknownTag],
+};
+
+const docComment: BeginEndRule = {
+  key: "doc-comment",
+  scope: "comment.block.tsp",
+  begin: "/\\**",
+  beginCaptures: {
+    "0": { scope: "comment.block.tsp" },
+  },
+  end: "\\*/",
+  endCaptures: {
+    "0": { scope: "comment.block.tsp" },
+  },
+  patterns: [docCommentBlock],
+};
+
 // Tokens that match standing alone in any context: literals and comments
 const token: IncludeRule = {
   key: "token",
   patterns: [
+    docComment,
     lineComment,
     blockComment,
+
     // `"""` must come before `"` or first two quotes of `"""` will match as
     // empty string
     tripleQuotedStringLiteral,
@@ -192,9 +250,10 @@ const parenthesizedExpression: BeginEndRule = {
 const decorator: BeginEndRule = {
   key: "decorator",
   scope: meta,
-  begin: `(@${qualifiedIdentifier})`,
+  begin: `((@)${qualifiedIdentifier})`,
   beginCaptures: {
     "1": { scope: "entity.name.tag.tsp" },
+    "2": { scope: "entity.name.tag.tsp" },
   },
   end: `${beforeIdentifier}|${universalEnd}`,
   patterns: [token, parenthesizedExpression],
@@ -203,9 +262,10 @@ const decorator: BeginEndRule = {
 const augmentDecoratorStatement: BeginEndRule = {
   key: "augment-decorator-statement",
   scope: meta,
-  begin: `(@@${qualifiedIdentifier})`,
+  begin: `((@@)${qualifiedIdentifier})`,
   beginCaptures: {
     "1": { scope: "entity.name.tag.tsp" },
+    "2": { scope: "entity.name.tag.tsp" },
   },
   end: `${beforeIdentifier}|${universalEnd}`,
   patterns: [token, parenthesizedExpression],