diff --git a/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json b/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json new file mode 100644 index 00000000000..755448413f8 --- /dev/null +++ b/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/compiler", + "comment": "**IDE** Add coloring for doc comment", + "type": "none" + } + ], + "packageName": "@typespec/compiler" +} \ No newline at end of file diff --git a/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json b/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json new file mode 100644 index 00000000000..61f0dadf97a --- /dev/null +++ b/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "typespec-vscode", + "comment": "Add colors definition for `param`", + "type": "none" + } + ], + "packageName": "typespec-vscode" +} \ No newline at end of file diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index 160040bffa7..45a34b99eef 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -70,6 +70,7 @@ import { import { Program, compile as compileProgram } from "../core/program.js"; import { Token, + TokenFlags, createScanner, isKeyword, isPunctuation, @@ -177,6 +178,8 @@ export enum SemanticTokenKind { Number, Regexp, Operator, + + DocCommentTag, } export interface SemanticToken { @@ -935,17 +938,37 @@ export function createServer(host: ServerHost): Server { const scanner = createScanner(file, () => {}); while (scanner.scan() !== Token.EndOfFile) { - const kind = classifyToken(scanner.token); - if (kind === ignore) { - continue; + if (scanner.tokenFlags & TokenFlags.DocComment) { + classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position }); + } else { + const kind = classifyToken(scanner.token); + if (kind === ignore) { + continue; + } + tokens.set(scanner.tokenPosition, { + kind: kind === defer ? undefined! : kind, + pos: scanner.tokenPosition, + end: scanner.position, + }); } - tokens.set(scanner.tokenPosition, { - kind: kind === defer ? undefined! : kind, - pos: scanner.tokenPosition, - end: scanner.position, - }); } return tokens; + + function classifyDocComment(range: TextRange) { + scanner.scanRange(range, () => { + while (scanner.scanDoc() !== Token.EndOfFile) { + const kind = classifyDocToken(scanner.token); + if (kind === ignore) { + continue; + } + tokens.set(scanner.tokenPosition, { + kind: kind === defer ? undefined! : kind, + pos: scanner.tokenPosition, + end: scanner.position, + }); + } + }); + } } function classifyToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { @@ -970,6 +993,23 @@ export function createServer(host: ServerHost): Server { } } + /** Classify tokens when scanning doc comment. */ + function classifyDocToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { + switch (token) { + case Token.NewLine: + case Token.Whitespace: + return ignore; + case Token.DocText: + case Token.Star: + case Token.Identifier: + return SemanticTokenKind.Comment; + case Token.At: + return defer; + default: + return ignore; + } + } + function classifyNode(node: Node) { switch (node.kind) { case SyntaxKind.DirectiveExpression: @@ -1053,16 +1093,43 @@ export function createServer(host: ServerHost): Server { case SyntaxKind.ProjectionMemberExpression: classifyReference(node.id); break; + case SyntaxKind.DocParamTag: + case SyntaxKind.DocTemplateTag: + classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag); + classifyOverride(node.paramName, SemanticTokenKind.Variable); + break; + case SyntaxKind.DocReturnsTag: + classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag); + break; + case SyntaxKind.DocUnknownTag: + classifyDocTag(node.tagName, SemanticTokenKind.Macro); + break; + default: + break; } visitChildren(node, classifyNode); } + function classifyDocTag(node: IdentifierNode, kind: SemanticTokenKind) { + classifyOverride(node, kind); + const token = tokens.get(node.pos - 1); // Get the `@` token + if (token) { + token.kind = kind; + } + } + function classify(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) { const token = tokens.get(node.pos); if (token && token.kind === undefined) { token.kind = kind; } } + function classifyOverride(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) { + const token = tokens.get(node.pos); + if (token) { + token.kind = kind; + } + } function classifyReference(node: Node, kind = SemanticTokenKind.Type) { switch (node.kind) { diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index b7209255954..12b75e94b8a 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -11,18 +11,26 @@ type MatchRule = tm.MatchRule; type Grammar = tm.Grammar; export type TypeSpecScope = + // Comments | "comment.block.tsp" | "comment.line.double-slash.tsp" + // Constants | "constant.character.escape.tsp" | "constant.numeric.tsp" | "constant.language.tsp" + // Keywords | "keyword.directive.name.tsp" + | "keyword.other.tsp" + | "keyword.tag.tspdoc" + // Entities | "entity.name.type.tsp" | "entity.name.function.tsp" | "entity.name.tag.tsp" - | "keyword.other.tsp" + | "entity.name.function.macro.tsp" + // Strings | "string.quoted.double.tsp" | "string.quoted.triple.tsp" + // Variables | "variable.name.tsp" // Operators | "keyword.operator.type.annotation.tsp" @@ -160,12 +168,62 @@ const blockComment: BeginEndRule = { end: "\\*/", }; +const docCommentParam: MatchRule = { + key: "doc-comment-param", + scope: "comment.block.tsp", + match: `(?x)((@)(?:param|template))\\s+(${identifier})\\b`, + captures: { + "1": { scope: "keyword.tag.tspdoc" }, + "2": { scope: "keyword.tag.tspdoc" }, + "3": { scope: "variable.name.tsp" }, + }, +}; +const docCommentReturn: MatchRule = { + key: "doc-comment-return-tag", + scope: "comment.block.tsp", + match: `(?x)((@)(?:returns))\\b`, + captures: { + "1": { scope: "keyword.tag.tspdoc" }, + "2": { scope: "keyword.tag.tspdoc" }, + }, +}; +const docCommentUnknownTag: MatchRule = { + key: "doc-comment-unknown-tag", + scope: "comment.block.tsp", + match: `(?x)((@)(?:${identifier}))\\b`, + captures: { + "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, + }, +}; + +const docCommentBlock: IncludeRule = { + key: "doc-comment-block", + patterns: [docCommentParam, docCommentReturn, docCommentUnknownTag], +}; + +const docComment: BeginEndRule = { + key: "doc-comment", + scope: "comment.block.tsp", + begin: "/\\**", + beginCaptures: { + "0": { scope: "comment.block.tsp" }, + }, + end: "\\*/", + endCaptures: { + "0": { scope: "comment.block.tsp" }, + }, + patterns: [docCommentBlock], +}; + // Tokens that match standing alone in any context: literals and comments const token: IncludeRule = { key: "token", patterns: [ + docComment, lineComment, blockComment, + // `"""` must come before `"` or first two quotes of `"""` will match as // empty string tripleQuotedStringLiteral, @@ -192,9 +250,10 @@ const parenthesizedExpression: BeginEndRule = { const decorator: BeginEndRule = { key: "decorator", scope: meta, - begin: `(@${qualifiedIdentifier})`, + begin: `((@)${qualifiedIdentifier})`, beginCaptures: { "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, }, end: `${beforeIdentifier}|${universalEnd}`, patterns: [token, parenthesizedExpression], @@ -203,9 +262,10 @@ const decorator: BeginEndRule = { const augmentDecoratorStatement: BeginEndRule = { key: "augment-decorator-statement", scope: meta, - begin: `(@@${qualifiedIdentifier})`, + begin: `((@@)${qualifiedIdentifier})`, beginCaptures: { "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, }, end: `${beforeIdentifier}|${universalEnd}`, patterns: [token, parenthesizedExpression], diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index ee63ed8cf0f..5680914e498 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -69,6 +69,10 @@ const Token = { type: (name: string) => createToken(name, "entity.name.type.tsp"), }, + tspdoc: { + tag: (name: string) => createToken(name, "keyword.tag.tspdoc"), + }, + operators: { assignment: createToken("=", "keyword.operator.assignment.tsp"), optional: createToken("?", "keyword.operator.optional.tsp"), @@ -100,6 +104,10 @@ const Token = { string: (text: string) => createToken(text.startsWith('"') ? text : '"' + text + '"', "string.quoted.double.tsp"), }, + comment: { + block: (text: string) => createToken(text, "comment.block.tsp"), + line: (text: string) => createToken(text, "comment.line.double-slash.tsp"), + }, } as const; testColorization("semantic colorization", tokenizeSemantic); @@ -117,6 +125,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.type("string"), ]); }); + it("templated alias", async () => { const tokens = await tokenize("alias Foo = T"); deepStrictEqual(tokens, [ @@ -181,13 +190,13 @@ function testColorization(description: string, tokenize: Tokenize) { describe("decorators", () => { it("simple parameterless decorator", async () => { const tokens = await tokenize("@foo"); - deepStrictEqual(tokens, [Token.identifiers.tag("@foo")]); + deepStrictEqual(tokens, [Token.identifiers.tag("@"), Token.identifiers.tag("foo")]); }); it("fully qualified decorator name", async () => { const tokens = await tokenize("@Foo.bar"); if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@Foo.bar")]); + deepStrictEqual(tokens, [Token.identifiers.tag("@"), Token.identifiers.tag("Foo.bar")]); } else { deepStrictEqual(tokens, [ Token.identifiers.tag("@"), @@ -201,7 +210,8 @@ function testColorization(description: string, tokenize: Tokenize) { it("decorator with parameters", async () => { const tokens = await tokenize(`@foo("param1", 123)`); deepStrictEqual(tokens, [ - Token.identifiers.tag("@foo"), + Token.identifiers.tag("@"), + Token.identifiers.tag("foo"), Token.punctuation.openParen, Token.literals.string("param1"), Token.punctuation.comma, @@ -224,22 +234,22 @@ function testColorization(description: string, tokenize: Tokenize) { it("decorator", async () => { const tokens = await tokenize(`@@foo(MyModel, "param1", 123)`); - if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@@foo"), ...params]); - } else { - deepStrictEqual(tokens, [ - Token.identifiers.tag("@@"), - Token.identifiers.tag("foo"), - ...params, - ]); - } + deepStrictEqual(tokens, [ + Token.identifiers.tag("@@"), + Token.identifiers.tag("foo"), + ...params, + ]); }); it("fully qualified decorator name", async () => { const tokens = await tokenize(`@@Foo.bar(MyModel, "param1", 123)`); if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@@Foo.bar"), ...params]); + deepStrictEqual(tokens, [ + Token.identifiers.tag("@@"), + Token.identifiers.tag("Foo.bar"), + ...params, + ]); } else { deepStrictEqual(tokens, [ Token.identifiers.tag("@@"), @@ -721,13 +731,15 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.functionName("foo"), Token.punctuation.openParen, - Token.identifiers.tag("@path"), + Token.identifiers.tag("@"), + Token.identifiers.tag("path"), Token.identifiers.variable("param1"), Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.punctuation.comma, - Token.identifiers.tag("@query"), + Token.identifiers.tag("@"), + Token.identifiers.tag("query"), Token.identifiers.variable("param2"), Token.operators.optional, Token.operators.typeAnnotation, @@ -809,6 +821,85 @@ function testColorization(description: string, tokenize: Tokenize) { }); }); + /** + * Doc comment + * @param foo Foo desc + */ + describe("doc comments", () => { + async function tokenizeDocComment(text: string) { + const tokens = await tokenize(text); + return tokens.filter((x) => !(x.scope === "comment.block.tsp")); + } + + const common = [ + Token.keywords.alias, + Token.identifiers.type("A"), + Token.operators.assignment, + Token.literals.numeric("1"), + Token.punctuation.semicolon, + ]; + it("tokenize @param", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @param foo Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.tspdoc.tag("@"), + Token.tspdoc.tag("param"), + Token.identifiers.variable("foo"), + ...common, + ]); + }); + + it("tokenize @template", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @template foo Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.tspdoc.tag("@"), + Token.tspdoc.tag("template"), + Token.identifiers.variable("foo"), + ...common, + ]); + }); + + it("tokenize @returns", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @returns Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [Token.tspdoc.tag("@"), Token.tspdoc.tag("returns"), ...common]); + }); + it("tokenize @custom", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @custom Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.identifiers.tag("@"), + Token.identifiers.tag("custom"), + ...common, + ]); + }); + }); + describe("projections", () => { it("simple projection", async () => { const tokens = await tokenize(` @@ -959,17 +1050,6 @@ export async function tokenizeSemantic(input: string): Promise { } } - // Make @myDec one token to match tmlanguage - for (let i = 0; i < tokens.length - 1; i++) { - if ( - tokens[i].scope === "entity.name.tag.tsp" && - tokens[i].text === "@" && - tokens[i + 1].scope === "entity.name.tag.tsp" - ) { - tokens[i].text = "@" + tokens[i + 1].text; - tokens.splice(i + 1, 1); - } - } return tokens; function convertSemanticToken(token: SemanticToken, text: string): Token | undefined { @@ -995,6 +1075,8 @@ export async function tokenizeSemantic(input: string): Promise { return Token.keywords.other(text); case SemanticTokenKind.String: return Token.literals.string(text); + case SemanticTokenKind.Comment: + return Token.comment.block(text); case SemanticTokenKind.Number: return Token.literals.numeric(text); case SemanticTokenKind.Operator: @@ -1003,6 +1085,8 @@ export async function tokenizeSemantic(input: string): Promise { const punctuation = punctuationMap.get(text); ok(punctuation, `No tmlanguage equivalent for punctuation: "${text}".`); return punctuation; + case SemanticTokenKind.DocCommentTag: + return Token.tspdoc.tag(text); default: ok(false, "Unexpected SemanticTokenKind: " + SemanticTokenKind[token.kind]); } diff --git a/packages/playground/src/services.ts b/packages/playground/src/services.ts index a25fa4aa667..6410e8f19f5 100644 --- a/packages/playground/src/services.ts +++ b/packages/playground/src/services.ts @@ -333,6 +333,8 @@ export async function registerMonacoLanguage(host: BrowserHost) { case "property": case "enumMember": return "variable"; + case "docCommentTag": + return "keyword"; default: return entry; } diff --git a/packages/typespec-vscode/package.json b/packages/typespec-vscode/package.json index 3b52f40d619..b9c026db76b 100644 --- a/packages/typespec-vscode/package.json +++ b/packages/typespec-vscode/package.json @@ -97,6 +97,9 @@ ], "macro": [ "entity.name.tag.tsp" + ], + "docCommentTag": [ + "keyword.other.tsp" ] } }