From fa7c56f1c9ad4a597ff931e6dac1be80bdeeaa2b Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 15 Aug 2023 13:38:47 -0700 Subject: [PATCH 1/8] Basic progress --- packages/compiler/src/server/serverlib.ts | 25 ++++++++++++++++- .../compiler/test/server/colorization.test.ts | 28 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index a10ff34b8a2..9a8081986c6 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -925,7 +925,9 @@ export function createServer(host: ServerHost): Server { const file = ast.file; const tokens = mapTokens(); classifyNode(ast); - return Array.from(tokens.values()).filter((t) => t.kind !== undefined); + return Array.from(tokens.values()) + .filter((t) => t.kind !== undefined) + .sort((a, b) => a.pos - b.pos); function mapTokens() { const tokens = new Map(); @@ -954,6 +956,7 @@ export function createServer(host: ServerHost): Server { case Token.NumericLiteral: return SemanticTokenKind.Number; case Token.MultiLineComment: + return ignore; case Token.SingleLineComment: return SemanticTokenKind.Comment; default: @@ -1050,6 +1053,20 @@ export function createServer(host: ServerHost): Server { case SyntaxKind.ProjectionMemberExpression: classifyReference(node.id); break; + // case SyntaxKind.Doc: + // classifyReference(node.id); + // break; + case SyntaxKind.DocParamTag: + case SyntaxKind.DocTemplateTag: + classifyReference(node.tagName, SemanticTokenKind.Keyword); + classifyReference(node.paramName, SemanticTokenKind.Parameter); + break; + case SyntaxKind.DocReturnsTag: + case SyntaxKind.DocUnknownTag: + classifyReference(node.tagName, SemanticTokenKind.Keyword); + break; + default: + break; } visitChildren(node, classifyNode); } @@ -1058,6 +1075,12 @@ export function createServer(host: ServerHost): Server { const token = tokens.get(node.pos); if (token && token.kind === undefined) { token.kind = kind; + } else if (token === undefined) { + tokens.set(node.pos, { + pos: node.pos, + end: node.end, + kind: kind, + }); } } diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index ee63ed8cf0f..868e0c9c2a7 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -100,10 +100,14 @@ const Token = { string: (text: string) => createToken(text.startsWith('"') ? text : '"' + text + '"', "string.quoted.double.tsp"), }, + comment: { + block: (text: string) => createToken(text, "comment.block.tsp"), + line: (text: string) => createToken(text, "comment.line.double-slash.tsp"), + }, } as const; testColorization("semantic colorization", tokenizeSemantic); -testColorization("tmlanguage", tokenizeTMLanguage); +// testColorization("tmlanguage", tokenizeTMLanguage); function testColorization(description: string, tokenize: Tokenize) { describe(`compiler: server: ${description}`, () => { @@ -809,6 +813,26 @@ function testColorization(description: string, tokenize: Tokenize) { }); }); + describe("doc comments", () => { + it("tokenize @param", async () => { + const tokens = await tokenize( + `/** + * Doc comment + * @param foo Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.keywords.alias, + Token.identifiers.type("A"), + Token.operators.assignment, + Token.literals.numeric("1"), + Token.punctuation.semicolon, + ]); + }); + }); + describe("projections", () => { it("simple projection", async () => { const tokens = await tokenize(` @@ -995,6 +1019,8 @@ export async function tokenizeSemantic(input: string): Promise { return Token.keywords.other(text); case SemanticTokenKind.String: return Token.literals.string(text); + case SemanticTokenKind.Comment: + return Token.comment.block(text); case SemanticTokenKind.Number: return Token.literals.numeric(text); case SemanticTokenKind.Operator: From f3de1181a733e99e8ed70a55250f2a79179361cf Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Tue, 15 Aug 2023 15:52:16 -0700 Subject: [PATCH 2/8] Add doc comment highlighting in tmlanguage --- packages/compiler/src/server/serverlib.ts | 4 +- packages/compiler/src/server/tmlanguage.ts | 51 +++++++++++++++++++ .../compiler/test/server/colorization.test.ts | 19 ++++++- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index 9a8081986c6..e83542db6d8 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -925,9 +925,7 @@ export function createServer(host: ServerHost): Server { const file = ast.file; const tokens = mapTokens(); classifyNode(ast); - return Array.from(tokens.values()) - .filter((t) => t.kind !== undefined) - .sort((a, b) => a.pos - b.pos); + return Array.from(tokens.values()).filter((t) => t.kind !== undefined); function mapTokens() { const tokens = new Map(); diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index b7209255954..b46e30712f5 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -24,6 +24,7 @@ export type TypeSpecScope = | "string.quoted.double.tsp" | "string.quoted.triple.tsp" | "variable.name.tsp" + | "keyword.doc.tag.tspdoc" // Operators | "keyword.operator.type.annotation.tsp" | "keyword.operator.assignment.tsp" @@ -160,12 +161,62 @@ const blockComment: BeginEndRule = { end: "\\*/", }; +const docCommentParam: MatchRule = { + key: "doc-comment-param", + scope: "comment.block.tsp", + match: `(?x)((@)(?:param|template))\\s+(${identifier})\\b`, + captures: { + "1": { scope: "keyword.doc.tag.tspdoc" }, + "2": { scope: "keyword.doc.tag.tspdoc" }, + "3": { scope: "variable.name.tsp" }, + }, +}; +const docCommentReturn: MatchRule = { + key: "doc-comment-return-tag", + scope: "comment.block.tsp", + match: `(?x)((@)(?:returns))\\b`, + captures: { + "1": { scope: "keyword.doc.tag.tspdoc" }, + "2": { scope: "keyword.doc.tag.tspdoc" }, + }, +}; +const docCommentUnknownTag: MatchRule = { + key: "doc-comment-unknown-tag", + scope: "comment.block.tsp", + match: `(?x)((@)(?:${identifier}))\\b`, + captures: { + "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, + }, +}; + +const docCommentBlock: IncludeRule = { + key: "doc-comment-block", + patterns: [docCommentParam, docCommentReturn, docCommentUnknownTag], +}; + +const docComment: BeginEndRule = { + key: "doc-comment", + scope: "comment.block.tsp", + begin: "/\\**", + beginCaptures: { + "0": { scope: "comment.block.tsp" }, + }, + end: "\\*/", + endCaptures: { + "0": { scope: "comment.block.tsp" }, + }, + patterns: [docCommentBlock], +}; + // Tokens that match standing alone in any context: literals and comments const token: IncludeRule = { key: "token", patterns: [ + docComment, lineComment, blockComment, + // `"""` must come before `"` or first two quotes of `"""` will match as // empty string tripleQuotedStringLiteral, diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index 868e0c9c2a7..b9bf3a1b23b 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -107,7 +107,7 @@ const Token = { } as const; testColorization("semantic colorization", tokenizeSemantic); -// testColorization("tmlanguage", tokenizeTMLanguage); +testColorization("tmlanguage", tokenizeTMLanguage); function testColorization(description: string, tokenize: Tokenize) { describe(`compiler: server: ${description}`, () => { @@ -121,6 +121,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.type("string"), ]); }); + it("templated alias", async () => { const tokens = await tokenize("alias Foo = T"); deepStrictEqual(tokens, [ @@ -813,10 +814,14 @@ function testColorization(description: string, tokenize: Tokenize) { }); }); + /** + * Doc comment + * @param foo Foo desc + */ describe("doc comments", () => { it("tokenize @param", async () => { const tokens = await tokenize( - `/** + `/** * Doc comment * @param foo Foo desc */ @@ -824,6 +829,16 @@ function testColorization(description: string, tokenize: Tokenize) { ); deepStrictEqual(tokens, [ + Token.comment.block("/**"), + Token.comment.block(" * Doc comment"), + Token.comment.block(" * "), + Token.identifiers.tag("@"), + Token.identifiers.tag("param"), + Token.comment.block(" "), + Token.identifiers.type("foo"), + Token.comment.block(" Foo desc"), + Token.comment.block(" "), + Token.comment.block("*/"), Token.keywords.alias, Token.identifiers.type("A"), Token.operators.assignment, From 39a658b45ef11461283f748e86428fc8eeb191c6 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Wed, 16 Aug 2023 15:50:25 -0700 Subject: [PATCH 3/8] fix --- packages/compiler/test/server/colorization.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index b9bf3a1b23b..fa31b57a1b8 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -69,6 +69,10 @@ const Token = { type: (name: string) => createToken(name, "entity.name.type.tsp"), }, + tspdoc: { + tag: (name: string) => createToken(name, "keyword.doc.tag.tspdoc"), + }, + operators: { assignment: createToken("=", "keyword.operator.assignment.tsp"), optional: createToken("?", "keyword.operator.optional.tsp"), @@ -832,10 +836,10 @@ function testColorization(description: string, tokenize: Tokenize) { Token.comment.block("/**"), Token.comment.block(" * Doc comment"), Token.comment.block(" * "), - Token.identifiers.tag("@"), - Token.identifiers.tag("param"), + Token.tspdoc.tag("@"), + Token.tspdoc.tag("param"), Token.comment.block(" "), - Token.identifiers.type("foo"), + Token.identifiers.variable("foo"), Token.comment.block(" Foo desc"), Token.comment.block(" "), Token.comment.block("*/"), From 54b9adcef323a9fa3ca25720882c412997b2dd0d Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 17 Aug 2023 14:05:35 -0700 Subject: [PATCH 4/8] Colorization of doc comments --- packages/compiler/src/server/serverlib.ts | 88 ++++++++++++++----- packages/compiler/src/server/tmlanguage.ts | 4 +- .../compiler/test/server/colorization.test.ts | 73 ++++++++++++--- packages/playground/src/services.ts | 2 + 4 files changed, 131 insertions(+), 36 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index e83542db6d8..fecd97ac802 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -70,6 +70,7 @@ import { import { Program, compile as compileProgram } from "../core/program.js"; import { Token, + TokenFlags, createScanner, isKeyword, isPunctuation, @@ -177,6 +178,8 @@ export enum SemanticTokenKind { Number, Regexp, Operator, + + DocCommentTag, } export interface SemanticToken { @@ -932,17 +935,37 @@ export function createServer(host: ServerHost): Server { const scanner = createScanner(file, () => {}); while (scanner.scan() !== Token.EndOfFile) { - const kind = classifyToken(scanner.token); - if (kind === ignore) { - continue; + if (scanner.tokenFlags & TokenFlags.DocComment) { + classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position }); + } else { + const kind = classifyToken(scanner.token); + if (kind === ignore) { + continue; + } + tokens.set(scanner.tokenPosition, { + kind: kind === defer ? undefined! : kind, + pos: scanner.tokenPosition, + end: scanner.position, + }); } - tokens.set(scanner.tokenPosition, { - kind: kind === defer ? undefined! : kind, - pos: scanner.tokenPosition, - end: scanner.position, - }); } return tokens; + + function classifyDocComment(range: TextRange) { + scanner.scanRange(range, () => { + while (scanner.scanDoc() !== Token.EndOfFile) { + const kind = classifyDocToken(scanner.token); + if (kind === ignore) { + continue; + } + tokens.set(scanner.tokenPosition, { + kind: kind === defer ? undefined! : kind, + pos: scanner.tokenPosition, + end: scanner.position, + }); + } + }); + } } function classifyToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { @@ -954,7 +977,6 @@ export function createServer(host: ServerHost): Server { case Token.NumericLiteral: return SemanticTokenKind.Number; case Token.MultiLineComment: - return ignore; case Token.SingleLineComment: return SemanticTokenKind.Comment; default: @@ -968,6 +990,23 @@ export function createServer(host: ServerHost): Server { } } + /** Classify tokens when scanning doc comment. */ + function classifyDocToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { + switch (token) { + case Token.NewLine: + case Token.Whitespace: + return ignore; + case Token.DocText: + case Token.Star: + case Token.Identifier: + return SemanticTokenKind.Comment; + case Token.At: + return defer; + default: + return ignore; + } + } + function classifyNode(node: Node) { switch (node.kind) { case SyntaxKind.DirectiveExpression: @@ -1051,17 +1090,16 @@ export function createServer(host: ServerHost): Server { case SyntaxKind.ProjectionMemberExpression: classifyReference(node.id); break; - // case SyntaxKind.Doc: - // classifyReference(node.id); - // break; case SyntaxKind.DocParamTag: case SyntaxKind.DocTemplateTag: - classifyReference(node.tagName, SemanticTokenKind.Keyword); - classifyReference(node.paramName, SemanticTokenKind.Parameter); + classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag); + classifyOverride(node.paramName, SemanticTokenKind.Variable); break; case SyntaxKind.DocReturnsTag: + classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag); + break; case SyntaxKind.DocUnknownTag: - classifyReference(node.tagName, SemanticTokenKind.Keyword); + classifyDocTag(node.tagName, SemanticTokenKind.Type); break; default: break; @@ -1069,16 +1107,24 @@ export function createServer(host: ServerHost): Server { visitChildren(node, classifyNode); } + function classifyDocTag(node: IdentifierNode, kind: SemanticTokenKind) { + classifyOverride(node, kind); + const token = tokens.get(node.pos - 1); // Get the `@` token + if (token) { + token.kind = kind; + } + } + function classify(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) { const token = tokens.get(node.pos); if (token && token.kind === undefined) { token.kind = kind; - } else if (token === undefined) { - tokens.set(node.pos, { - pos: node.pos, - end: node.end, - kind: kind, - }); + } + } + function classifyOverride(node: IdentifierNode | StringLiteralNode, kind: SemanticTokenKind) { + const token = tokens.get(node.pos); + if (token) { + token.kind = kind; } } diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index b46e30712f5..b10e2e3a652 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -185,8 +185,8 @@ const docCommentUnknownTag: MatchRule = { scope: "comment.block.tsp", match: `(?x)((@)(?:${identifier}))\\b`, captures: { - "1": { scope: "entity.name.tag.tsp" }, - "2": { scope: "entity.name.tag.tsp" }, + "1": { scope: "entity.name.type.tsp" }, + "2": { scope: "entity.name.type.tsp" }, }, }; diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index fa31b57a1b8..04d1ceedce9 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -823,8 +823,20 @@ function testColorization(description: string, tokenize: Tokenize) { * @param foo Foo desc */ describe("doc comments", () => { + async function tokenizeDocComment(text: string) { + const tokens = await tokenize(text); + return tokens.filter((x) => !(x.scope === "comment.block.tsp")); + } + + const common = [ + Token.keywords.alias, + Token.identifiers.type("A"), + Token.operators.assignment, + Token.literals.numeric("1"), + Token.punctuation.semicolon, + ]; it("tokenize @param", async () => { - const tokens = await tokenize( + const tokens = await tokenizeDocComment( `/** * Doc comment * @param foo Foo desc @@ -833,21 +845,54 @@ function testColorization(description: string, tokenize: Tokenize) { ); deepStrictEqual(tokens, [ - Token.comment.block("/**"), - Token.comment.block(" * Doc comment"), - Token.comment.block(" * "), Token.tspdoc.tag("@"), Token.tspdoc.tag("param"), - Token.comment.block(" "), Token.identifiers.variable("foo"), - Token.comment.block(" Foo desc"), - Token.comment.block(" "), - Token.comment.block("*/"), - Token.keywords.alias, - Token.identifiers.type("A"), - Token.operators.assignment, - Token.literals.numeric("1"), - Token.punctuation.semicolon, + ...common, + ]); + }); + + it("tokenize @template", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @template foo Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.tspdoc.tag("@"), + Token.tspdoc.tag("template"), + Token.identifiers.variable("foo"), + ...common, + ]); + }); + + it("tokenize @returns", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @returns Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [Token.tspdoc.tag("@"), Token.tspdoc.tag("returns"), ...common]); + }); + it("tokenize @custom", async () => { + const tokens = await tokenizeDocComment( + `/** + * Doc comment + * @custom Foo desc + */ + alias A = 1;` + ); + + deepStrictEqual(tokens, [ + Token.identifiers.type("@"), + Token.identifiers.type("custom"), + ...common, ]); }); }); @@ -1048,6 +1093,8 @@ export async function tokenizeSemantic(input: string): Promise { const punctuation = punctuationMap.get(text); ok(punctuation, `No tmlanguage equivalent for punctuation: "${text}".`); return punctuation; + case SemanticTokenKind.DocCommentTag: + return Token.tspdoc.tag(text); default: ok(false, "Unexpected SemanticTokenKind: " + SemanticTokenKind[token.kind]); } diff --git a/packages/playground/src/services.ts b/packages/playground/src/services.ts index a25fa4aa667..6410e8f19f5 100644 --- a/packages/playground/src/services.ts +++ b/packages/playground/src/services.ts @@ -333,6 +333,8 @@ export async function registerMonacoLanguage(host: BrowserHost) { case "property": case "enumMember": return "variable"; + case "docCommentTag": + return "keyword"; default: return entry; } From 40bb1e54f0deda682c123796526456dd43ec8e5f Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 17 Aug 2023 14:41:08 -0700 Subject: [PATCH 5/8] Fix up --- packages/compiler/src/server/serverlib.ts | 2 +- packages/compiler/src/server/tmlanguage.ts | 29 +++++++---- .../compiler/test/server/colorization.test.ts | 50 ++++++++----------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index fecd97ac802..b98dc9c61b3 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -1099,7 +1099,7 @@ export function createServer(host: ServerHost): Server { classifyDocTag(node.tagName, SemanticTokenKind.DocCommentTag); break; case SyntaxKind.DocUnknownTag: - classifyDocTag(node.tagName, SemanticTokenKind.Type); + classifyDocTag(node.tagName, SemanticTokenKind.Macro); break; default: break; diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index b10e2e3a652..12b75e94b8a 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -11,20 +11,27 @@ type MatchRule = tm.MatchRule; type Grammar = tm.Grammar; export type TypeSpecScope = + // Comments | "comment.block.tsp" | "comment.line.double-slash.tsp" + // Constants | "constant.character.escape.tsp" | "constant.numeric.tsp" | "constant.language.tsp" + // Keywords | "keyword.directive.name.tsp" + | "keyword.other.tsp" + | "keyword.tag.tspdoc" + // Entities | "entity.name.type.tsp" | "entity.name.function.tsp" | "entity.name.tag.tsp" - | "keyword.other.tsp" + | "entity.name.function.macro.tsp" + // Strings | "string.quoted.double.tsp" | "string.quoted.triple.tsp" + // Variables | "variable.name.tsp" - | "keyword.doc.tag.tspdoc" // Operators | "keyword.operator.type.annotation.tsp" | "keyword.operator.assignment.tsp" @@ -166,8 +173,8 @@ const docCommentParam: MatchRule = { scope: "comment.block.tsp", match: `(?x)((@)(?:param|template))\\s+(${identifier})\\b`, captures: { - "1": { scope: "keyword.doc.tag.tspdoc" }, - "2": { scope: "keyword.doc.tag.tspdoc" }, + "1": { scope: "keyword.tag.tspdoc" }, + "2": { scope: "keyword.tag.tspdoc" }, "3": { scope: "variable.name.tsp" }, }, }; @@ -176,8 +183,8 @@ const docCommentReturn: MatchRule = { scope: "comment.block.tsp", match: `(?x)((@)(?:returns))\\b`, captures: { - "1": { scope: "keyword.doc.tag.tspdoc" }, - "2": { scope: "keyword.doc.tag.tspdoc" }, + "1": { scope: "keyword.tag.tspdoc" }, + "2": { scope: "keyword.tag.tspdoc" }, }, }; const docCommentUnknownTag: MatchRule = { @@ -185,8 +192,8 @@ const docCommentUnknownTag: MatchRule = { scope: "comment.block.tsp", match: `(?x)((@)(?:${identifier}))\\b`, captures: { - "1": { scope: "entity.name.type.tsp" }, - "2": { scope: "entity.name.type.tsp" }, + "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, }, }; @@ -243,9 +250,10 @@ const parenthesizedExpression: BeginEndRule = { const decorator: BeginEndRule = { key: "decorator", scope: meta, - begin: `(@${qualifiedIdentifier})`, + begin: `((@)${qualifiedIdentifier})`, beginCaptures: { "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, }, end: `${beforeIdentifier}|${universalEnd}`, patterns: [token, parenthesizedExpression], @@ -254,9 +262,10 @@ const decorator: BeginEndRule = { const augmentDecoratorStatement: BeginEndRule = { key: "augment-decorator-statement", scope: meta, - begin: `(@@${qualifiedIdentifier})`, + begin: `((@@)${qualifiedIdentifier})`, beginCaptures: { "1": { scope: "entity.name.tag.tsp" }, + "2": { scope: "entity.name.tag.tsp" }, }, end: `${beforeIdentifier}|${universalEnd}`, patterns: [token, parenthesizedExpression], diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index 04d1ceedce9..5680914e498 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -70,7 +70,7 @@ const Token = { }, tspdoc: { - tag: (name: string) => createToken(name, "keyword.doc.tag.tspdoc"), + tag: (name: string) => createToken(name, "keyword.tag.tspdoc"), }, operators: { @@ -190,13 +190,13 @@ function testColorization(description: string, tokenize: Tokenize) { describe("decorators", () => { it("simple parameterless decorator", async () => { const tokens = await tokenize("@foo"); - deepStrictEqual(tokens, [Token.identifiers.tag("@foo")]); + deepStrictEqual(tokens, [Token.identifiers.tag("@"), Token.identifiers.tag("foo")]); }); it("fully qualified decorator name", async () => { const tokens = await tokenize("@Foo.bar"); if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@Foo.bar")]); + deepStrictEqual(tokens, [Token.identifiers.tag("@"), Token.identifiers.tag("Foo.bar")]); } else { deepStrictEqual(tokens, [ Token.identifiers.tag("@"), @@ -210,7 +210,8 @@ function testColorization(description: string, tokenize: Tokenize) { it("decorator with parameters", async () => { const tokens = await tokenize(`@foo("param1", 123)`); deepStrictEqual(tokens, [ - Token.identifiers.tag("@foo"), + Token.identifiers.tag("@"), + Token.identifiers.tag("foo"), Token.punctuation.openParen, Token.literals.string("param1"), Token.punctuation.comma, @@ -233,22 +234,22 @@ function testColorization(description: string, tokenize: Tokenize) { it("decorator", async () => { const tokens = await tokenize(`@@foo(MyModel, "param1", 123)`); - if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@@foo"), ...params]); - } else { - deepStrictEqual(tokens, [ - Token.identifiers.tag("@@"), - Token.identifiers.tag("foo"), - ...params, - ]); - } + deepStrictEqual(tokens, [ + Token.identifiers.tag("@@"), + Token.identifiers.tag("foo"), + ...params, + ]); }); it("fully qualified decorator name", async () => { const tokens = await tokenize(`@@Foo.bar(MyModel, "param1", 123)`); if (tokenize === tokenizeTMLanguage) { - deepStrictEqual(tokens, [Token.identifiers.tag("@@Foo.bar"), ...params]); + deepStrictEqual(tokens, [ + Token.identifiers.tag("@@"), + Token.identifiers.tag("Foo.bar"), + ...params, + ]); } else { deepStrictEqual(tokens, [ Token.identifiers.tag("@@"), @@ -730,13 +731,15 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.functionName("foo"), Token.punctuation.openParen, - Token.identifiers.tag("@path"), + Token.identifiers.tag("@"), + Token.identifiers.tag("path"), Token.identifiers.variable("param1"), Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.punctuation.comma, - Token.identifiers.tag("@query"), + Token.identifiers.tag("@"), + Token.identifiers.tag("query"), Token.identifiers.variable("param2"), Token.operators.optional, Token.operators.typeAnnotation, @@ -890,8 +893,8 @@ function testColorization(description: string, tokenize: Tokenize) { ); deepStrictEqual(tokens, [ - Token.identifiers.type("@"), - Token.identifiers.type("custom"), + Token.identifiers.tag("@"), + Token.identifiers.tag("custom"), ...common, ]); }); @@ -1047,17 +1050,6 @@ export async function tokenizeSemantic(input: string): Promise { } } - // Make @myDec one token to match tmlanguage - for (let i = 0; i < tokens.length - 1; i++) { - if ( - tokens[i].scope === "entity.name.tag.tsp" && - tokens[i].text === "@" && - tokens[i + 1].scope === "entity.name.tag.tsp" - ) { - tokens[i].text = "@" + tokens[i + 1].text; - tokens.splice(i + 1, 1); - } - } return tokens; function convertSemanticToken(token: SemanticToken, text: string): Token | undefined { From 53a3c4d9dd6b1a4620c71141786717b8857b11f1 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 17 Aug 2023 14:42:39 -0700 Subject: [PATCH 6/8] changelog --- .../feature-doc-comment-color_2023-08-17-21-42.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json diff --git a/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json b/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json new file mode 100644 index 00000000000..755448413f8 --- /dev/null +++ b/common/changes/@typespec/compiler/feature-doc-comment-color_2023-08-17-21-42.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/compiler", + "comment": "**IDE** Add coloring for doc comment", + "type": "none" + } + ], + "packageName": "@typespec/compiler" +} \ No newline at end of file From 3f313153ad6fb78618f11c83daaf923c6ddd67b7 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Wed, 6 Sep 2023 16:05:18 -0700 Subject: [PATCH 7/8] Fix doc coloring in vscode --- packages/typespec-vscode/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/typespec-vscode/package.json b/packages/typespec-vscode/package.json index 3b52f40d619..b9c026db76b 100644 --- a/packages/typespec-vscode/package.json +++ b/packages/typespec-vscode/package.json @@ -97,6 +97,9 @@ ], "macro": [ "entity.name.tag.tsp" + ], + "docCommentTag": [ + "keyword.other.tsp" ] } } From 6698690a58c4af9bf8233c4824f6a8324dd63c9d Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 7 Sep 2023 10:48:15 -0700 Subject: [PATCH 8/8] changelog --- .../feature-doc-comment-color_2023-09-07-17-48.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json diff --git a/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json b/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json new file mode 100644 index 00000000000..61f0dadf97a --- /dev/null +++ b/common/changes/typespec-vscode/feature-doc-comment-color_2023-09-07-17-48.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "typespec-vscode", + "comment": "Add colors definition for `param`", + "type": "none" + } + ], + "packageName": "typespec-vscode" +} \ No newline at end of file