Add support for the css-variables theme to ansi rendering

docs/themes.md

@@ -87,9 +87,44 @@ Note that this client-side theme is less granular than most other supported VSCo
     --shiki-token-string-expression: #CC0000;
     --shiki-token-punctuation: #DD0000;
     --shiki-token-link: #EE0000;
+
+    /* Only required if using ansiToHtml function */
+    --shiki-color-ansi-black: #000000;
+    --shiki-color-ansi-black-dim: #00000080;
+    --shiki-color-ansi-red: #bb0000;
+    --shiki-color-ansi-red-dim: #bb000080;
+    --shiki-color-ansi-green: #00bb00;
+    --shiki-color-ansi-green-dim: #00bb0080;
+    --shiki-color-ansi-yellow: #bbbb00;
+    --shiki-color-ansi-yellow-dim: #bbbb0080;
+    --shiki-color-ansi-blue: #0000bb;
+    --shiki-color-ansi-blue-dim: #0000bb80;
+    --shiki-color-ansi-magenta: #ff00ff;
+    --shiki-color-ansi-magenta-dim: #ff00ff80;
+    --shiki-color-ansi-cyan: #00bbbb;
+    --shiki-color-ansi-cyan-dim: #00bbbb80;
+    --shiki-color-ansi-white: #eeeeee;
+    --shiki-color-ansi-white-dim: #eeeeee80;
+    --shiki-color-ansi-bright-black: #555555;
+    --shiki-color-ansi-bright-black-dim: #55555580;
+    --shiki-color-ansi-bright-red: #ff5555;
+    --shiki-color-ansi-bright-red-dim: #ff555580;
+    --shiki-color-ansi-bright-green: #00ff00;
+    --shiki-color-ansi-bright-green-dim: #00ff0080;
+    --shiki-color-ansi-bright-yellow: #ffff55;
+    --shiki-color-ansi-bright-yellow-dim: #ffff5580;
+    --shiki-color-ansi-bright-blue: #5555ff;
+    --shiki-color-ansi-bright-blue-dim: #5555ff80;
+    --shiki-color-ansi-bright-magenta: #ff55ff;
+    --shiki-color-ansi-bright-magenta-dim: #ff55ff80;
+    --shiki-color-ansi-bright-cyan: #55ffff;
+    --shiki-color-ansi-bright-cyan-dim: #55ffff80;
+    --shiki-color-ansi-bright-white: #ffffff;
+    --shiki-color-ansi-bright-white-dim: #ffffff80;
   }
 </style>
 ```
+
 ## All Themes
 
 ```ts

packages/shiki/src/__tests__/__snapshots__/ansi.test.ts.snap

@@ -1,5 +1,46 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`renders ansi to html with css-variables theme 1`] = `
+"<pre class=\\"shiki css-variables\\" style=\\"background-color: var(--shiki-color-background)\\" tabindex=\\"0\\"><code><span class=\\"line\\"></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-black)\\">black</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-black-dim)\\">black (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-red)\\">red</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-red-dim)\\">red (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-green)\\">green</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-green-dim)\\">green (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-yellow)\\">yellow</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-yellow-dim)\\">yellow (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-blue)\\">blue</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-blue-dim)\\">blue (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-magenta)\\">magenta</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-magenta-dim)\\">magenta (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-cyan)\\">cyan</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-cyan-dim)\\">cyan (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-white)\\">white</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-white-dim)\\">white (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-black)\\">brightBlack</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-black-dim)\\">brightBlack (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-red)\\">brightRed</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-red-dim)\\">brightRed (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-green)\\">brightGreen</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-green-dim)\\">brightGreen (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-yellow)\\">brightYellow</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-yellow-dim)\\">brightYellow (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-blue)\\">brightBlue</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-blue-dim)\\">brightBlue (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-magenta)\\">brightMagenta</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-magenta-dim)\\">brightMagenta (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-cyan)\\">brightCyan</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-cyan-dim)\\">brightCyan (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-white)\\">brightWhite</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-ansi-bright-white-dim)\\">brightWhite (dim)</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-text); font-weight: bold\\">bold</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-text); font-style: italic\\">italic</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-text); font-style: italic\\">underline</span></span>
+<span class=\\"line\\"><span style=\\"color: var(--shiki-color-text)\\">strikethrough</span></span>
+<span class=\\"line\\"></span></code></pre>"
+`;
+
 exports[`renders ansi to html with default theme 1`] = `
 "<pre class=\\"shiki monokai\\" style=\\"background-color: #272822\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span style=\\"color: #333333\\"> WARN </span><span style=\\"color: #F8F8F2\\"> using --force I sure hope you know what you are doing</span></span>
 <span class=\\"line\\"><span style=\\"color: #F8F8F2\\">Scope: all 6 workspace projects</span></span>

packages/shiki/src/__tests__/ansi.test.ts

@@ -45,3 +45,52 @@ Done in 15.7s`)
 
   expect(out).toMatchSnapshot()
 })
+
+test('renders ansi to html with css-variables theme', async () => {
+  const highlighter = await getHighlighter({
+    theme: 'css-variables'
+  })
+
+  const allAnsiStyles = `
+black
+black (dim)
+red
+red (dim)
+green
+green (dim)
+yellow
+yellow (dim)
+blue
+blue (dim)
+magenta
+magenta (dim)
+cyan
+cyan (dim)
+white
+white (dim)
+brightBlack
+brightBlack (dim)
+brightRed
+brightRed (dim)
+brightGreen
+brightGreen (dim)
+brightYellow
+brightYellow (dim)
+brightBlue
+brightBlue (dim)
+brightMagenta
+brightMagenta (dim)
+brightCyan
+brightCyan (dim)
+brightWhite
+brightWhite (dim)
+bold
+italic
+underline
+strikethrough
+`
+
+  const out = highlighter.ansiToHtml(allAnsiStyles)
+
+  expect(out).toMatchSnapshot()
+})

packages/shiki/src/ansiThemedTokenizer.ts

@@ -51,7 +51,7 @@ export function tokenizeAnsiWithTheme(theme: IShikiTheme, fileContents: string):
 }
 
 /**
- * Adds 50% alpha to a hex color string
+ * Adds 50% alpha to a hex color string or the "-dim" postfix to a CSS variable
  */
 function dimColor(color: string) {
   const hexMatch = color.match(/#([0-9a-f]{3})([0-9a-f]{3})?([0-9a-f]{2})?/)
@@ -72,5 +72,11 @@ function dimColor(color: string) {
         .join('')}80`
     }
   }
+
+  const cssVarMatch = color.match(/var\((--shiki-color-ansi-[\w-]+)\)/)
+  if (cssVarMatch) {
+    return `var(${cssVarMatch[1]}-dim)`
+  }
+
   return color
 }

packages/shiki/src/highlighter.ts

@@ -17,6 +17,7 @@ import { getOniguruma, WASM_PATH } from './loader'
 import { Lang, languages as BUNDLED_LANGUAGES } from './languages'
 import { Registry } from './registry'
 import { Theme } from './themes'
+import { namedColors as ansiNamedColors } from 'ansi-sequence-parser'
 
 function resolveLang(lang: ILanguageRegistration | Lang) {
   return typeof lang === 'string'
@@ -46,6 +47,32 @@ function resolveOptions(options: HighlighterOptions) {
   return { _languages, _themes, _wasmPath }
 }
 
+function generateDefaultColorReplacements() {
+  const replacements: Record<string, string> = {
+    '#000001': 'var(--shiki-color-text)',
+    '#000002': 'var(--shiki-color-background)',
+    '#000004': 'var(--shiki-token-constant)',
+    '#000005': 'var(--shiki-token-string)',
+    '#000006': 'var(--shiki-token-comment)',
+    '#000007': 'var(--shiki-token-keyword)',
+    '#000008': 'var(--shiki-token-parameter)',
+    '#000009': 'var(--shiki-token-function)',
+    '#000010': 'var(--shiki-token-string-expression)',
+    '#000011': 'var(--shiki-token-punctuation)',
+    '#000012': 'var(--shiki-token-link)'
+  }
+
+  // Generates a color replacement for each ANSI color with the form #A00000, #A00001, etc.
+  // Note that dimmed colors will use variables with the name var(--shiki-color-ansi-<color>-dim)
+  for (let i = 0; i < ansiNamedColors.length; i++) {
+    const code = `#A${i.toString().padStart(5, '0')}`
+    const colorNameKebab = ansiNamedColors[i].replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase()
+    replacements[code] = `var(--shiki-color-ansi-${colorNameKebab})`
+  }
+
+  return replacements
+}
+
 export async function getHighlighter(options: HighlighterOptions): Promise<Highlighter> {
   const { _languages, _themes, _wasmPath } = resolveOptions(options)
   const _resolver = new Resolver(getOniguruma(_wasmPath), 'vscode-oniguruma')
@@ -75,25 +102,18 @@ export async function getHighlighter(options: HighlighterOptions): Promise<Highl
    * Instead, we work around this by using valid hex color codes as lookups in a
    * final "repair" step which translates those codes to the correct CSS variables.
    */
-  let COLOR_REPLACEMENTS: Record<string, string> = {
-    '#000001': 'var(--shiki-color-text)',
-    '#000002': 'var(--shiki-color-background)',
-    '#000004': 'var(--shiki-token-constant)',
-    '#000005': 'var(--shiki-token-string)',
-    '#000006': 'var(--shiki-token-comment)',
-    '#000007': 'var(--shiki-token-keyword)',
-    '#000008': 'var(--shiki-token-parameter)',
-    '#000009': 'var(--shiki-token-function)',
-    '#000010': 'var(--shiki-token-string-expression)',
-    '#000011': 'var(--shiki-token-punctuation)',
-    '#000012': 'var(--shiki-token-link)'
-  }
+  let COLOR_REPLACEMENTS = generateDefaultColorReplacements()
   function setColorReplacements(map: Record<string, string>) {
     COLOR_REPLACEMENTS = map
   }
   function fixCssVariablesTheme(theme: IShikiTheme, colorMap: string[]) {
     theme.bg = COLOR_REPLACEMENTS[theme.bg] || theme.bg
     theme.fg = COLOR_REPLACEMENTS[theme.fg] || theme.fg
+
+    Object.entries(theme.colors).forEach(([key, value]) => {
+      theme.colors[key] = COLOR_REPLACEMENTS[value] || value
+    })
+
     colorMap.forEach((val, i) => {
       colorMap[i] = COLOR_REPLACEMENTS[val] || val
     })

packages/shiki/themes/css-variables.json

@@ -3,7 +3,23 @@
   "type": "css",
   "colors": {
     "editor.foreground": "#000001",
-    "editor.background": "#000002"
+    "editor.background": "#000002",
+    "terminal.ansiBlack": "#A00000",
+    "terminal.ansiRed": "#A00001",
+    "terminal.ansiGreen": "#A00002",
+    "terminal.ansiYellow": "#A00003",
+    "terminal.ansiBlue": "#A00004",
+    "terminal.ansiMagenta": "#A00005",
+    "terminal.ansiCyan": "#A00006",
+    "terminal.ansiWhite": "#A00007",
+    "terminal.ansiBrightBlack": "#A00008",
+    "terminal.ansiBrightRed": "#A00009",
+    "terminal.ansiBrightGreen": "#A00010",
+    "terminal.ansiBrightYellow": "#A00011",
+    "terminal.ansiBrightBlue": "#A00012",
+    "terminal.ansiBrightMagenta": "#A00013",
+    "terminal.ansiBrightCyan": "#A00014",
+    "terminal.ansiBrightWhite": "#A00015"
   },
   "tokenColors": [
     {