feat!: change codeToThemedTokens options

Author: antfu

src/bundled/singleton.ts

@@ -1,4 +1,4 @@
-import type { BuiltinLanguages, BuiltinThemes, CodeToHtmlOptions, PlainTextLanguage, RequireKeys } from '../types'
+import type { BuiltinLanguages, BuiltinThemes, CodeToHtmlOptions, CodeToThemedTokensOptions, PlainTextLanguage, RequireKeys } from '../types'
 import { getHighlighter } from './highlighter'
 import type { Highlighter } from './highlighter'
 
@@ -23,7 +23,7 @@ async function getShikiWithThemeLang(options: { theme: BuiltinThemes; lang: Buil
 }
 
 /**
- * Shorthand for codeToHtml with auto-loaded theme and language.
+ * Shorthand for `codeToHtml` with auto-loaded theme and language.
  * A singleton highlighter it maintained internally.
  *
  * Differences from `shiki.codeToHtml()`, this function is async.
@@ -32,3 +32,14 @@ export async function codeToHtml(code: string, options: RequireKeys<CodeToHtmlOp
   const shiki = await getShikiWithThemeLang(options)
   return shiki.codeToHtml(code, options)
 }
+
+/**
+ * Shorthand for `codeToThemedTokens` with auto-loaded theme and language.
+ * A singleton highlighter it maintained internally.
+ *
+ * Differences from `shiki.codeToThemedTokens()`, this function is async.
+ */
+export async function codeToThemedTokens(code: string, options: RequireKeys<CodeToThemedTokensOptions<BuiltinLanguages, BuiltinThemes>, 'theme' | 'lang'>) {
+  const shiki = await getShikiWithThemeLang(options)
+  return shiki.codeToThemedTokens(code, options)
+}

src/core/index.ts

@@ -1,4 +1,4 @@
-import type { CodeToHtmlOptions, LanguageInput, MaybeGetter, ThemeInput } from '../types'
+import type { CodeToHtmlOptions, CodeToThemedTokensOptions, LanguageInput, MaybeGetter, ThemeInput } from '../types'
 import type { OnigurumaLoadOptions } from '../oniguruma'
 import { createOnigScanner, createOnigString, loadWasm } from '../oniguruma'
 import { Registry } from './registry'
@@ -55,17 +55,23 @@ export async function getHighlighterCore(options: HighlighterCoreOptions) {
 
   function codeToThemedTokens(
     code: string,
-    lang = 'text',
-    theme = defaultTheme,
-    options = { includeExplanation: true },
+    options: CodeToThemedTokensOptions = {},
   ) {
+    const {
+      lang = 'text',
+      theme = defaultTheme,
+      includeExplanation = true,
+    } = options
+
     if (isPlaintext(lang)) {
       const lines = code.split(/\r\n|\r|\n/)
       return [...lines.map(line => [{ content: line }])]
     }
     const _grammar = getLang(lang)
     const { _theme, _colorMap } = getTheme(theme)
-    return tokenizeWithTheme(_theme, _colorMap, code, _grammar, options)
+    return tokenizeWithTheme(code, _grammar, _theme, _colorMap, {
+      includeExplanation,
+    })
   }
 
   function getLang(name: string) {
@@ -88,7 +94,8 @@ export async function getHighlighterCore(options: HighlighterCoreOptions) {
   }
 
   function codeToHtml(code: string, options: CodeToHtmlOptions = {}): string {
-    const tokens = codeToThemedTokens(code, options.lang, options.theme, {
+    const tokens = codeToThemedTokens(code, {
+      ...options,
       includeExplanation: false,
     })
     const { _theme } = getTheme(options.theme)

src/core/themedTokenizer.ts

@@ -10,10 +10,10 @@ import type { FontStyle } from './stackElementMetadata'
 import { StackElementMetadata } from './stackElementMetadata'
 
 export function tokenizeWithTheme(
-  theme: IRawTheme,
-  colorMap: string[],
   fileContents: string,
   grammar: IGrammar,
+  theme: IRawTheme,
+  colorMap: string[],
   options: { includeExplanation?: boolean },
 ): ThemedToken[][] {
   const lines = fileContents.split(/\r\n|\r|\n/)

src/types.ts

@@ -47,6 +47,17 @@ export interface CodeToHtmlOptions<Languages = string, Themes = string> {
   lineOptions?: LineOption[]
 }
 
+export interface CodeToThemedTokensOptions<Languages = string, Themes = string> {
+  lang?: Languages | PlainTextLanguage
+  theme?: Themes
+  /**
+   * Include explanation of why a token is given a color.
+   *
+   * @default true
+   */
+  includeExplanation?: boolean
+}
+
 export interface LineOption {
   /**
    * 1-based line number.

test/singleton.test.ts

@@ -1,12 +1,70 @@
 import { describe, expect, it } from 'vitest'
-import { codeToHtml } from '../src'
+import { codeToHtml, codeToThemedTokens } from '../src'
 
 describe('should', () => {
-  it('works', async () => {
+  it('codeToHtml', async () => {
     expect(await codeToHtml('console.log("hello")', { lang: 'js', theme: 'vitesse-light' }))
       .toMatchInlineSnapshot('"<pre class=\\"shiki vitesse-light\\" style=\\"background-color: #ffffff\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span style=\\"color: #B07D48\\">console</span><span style=\\"color: #999999\\">.</span><span style=\\"color: #59873A\\">log</span><span style=\\"color: #999999\\">(</span><span style=\\"color: #B5695999\\">&quot;</span><span style=\\"color: #B56959\\">hello</span><span style=\\"color: #B5695999\\">&quot;</span><span style=\\"color: #999999\\">)</span></span></code></pre>"')
 
     expect(await codeToHtml('<div class="foo">bar</div>', { lang: 'html', theme: 'min-dark' }))
       .toMatchInlineSnapshot('"<pre class=\\"shiki min-dark\\" style=\\"background-color: #1f1f1f\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span style=\\"color: #B392F0\\">&lt;</span><span style=\\"color: #FFAB70\\">div</span><span style=\\"color: #B392F0\\"> class</span><span style=\\"color: #F97583\\">=</span><span style=\\"color: #FFAB70\\">&quot;foo&quot;</span><span style=\\"color: #B392F0\\">&gt;bar&lt;/</span><span style=\\"color: #FFAB70\\">div</span><span style=\\"color: #B392F0\\">&gt;</span></span></code></pre>"')
   })
+
+  it('codeToThemedTokens', async () => {
+    expect(await codeToThemedTokens('console.log("hello")', { lang: 'js', theme: 'vitesse-dark', includeExplanation: false }))
+      .toMatchInlineSnapshot(`
+        [
+          [
+            {
+              "color": "#BD976A",
+              "content": "console",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#666666",
+              "content": ".",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#80A665",
+              "content": "log",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#666666",
+              "content": "(",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#C98A7D99",
+              "content": "\\"",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#C98A7D",
+              "content": "hello",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#C98A7D99",
+              "content": "\\"",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+            {
+              "color": "#666666",
+              "content": ")",
+              "explanation": [],
+              "fontStyle": 0,
+            },
+          ],
+        ]
+      `)
+  })
 })