feat!: use hast, unify codeToHtmlThemes to codeToHtml (#9)

Author: antfu

README.md

@@ -39,7 +39,7 @@ const shiki = await getHighlighter({
 await shiki.loadTheme('vitesse-light')
 await shiki.loadLanguage('css')
 
-const code = shiki.codeToHtml('const a = 1', { lang: 'javascript' })
+const code = shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'vitesse-light' })
 ```
 
 Unlike `shiki`, `shikiji` does not load any themes or languages when not specified.
@@ -49,7 +49,7 @@ import { getHighlighter } from 'shikiji'
 
 const shiki = await getHighlighter()
 
-shiki.codeToHtml('const a = 1', { lang: 'javascript' }) // throws error, `javascript` is not loaded
+shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'nord' }) // throws error, `javascript` is not loaded
 
 await shiki.loadLanguage('javascript') // load the language
 ```
@@ -103,7 +103,7 @@ const shiki = await getHighlighterCore({
 // optionally, load themes and languages after creation
 await shiki.loadTheme(import('shikiji/themes/vitesse-light.mjs'))
 
-const code = shiki.codeToHtml('const a = 1', { lang: 'javascript' })
+const code = shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'nord' })
 ```
 
 ### CJS Usage
@@ -197,6 +197,15 @@ export default {
 
 ## Additional Features
 
+### `codeToHast`
+
+`shikiji` used [`hast`](https://github.com/syntax-tree/hast) to generate HTML. You can use `codeToHast` to generate the AST and use it with tools like [unified](https://github.com/unifiedjs).
+
+```js
+const root = shiki.codeToHast('const a = 1', { lang: 'javascript', theme: 'nord' })
+```
+
+
 ### Shorthands
 
 In addition to the `getHighlighter` function, `shikiji` also provides some shorthand functions for simpler usage.
@@ -212,7 +221,7 @@ Currently supports:
 
 - `codeToThemedTokens`
 - `codeToHtml`
-- `codeToHtmlThemes`
+- `codeToHast`
 
 Internally they maintain a singleton highlighter instance and load the theme/language on demand. Different from `shiki.codeToHtml`, the `codeToHtml` shorthand function returns a Promise and `lang` and `theme` options are required.
 
@@ -222,7 +231,7 @@ Internally they maintain a singleton highlighter instance and load the theme/lan
 
 `shikiji` added an experimental light/dark dual themes support. Different from [markdown-it-shiki](https://github.com/antfu/markdown-it-shiki#dark-mode)'s approach which renders the code twice, `shikiji`'s dual themes approach uses CSS variables to store the colors on each token. It's more performant with a smaller bundle size.
 
-Use `codeToHtmlThemes` to render the code with dual themes:
+Changing the `theme` option in `codeToHtml` to `options` with `light` and `dark` key to generate with two themes.
 
 ```js
 import { getHighlighter } from 'shikiji'
@@ -232,7 +241,7 @@ const shiki = await getHighlighter({
   langs: ['javascript'],
 })
 
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'vitesse-light',
@@ -293,7 +302,7 @@ html.dark .shiki span {
 It's also possible to support more than two themes. In the `themes` object, you can have an arbitrary number of themes, and specify the default theme with `defaultColor` option.
 
 ```js
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'github-light',
@@ -323,7 +332,7 @@ And then update your CSS snippet to control then each theme taking effect. Here
 If you want to take full control of the colors, or avoid using `!important` to override, you can optionally disable the default color by setting `defaultColor` to `false`.
 
 ```js
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'vitesse-light',
@@ -343,6 +352,17 @@ In that case, the generated HTML would have no style out of the box, you need to
 
 It's also possible to control the theme in CSS variables, for more, reference to the great research and examples by [@mayank99](https://github.com/mayank99) in [this issue #6](https://github.com/antfu/shikiji/issues/6).
 
+## Breaking Changes from Shiki
+
+As of [`shiki@0.4.3`](https://github.com/shikijs/shiki/releases/tag/v0.14.3):
+
+- Top level named export `setCDN`, `loadLanguage`, `loadLanguage`, `setWasm`, are dropped.
+- `BUNDLED_LANGUAGES`, `BUNDLED_THEMES` are moved to `shikiji/langs` and `shikiji/themes` and renamed to `bundledLanguages` and `bundledThemes` respectively.
+- `theme` option for `getHighlighter` is dropped, use `themes` with an array instead.
+- Highlighter does not maintain an internal default theme context. `theme` option is required for `codeToHtml` and `codeToThemedTokens`.
+- CJS and IIFE builds are dropped.
+- `LanguageRegistration`'s `grammar` field if flattened to `LanguageRegistration` itself (refer to the types for more details).
+
 ## Bundle Size
 
 You can inspect the bundle size in detail on [pkg-size.dev/shikiji](https://pkg-size.dev/shikiji).

package.json

@@ -20,13 +20,16 @@
     "@rollup/plugin-replace": "^5.0.2",
     "@rollup/plugin-terser": "^0.4.3",
     "@types/fs-extra": "^11.0.1",
+    "@types/hast": "^3.0.0",
     "@types/node": "^20.4.10",
     "@vitest/coverage-v8": "^0.34.1",
     "bumpp": "^9.1.1",
     "eslint": "^8.47.0",
     "esno": "^0.17.0",
     "fast-glob": "^3.3.1",
     "fs-extra": "^11.1.1",
+    "hast-util-to-html": "^9.0.0",
+    "hastscript": "^8.0.0",
     "jsonc-parser": "^3.2.0",
     "lint-staged": "^13.2.3",
     "pnpm": "^8.6.12",

packages/shikiji/README.md

@@ -209,7 +209,7 @@ Currently supports:
 
 - `codeToThemedTokens`
 - `codeToHtml`
-- `codeToHtmlThemes`
+- `codeToHast`
 
 Internally they maintain a singleton highlighter instance and load the theme/language on demand. Different from `shiki.codeToHtml`, the `codeToHtml` shorthand function returns a Promise and `lang` and `theme` options are required.
 
@@ -219,7 +219,7 @@ Internally they maintain a singleton highlighter instance and load the theme/lan
 
 `shikiji` added an experimental light/dark dual themes support. Different from [markdown-it-shiki](https://github.com/antfu/markdown-it-shiki#dark-mode)'s approach which renders the code twice, `shikiji`'s dual themes approach uses CSS variables to store the colors on each token. It's more performant with a smaller bundle size.
 
-Use `codeToHtmlThemes` to render the code with dual themes:
+Use `codeToHtml` to render the code with dual themes:
 
 ```js
 import { getHighlighter } from 'shikiji'
@@ -229,7 +229,7 @@ const shiki = await getHighlighter({
   langs: ['javascript'],
 })
 
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'vitesse-light',
@@ -290,7 +290,7 @@ html.dark .shiki span {
 It's also possible to support more than two themes. In the `themes` object, you can have an arbitrary number of themes, and specify the default theme with `defaultColor` option.
 
 ```js
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'github-light',
@@ -320,7 +320,7 @@ And then update your CSS snippet to control then each theme taking effect. Here
 If you want to take full control of the colors, or avoid using `!important` to override, you can optionally disable the default color by setting `defaultColor` to `false`.
 
 ```js
-const code = shiki.codeToHtmlThemes('console.log("hello")', {
+const code = shiki.codeToHtml('console.log("hello")', {
   lang: 'javascript',
   themes: {
     light: 'vitesse-light',

packages/shikiji/src/core/bundle-factory.ts

@@ -1,6 +1,6 @@
-import type { BundledHighlighterOptions, CodeToHtmlOptions, CodeToHtmlThemesOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, RequireKeys, ThemeInput } from '../types'
+import type { BundledHighlighterOptions, CodeToHastOptions, CodeToThemedTokensOptions, CodeToTokensWithThemesOptions, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, MaybeArray, PlainTextLanguage, RequireKeys, ThemeInput } from '../types'
 import { isPlaintext, toArray } from './utils'
-import { getHighlighterCore } from './core'
+import { getHighlighterCore } from './highlighter'
 
 export type GetHighlighterFactory<L extends string, T extends string> = (options?: BundledHighlighterOptions<L, T>) => Promise<HighlighterGeneric<L, T>>
 
@@ -53,9 +53,6 @@ export function createdBundledHighlighter<BundledLangs extends string, BundledTh
 
     return {
       ...core,
-      codeToHtml(code, options = {}) {
-        return core.codeToHtml(code, options)
-      },
       loadLanguage(...langs) {
         return core.loadLanguage(...langs.map(resolveLang))
       },
@@ -95,34 +92,37 @@ export function createSingletonShorthands<L extends string, T extends string >(g
    *
    * Differences from `shiki.codeToHtml()`, this function is async.
    */
-  async function codeToHtml(code: string, options: RequireKeys<CodeToHtmlOptions<L, T>, 'theme' | 'lang'>) {
-    const shiki = await _getHighlighter(options)
+  async function codeToHtml(code: string, options: CodeToHastOptions<L, T>) {
+    const shiki = await _getHighlighter({
+      lang: options.lang,
+      theme: 'theme' in options ? [options.theme] : Object.values(options.themes) as T[],
+    })
     return shiki.codeToHtml(code, options)
   }
 
   /**
-   * Shorthand for `codeToThemedTokens` with auto-loaded theme and language.
+   * Shorthand for `codeToHtml` with auto-loaded theme and language.
    * A singleton highlighter it maintained internally.
    *
-   * Differences from `shiki.codeToThemedTokens()`, this function is async.
+   * Differences from `shiki.codeToHtml()`, this function is async.
    */
-  async function codeToThemedTokens(code: string, options: RequireKeys<CodeToThemedTokensOptions<L, T>, 'theme' | 'lang'>) {
-    const shiki = await _getHighlighter(options)
-    return shiki.codeToThemedTokens(code, options)
+  async function codeToHast(code: string, options: CodeToHastOptions<L, T>) {
+    const shiki = await _getHighlighter({
+      lang: options.lang,
+      theme: 'theme' in options ? [options.theme] : Object.values(options.themes) as T[],
+    })
+    return shiki.codeToHast(code, options)
   }
 
   /**
-   * Shorthand for `codeToHtmlThemes` with auto-loaded theme and language.
+   * Shorthand for `codeToThemedTokens` with auto-loaded theme and language.
    * A singleton highlighter it maintained internally.
    *
-   * Differences from `shiki.codeToHtmlThemes()`, this function is async.
+   * Differences from `shiki.codeToThemedTokens()`, this function is async.
    */
-  async function codeToHtmlThemes(code: string, options: RequireKeys<CodeToHtmlThemesOptions<L, T>, 'themes' | 'lang'>) {
-    const shiki = await _getHighlighter({
-      lang: options.lang,
-      theme: Object.values(options.themes).filter(Boolean) as T[],
-    })
-    return shiki.codeToHtmlThemes(code, options)
+  async function codeToThemedTokens(code: string, options: RequireKeys<CodeToThemedTokensOptions<L, T>, 'theme' | 'lang'>) {
+    const shiki = await _getHighlighter(options)
+    return shiki.codeToThemedTokens(code, options)
   }
 
   /**
@@ -141,7 +141,7 @@ export function createSingletonShorthands<L extends string, T extends string >(g
 
   return {
     codeToHtml,
-    codeToHtmlThemes,
+    codeToHast,
     codeToThemedTokens,
     codeToTokensWithThemes,
   }

packages/shikiji/src/core/context.ts

@@ -0,0 +1,95 @@
+import type { HighlighterCoreOptions, LanguageInput, MaybeGetter, ShikiContext, ThemeInput } from '../types'
+import { createOnigScanner, createOnigString, loadWasm } from '../oniguruma'
+import { Registry } from './registry'
+import { Resolver } from './resolver'
+
+/**
+* Get the minimal shiki context for rendering.
+*/
+export async function getShikiContext(options: HighlighterCoreOptions = {}): Promise<ShikiContext> {
+  async function normalizeGetter<T>(p: MaybeGetter<T>): Promise<T> {
+    return Promise.resolve(typeof p === 'function' ? (p as any)() : p).then(r => r.default || r)
+  }
+
+  async function resolveLangs(langs: LanguageInput[]) {
+    return Array.from(new Set((await Promise.all(
+      langs.map(async lang => await normalizeGetter(lang).then(r => Array.isArray(r) ? r : [r])),
+    )).flat()))
+  }
+
+  const [
+    themes, langs,
+  ] = await Promise.all([
+    Promise.all((options.themes || []).map(normalizeGetter)),
+    resolveLangs(options.langs || []),
+    typeof options.loadWasm === 'function'
+      ? Promise.resolve(options.loadWasm()).then(r => loadWasm(r))
+      : options.loadWasm
+        ? loadWasm(options.loadWasm)
+        : undefined,
+  ] as const)
+
+  const resolver = new Resolver(Promise.resolve({
+    createOnigScanner(patterns) {
+      return createOnigScanner(patterns)
+    },
+    createOnigString(s) {
+      return createOnigString(s)
+    },
+  }), 'vscode-oniguruma', langs)
+
+  const _registry = new Registry(resolver, themes, langs)
+  await _registry.init()
+
+  function getLangGrammar(name: string) {
+    const _lang = _registry.getGrammar(name)
+    if (!_lang)
+      throw new Error(`[shikiji] Language \`${name}\` not found, you may need to load it first`)
+    return _lang
+  }
+
+  function getTheme(name: string) {
+    const _theme = _registry.getTheme(name!)
+    if (!_theme)
+      throw new Error(`[shikiji] Theme \`${name}\` not found, you may need to load it first`)
+    return _theme
+  }
+
+  function setTheme(name: string) {
+    const theme = getTheme(name)
+    _registry.setTheme(theme)
+    const colorMap = _registry.getColorMap()
+    return {
+      theme,
+      colorMap,
+    }
+  }
+
+  function getLoadedThemes() {
+    return _registry.getLoadedThemes()
+  }
+
+  function getLoadedLanguages() {
+    return _registry.getLoadedLanguages()
+  }
+
+  async function loadLanguage(...langs: LanguageInput[]) {
+    await _registry.loadLanguages(await resolveLangs(langs))
+  }
+
+  async function loadTheme(...themes: ThemeInput[]) {
+    await Promise.all(
+      themes.map(async theme => _registry.loadTheme(await normalizeGetter(theme))),
+    )
+  }
+
+  return {
+    setTheme,
+    getTheme,
+    getLangGrammar,
+    getLoadedThemes,
+    getLoadedLanguages,
+    loadLanguage,
+    loadTheme,
+  }
+}