fix(compat): support ansiToThemedTokens

Author: antfu

packages/shikiji-compat/src/index.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import type { BuiltinLanguage, BuiltinTheme, CodeToThemedTokensOptions, MaybeGetter, StringLiteralUnion, ThemeInput, ThemeRegistration, ThemedToken } from 'shikiji'
-import { bundledLanguages, bundledThemes, getHighlighter as getShikiji, toShikiTheme } from 'shikiji'
+import { bundledLanguages, bundledThemes, getHighlighter as getShikiji, toShikiTheme, tokenizeAnsiWithTheme } from 'shikiji'
 import { transformerCompactLineOptions } from 'shikiji-transformers'
 import type { AnsiToHtmlOptions, CodeToHtmlOptions, CodeToHtmlOptionsExtra, HighlighterOptions } from './types'
 
@@ -73,8 +73,17 @@ export async function getHighlighter(options: HighlighterOptions = {}) {
     return shikiji.codeToHtml(code, options as any)
   }
 
+  function ansiToThemedTokens(
+    ansi: string,
+    options: CodeToThemedTokensOptions = {},
+  ) {
+    const theme = shikiji.getTheme(options.theme || context.getLoadedThemes()[0])
+    return tokenizeAnsiWithTheme(theme, ansi)
+  }
+
   return {
     ...shikiji,
+    ansiToThemedTokens,
     codeToThemedTokens,
     codeToHtml,
     ansiToHtml(code: string, options?: AnsiToHtmlOptions) {

packages/shikiji-compat/test/types.test.ts

@@ -36,7 +36,6 @@ it('run', async () => {
 
   expect.soft(keysDiff).toMatchInlineSnapshot(`
     [
-      "ansiToThemedTokens",
       "getBackgroundColor",
       "getForegroundColor",
       "setColorReplacements",

packages/shikiji-transformers/src/utils.ts

@@ -1,23 +1,6 @@
 import type { Element } from 'hast'
 import type { ShikijiTransformer, ShikijiTransformerContext } from 'shikiji'
 
-/**
- * Check if a node is comment-like,
- * e.g. `<!-- comment -->`, `/* comment ..`, `// comment`
- */
-export function isCommentLike(node: Element, line: Element) {
-  if (node.children?.[0].type !== 'text')
-    return false
-  const text = node.children[0].value.trim()
-  if (text.startsWith('<!--') && text.endsWith('-->'))
-    return true
-  if (text.startsWith('/*') && text.endsWith('*/'))
-    return true
-  if (text.startsWith('//') && line.children.indexOf(node) === line.children.length - 1)
-    return true
-  return false
-}
-
 export function createCommentNotationTransformer(
   name: string,
   regex: RegExp,

packages/shikiji/src/core/index.ts

@@ -5,4 +5,6 @@ export * from './highlighter'
 export * from './bundle-factory'
 export * from './utils'
 
+export { codeToThemedTokens } from './tokenizer'
+export { tokenizeAnsiWithTheme } from './tokenizer-ansi'
 export { toShikiTheme } from './normalize'

packages/shikiji/src/core/renderer-ansi.ts

@@ -26,7 +26,7 @@ export default defineConfig({
     coverage: {
       provider: 'v8',
       include: [
-        '**/packages/*/src/*.ts',
+        '**/packages/*/src/**/*.ts',
       ],
       exclude: [
         '**/src/oniguruma/**',