feat: support mergeWhitespaces: never for themes are not handling perfectly

Author: antfu

packages/shikiji-core/src/renderer-hast.ts

@@ -157,8 +157,10 @@ export function tokensToHast(
     console.warn('[shikiji] `transforms` option is deprecated, use `transformers` instead')
   }
 
-  if (mergeWhitespaces)
+  if (mergeWhitespaces === true)
     tokens = mergeWhitespaceTokens(tokens)
+  else if (mergeWhitespaces === 'never')
+    tokens = splitWhitespaceTokens(tokens)
 
   const lines: (Element | Text)[] = []
   const tree: Root = {
@@ -325,3 +327,28 @@ function mergeWhitespaceTokens(tokens: ThemedToken[][]) {
     return newLine
   })
 }
+
+function splitWhitespaceTokens(tokens: ThemedToken[][]) {
+  return tokens.map((line) => {
+    return line.flatMap((token) => {
+      if (token.content.match(/^\s+$/))
+        return token
+      const match = token.content.match(/^(\s*)(.*?)(\s*)$/)
+      if (!match)
+        return token
+      const [, leading, content, trailing] = match
+      if (!leading && !trailing)
+        return token
+
+      const expanded = [{
+        ...token,
+        content,
+      }]
+      if (leading)
+        expanded.unshift({ content: leading })
+      if (trailing)
+        expanded.push({ content: trailing })
+      return expanded
+    })
+  })
+}

packages/shikiji-core/src/types.ts

@@ -227,12 +227,15 @@ export interface CodeToHastOptionsCommon<Languages extends string = string> exte
   lang: StringLiteralUnion<Languages | SpecialLanguage>
 
   /**
-   * Merge token with only whitespace to the next token,
-   * Saving a few extra `<span>`
+   * Merge whitespace tokens to saving extra `<span>`.
+   *
+   * When set to true, it will merge whitespace tokens with the next token.
+   * When set to false, it keep the output as-is.
+   * When set to `never`, it will force to separate leading and trailing spaces from tokens.
    *
    * @default true
    */
-  mergeWhitespaces?: boolean
+  mergeWhitespaces?: boolean | 'never'
 }
 
 export interface CodeToTokensWithThemesOptions<Languages = string, Themes = string> {

packages/shikiji-transformers/src/transformers/_utils.ts

@@ -35,7 +35,7 @@ export function isSpace(part: string) {
   return part === ' ' || part === '\t'
 }
 
-export function mergeSpaces(
+export function splitSpaces(
   parts: string[],
   type: 'all' | 'boundary' | 'trailing',
   renderContinuousSpaces = true,

packages/shikiji-transformers/src/transformers/render-whitespace.ts

@@ -1,7 +1,7 @@
 import type { ShikijiTransformer } from 'shikiji'
 import { addClassToHast } from 'shikiji'
 import type { Element } from 'hast'
-import { mergeSpaces } from './_utils'
+import { splitSpaces } from './_utils'
 
 export interface TransformerRenderWhitespaceOptions {
   /**
@@ -64,7 +64,7 @@ export function transformerRenderWhitespace(
             return token
 
           // Split by whitespaces
-          const parts = mergeSpaces(
+          const parts = splitSpaces(
             node.value.split(/([ \t])/).filter(i => i.length),
             (position === 'boundary' && index === last && last !== 0)
               ? 'trailing'

packages/shikiji-twoslash/src/core.ts

@@ -49,7 +49,7 @@ export function createTransformerFactory(defaultTwoslasher: typeof twoslasher) {
           lang = langAlias[shikijiOptions.lang]
 
         if (filter(lang, code, shikijiOptions)) {
-          shikijiOptions.mergeWhitespaces = false
+          shikijiOptions.mergeWhitespaces = 'never'
           const twoslash = twoslasher(code, lang, twoslashOptions)
           this.meta.twoslash = twoslash
           return twoslash.code