feat(html): add mergeWhitespaces option, thanks to @dominikg

Author: antfu

packages/shikiji/src/core/renderer-html.ts

@@ -17,6 +17,13 @@ const defaultElements: ElementsOptions = {
 }
 
 export function renderToHtml(lines: ThemedToken[][], options: HtmlRendererOptions = {}) {
+  const {
+    mergeWhitespaces = true,
+  } = options
+
+  if (mergeWhitespaces)
+    lines = mergeWhitespaceTokens(lines)
+
   const bg = options.bg || '#fff'
   const fg = options.bg || '#000'
   const optionsByLineNumber = groupBy(options.lineOptions ?? [], option => option.line)
@@ -127,3 +134,28 @@ export function groupBy<TElement, TKey>(
   }
   return map
 }
+
+function mergeWhitespaceTokens(tokens: ThemedToken[][]) {
+  return tokens.map((line) => {
+    const newLine: ThemedToken[] = []
+    let carryOnContent = ''
+    line.forEach((token, idx) => {
+      if (token.content.match(/^\s+$/) && line[idx + 1]) {
+        carryOnContent += token.content
+      }
+      else {
+        if (carryOnContent) {
+          newLine.push({
+            ...token,
+            content: carryOnContent + token.content,
+          })
+          carryOnContent = ''
+        }
+        else {
+          newLine.push(token)
+        }
+      }
+    })
+    return newLine
+  })
+}

packages/shikiji/src/core/themedTokenizer.ts

@@ -9,6 +9,10 @@ import type { ThemedToken, ThemedTokenExplanation, ThemedTokenScopeExplanation }
 import type { FontStyle } from './stackElementMetadata'
 import { StackElementMetadata } from './stackElementMetadata'
 
+export interface TokenizeWithThemeOptions {
+  includeExplanation?: boolean
+}
+
 export function tokenizeWithTheme(
   fileContents: string,
   grammar: IGrammar,

packages/shikiji/src/types.ts

@@ -175,6 +175,13 @@ export interface HtmlRendererOptions {
    * When specified, `fg` and `bg` will be ignored.
    */
   rootStyle?: string
+  /**
+   * Merge token with only whitespace to the next token,
+   * Saving a few extra `<span>`
+   *
+   * @default true
+   */
+  mergeWhitespaces?: boolean
 }
 
 export interface ElementsOptions {

packages/shikiji/test/core.test.ts

@@ -55,7 +55,7 @@ describe('should', () => {
       `)
 
     expect(shiki.codeToHtml('print 1', { lang: 'python', theme: 'vitesse-light' }))
-      .toMatchInlineSnapshot('"<pre class=\\"shiki vitesse-light\\" style=\\"background-color:#ffffff;color:#ffffff\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span style=\\"color:#998418\\">print</span><span style=\\"color:#393A34\\"> </span><span style=\\"color:#2F798A\\">1</span></span></code></pre>"')
+      .toMatchInlineSnapshot('"<pre class=\\"shiki vitesse-light\\" style=\\"background-color:#ffffff;color:#ffffff\\" tabindex=\\"0\\"><code><span class=\\"line\\"><span style=\\"color:#998418\\">print</span><span style=\\"color:#2F798A\\"> 1</span></span></code></pre>"')
   })
 
   it('requires nested lang', async () => {