feat: introduce web bundle (#56)

Author: antfu

docs/.vitepress/components/LanguagesList.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { ref } from 'vue'
+import { computed, ref } from 'vue'
 import { usePlayground } from '../store/playground'
 
 const play = usePlayground()
@@ -9,10 +9,27 @@ function preview(id: string) {
   play.lang = id
   showModel.value = true
 }
+
+const bundle = ref('all')
+
+const langs = computed(() => {
+  if (bundle.value === 'web')
+    return play.bundledLangsWeb
+  return play.bundledLangsFull
+})
 </script>
 
 <template>
   <div>
+    <div flex="~ gap-0.5 items-center">
+      <input id="radio-all" v-model="bundle" type="radio" name="lang" value="all">
+      <label for="radio-all">Full Bundle</label>
+      <div mx2 />
+      <input id="radio-web" v-model="bundle" type="radio" name="lang" value="web">
+      <label for="radio-web">Web Bundle</label>
+      <div mx2 />
+      <a href="/guide/bundles">?</a>
+    </div>
     <table>
       <thead>
         <tr>
@@ -23,7 +40,7 @@ function preview(id: string) {
         </tr>
       </thead>
       <tbody>
-        <tr v-for="l in play.allLanguages" :key="l.id">
+        <tr v-for="l in langs" :key="l.id">
           <td>{{ l.name }}</td>
           <td><code>{{ l.id }}</code></td>
           <td>

docs/.vitepress/config.ts

@@ -9,6 +9,7 @@ const GUIDES: DefaultTheme.NavItemWithLink[] = [
   { text: 'Getting Started', link: '/guide/' },
   { text: 'Installation', link: '/guide/install' },
   { text: 'Shorthands', link: '/guide/shorthands' },
+  { text: 'Bundles', link: '/guide/bundles' },
   { text: 'Dual Themes', link: '/guide/dual-themes' },
   { text: 'Transformers', link: '/guide/transformers' },
   { text: 'Compat Build', link: '/guide/compat' },

docs/.vitepress/store/playground.ts

@@ -23,6 +23,9 @@ export const usePlayground = defineStore('playground', () => {
       import: undefined!,
     },
   ])
+  const bundledLangsFull = shallowRef<BundledLanguageInfo[]>([])
+  const bundledLangsWeb = shallowRef<BundledLanguageInfo[]>([])
+
   const input = useLocalStorage('shikiji-playground-input', '')
   const output = ref('<pre></pre>')
   const preStyle = ref('')
@@ -38,7 +41,8 @@ export const usePlayground = defineStore('playground', () => {
   if (typeof window !== 'undefined') {
     (async () => {
       const { getHighlighter, addClassToHast } = await import('shikiji')
-      const { bundledLanguagesInfo } = await import('shikiji/langs')
+      const { bundledLanguagesInfo: bundleFull } = await import('shikiji/bundle/full')
+      const { bundledLanguagesInfo: bundleWeb } = await import('shikiji/bundle/web')
       const { bundledThemesInfo } = await import('shikiji/themes')
       const highlighter = await getHighlighter({
         themes: [theme.value],
@@ -60,7 +64,9 @@ export const usePlayground = defineStore('playground', () => {
       }
 
       allThemes.value = bundledThemesInfo
-      allLanguages.value = bundledLanguagesInfo
+      allLanguages.value = bundleFull
+      bundledLangsFull.value = bundleFull
+      bundledLangsWeb.value = bundleWeb
 
       watch(input, run, { immediate: true })
 
@@ -102,6 +108,8 @@ export const usePlayground = defineStore('playground', () => {
     theme,
     allLanguages,
     allThemes,
+    bundledLangsFull,
+    bundledLangsWeb,
     input,
     output,
     isLoading,

docs/guide/bundles.md

@@ -0,0 +1,33 @@
+---
+outline: deep
+---
+
+# Bundles
+
+The main `shikiji` entries bundles all supported themes and languages via lazy dynamic imports. The efficiency shouldn't be a concern to most of the scenarios as the grammar would only be imported/downloaded when it is used. However, when you bundle Shikiji into browsers runtime or web workers, even those files are not imported, they still add up to your dist size. We provide the [fine-grained bundle](/guide/install#fine-grained-bundle) to help you compose languages and themes one-by-one as you need.
+
+To make it easier, we also provide some pre-composed bundles for you to use:
+
+## `shikiji/bundle/full`
+
+The full bundle includes all themes and languages, same as the main `shikiji` entry.
+
+## `shikiji/bundle/web`
+
+The bundle the includes all themes and common web languages like (HTML, CSS, JS, TS, JSON, Markdown, etc.) and some web frameworks (Vue, JSX, Svelte, etc.).
+
+Use as normal, all functions from `shikiji` are also available in the bundle:
+
+```ts
+import {
+  BundledLanguage,
+  BundledTheme,
+  codeToHtml,
+  getHighlighter
+} from 'shikiji/bundle/web' // [!code highlight]
+
+const highlighter = await getHighlighter({
+  langs: ['html', 'css', 'js'],
+  themes: ['github-dark', 'github-light'],
+})
+```

docs/guide/install.md

@@ -126,6 +126,10 @@ const code = shiki.codeToHtml('const a = 1', {
 })
 ```
 
+### Bundle Presets
+
+We also provide some pre-composed bundles for you to use easily, learn more about them in the [bundles section](/guide/bundles).
+
 ### CJS Usage
 
 `shikiji` is published as ESM-only to reduce the package size. It's still possible to use it in CJS, as Node.js supports importing ESM modules dynamically in CJS.
@@ -138,7 +142,7 @@ import { getHighlighter } from 'shikiji'
 
 async function main() {
   const shiki = await getHighlighter({
-    themes: ['nord'],
+    themes: ['vitesse-dark'],
     langs: ['javascript'],
   })
 
@@ -154,7 +158,7 @@ async function main() {
   const { getHighlighter } = await import('shikiji')
 
   const shiki = await getHighlighter({
-    themes: ['nord'],
+    themes: ['vitesse-dark'],
     langs: ['javascript'],
   })
 
@@ -166,7 +170,7 @@ async function main() {
 
 To use `shikiji` in the browser via CDN, you can use [esm.run](https://esm.run) or [esm.sh](https://esm.sh).
 
-```html
+```html theme:rose-pine
 <body>
   <div id="foo"></div>
 
@@ -177,7 +181,10 @@ To use `shikiji` in the browser via CDN, you can use [esm.run](https://esm.run)
     // import { codeToHtml } from 'https://esm.run/shikiji@0.8.0'
 
     const foo = document.getElementById('foo')
-    foo.innerHTML = await codeToHtml('console.log("Hi, Shiki on CDN :)")', { lang: 'js', theme: 'vitesse-light' })
+    foo.innerHTML = await codeToHtml('console.log("Hi, Shiki on CDN :)")', {
+      lang: 'js',
+      theme: 'rose-pine'
+    })
   </script>
 </body>
 ```
@@ -192,7 +199,7 @@ Cloudflare Workers [does not support initializing WebAssembly from binary data](
 
 Meanwhile, it's also recommended to use the [Fine-grained Bundle](#fine-grained-bundle) approach to reduce the bundle size.
 
-```ts
+```ts theme:nord
 import { getHighlighterCore, loadWasm } from 'shikiji/core'
 import nord from 'shikiji/themes/nord.mjs'
 import js from 'shikiji/langs/javascript.mjs'

docs/package.json

@@ -21,6 +21,6 @@
     "unocss": "^0.58.0",
     "unplugin-vue-components": "^0.26.0",
     "vitepress": "1.0.0-rc.30",
-    "vue": "^3.3.11"
+    "vue": "^3.3.13"
   }
 }

package.json

@@ -24,8 +24,8 @@
     "@rollup/plugin-terser": "^0.4.4",
     "@types/fs-extra": "^11.0.4",
     "@types/hast": "^3.0.3",
-    "@types/node": "^20.10.4",
-    "@vitest/coverage-v8": "^1.0.4",
+    "@types/node": "^20.10.5",
+    "@vitest/coverage-v8": "^1.1.0",
     "ansi-sequence-parser": "^1.1.1",
     "bumpp": "^9.2.1",
     "eslint": "npm:eslint-ts-patch@8.55.0-1",
@@ -42,7 +42,7 @@
     "pnpm": "^8.12.1",
     "prettier": "^3.1.1",
     "rimraf": "^5.0.5",
-    "rollup": "^4.9.0",
+    "rollup": "^4.9.1",
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-dts": "^6.1.0",
     "rollup-plugin-esbuild": "^6.1.0",
@@ -53,9 +53,9 @@
     "typescript": "^5.3.3",
     "unbuild": "^2.0.0",
     "vite": "^5.0.10",
-    "vitest": "^1.0.4",
+    "vitest": "^1.1.0",
     "vue-tsc": "^1.8.25",
-    "wrangler": "^3.21.0"
+    "wrangler": "^3.22.1"
   },
   "pnpm": {
     "patchedDependencies": {

packages/shikiji-core/src/types.ts

@@ -578,4 +578,21 @@ export interface ThemedTokenWithVariants extends TokenBase {
   variants: Record<string, TokenStyles>
 }
 
+export type DynamicImportLanguageRegistration = () => Promise<{ default: LanguageRegistration[] }>
+export type DynamicImportThemeRegistration = () => Promise<{ default: ThemeRegistration }>
+
+export interface BundledLanguageInfo {
+  id: string
+  name: string
+  import: DynamicImportLanguageRegistration
+  aliases?: string[]
+}
+
+export interface BundledThemeInfo {
+  id: string
+  displayName: string
+  type: 'light' | 'dark'
+  import: DynamicImportThemeRegistration
+}
+
 export {}

packages/shikiji-twoslash/package.json

@@ -59,7 +59,7 @@
     "shikiji-core": "workspace:*"
   },
   "devDependencies": {
-    "@iconify-json/carbon": "^1.1.26",
+    "@iconify-json/carbon": "^1.1.27",
     "@iconify-json/codicon": "^1.1.39",
     "hast-util-from-html": "^2.0.1",
     "shiki": "^0.14.7",

packages/shikiji/package.json

@@ -37,6 +37,14 @@
       "types": "./dist/themes.d.mts",
       "default": "./dist/themes.mjs"
     },
+    "./bundle/full": {
+      "types": "./dist/bundle-full.d.mts",
+      "default": "./dist/bundle-full.mjs"
+    },
+    "./bundle/web": {
+      "types": "./dist/bundle-web.d.mts",
+      "default": "./dist/bundle-web.mjs"
+    },
     "./dist/*": "./dist/*",
     "./*": "./dist/*"
   },
@@ -57,6 +65,12 @@
       "themes": [
         "./dist/themes.d.mts"
       ],
+      "bundle/full": [
+        "./dist/bundle-full.d.mts"
+      ],
+      "bundle/web": [
+        "./dist/bundle-web.d.mts"
+      ],
       "*": [
         "./dist/*",
         "./*"
@@ -78,8 +92,8 @@
     "shikiji-core": "workspace:*"
   },
   "devDependencies": {
-    "tm-grammars": "^0.0.6",
-    "tm-themes": "^0.0.3",
+    "tm-grammars": "^1.0.2",
+    "tm-themes": "^1.0.0",
     "vscode-oniguruma": "^1.7.0"
   }
 }

packages/shikiji/rollup.config.mjs

@@ -17,6 +17,8 @@ const entries = [
   'src/themes.ts',
   'src/langs.ts',
   'src/wasm.ts',
+  'src/bundle-full.ts',
+  'src/bundle-web.ts',
 ]
 
 const external = [

packages/shikiji/scripts/prepare/langs.ts

@@ -22,12 +22,12 @@ export async function prepareLangs() {
     }
 
     const json: LanguageRegistration = {
-      ...lang,
       ...content,
       name: content.name || lang.name,
       scopeName: content.scopeName || lang.scopeName,
       displayName: lang.displayName,
       embeddedLangs: lang.embedded,
+      aliases: lang.aliases,
     }
 
     // F# and Markdown has circular dependency
@@ -43,7 +43,7 @@ import type { LanguageRegistration } from 'shikiji-core'
 
 ${deps.map(i => `import ${i.replace(/[^\w]/g, '_')} from './${i}'`).join('\n')}
 
-const lang = Object.freeze(${JSON.stringify(json, null, 2)}) as unknown as LanguageRegistration
+const lang = Object.freeze(${JSON.stringify(json)}) as unknown as LanguageRegistration
 
 export default [
 ${[
@@ -54,14 +54,18 @@ ${[
 `, 'utf-8')
   }
 
-  async function writeLanguageBundleIndex(fileName: string, ids: string[]) {
-    const bundled = ids.map(id => grammars.find(i => i.name === id)!)
+  async function writeLanguageBundleIndex(
+    fileName: string,
+    ids: string[],
+    exclude: string[] = [],
+  ) {
+    const bundled = ids.map(id => grammars.find(i => i.name === id)!).filter(i => !exclude.includes(i.name))
 
     const info = bundled.map(i => ({
       id: i.name,
       name: i.displayName || i.name,
       aliases: i.aliases,
-      import: `__(() => import('./langs/${i.name}')) as DynamicLangReg__`,
+      import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
     }) as const)
       .sort((a, b) => a.id.localeCompare(b.id))
 
@@ -70,33 +74,37 @@ ${[
     await fs.writeFile(
     `src/assets/${fileName}.ts`,
   `${COMMENT_HEAD}
-import type { LanguageRegistration } from 'shikiji-core'
-
-type DynamicLangReg = () => Promise<{ default: LanguageRegistration[] }>
-
-export interface BundledLanguageInfo {
-  id: string
-  name: string
-  import: DynamicLangReg
-  aliases?: string[]
-}
+import type { DynamicImportLanguageRegistration, BundledLanguageInfo } from 'shikiji-core'
 
 export const bundledLanguagesInfo: BundledLanguageInfo[] = ${JSON.stringify(info, null, 2).replace(/"__|__"/g, '').replace(/"/g, '\'')}
 
 export const bundledLanguagesBase = Object.fromEntries(bundledLanguagesInfo.map(i => [i.id, i.import]))
 
 export const bundledLanguagesAlias = Object.fromEntries(bundledLanguagesInfo.flatMap(i => i.aliases?.map(a => [a, i.import]) || []))
 
-export type BuiltinLanguage = ${type}
+export type BundledLanguage = ${type}
 
 export const bundledLanguages = {
   ...bundledLanguagesBase,
   ...bundledLanguagesAlias,
-} as Record<BuiltinLanguage, DynamicLangReg>
+} as Record<BundledLanguage, DynamicImportLanguageRegistration>
 `,
   'utf-8',
     )
   }
 
-  await writeLanguageBundleIndex('langs', grammars.map(i => i.name))
+  await writeLanguageBundleIndex(
+    'langs-bundle-full',
+    grammars.map(i => i.name),
+  )
+  await writeLanguageBundleIndex(
+    'langs-bundle-web',
+    [
+      ...grammars.filter(i => i.categories?.includes('web')).map(i => i.name),
+      'shellscript',
+    ],
+    [
+      'coffee',
+    ],
+  )
 }