Skip to content

Commit d7f66fd

Browse files
antfuantfu
committed
refactor!: flat lang registration
1 parent 4854010 commit d7f66fd

16 files changed

+77
-72
lines changed

‎package.json

+12
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,18 @@
2525
"types": "./dist/core.d.mts",
2626
"import": "./dist/core.mjs"
2727
},
28+
"./wasm": {
29+
"types": "./dist/wasm.d.mts",
30+
"import": "./dist/wasm.mjs"
31+
},
32+
"./languages": {
33+
"types": "./dist/languages.d.mts",
34+
"import": "./dist/languages.mjs"
35+
},
36+
"./themes": {
37+
"types": "./dist/themes.d.mts",
38+
"import": "./dist/themes.mjs"
39+
},
2840
"./dist/*": "./dist/*",
2941
"./*": "./dist/*"
3042
},

‎rollup.config.mjs

+11-11
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ import fs from 'fs-extra'
1111
import fg from 'fast-glob'
1212
import { wasmPlugin } from './build/wasm.mjs'
1313

14+
const entries = [
15+
'src/index.ts',
16+
'src/core.ts',
17+
'src/types.ts',
18+
'src/themes.ts',
19+
'src/languages.ts',
20+
'src/wasm.ts',
21+
]
22+
1423
const plugins = [
1524
esbuild(),
1625
nodeResolve(),
@@ -25,11 +34,7 @@ const plugins = [
2534

2635
export default defineConfig([
2736
{
28-
input: [
29-
'src/index.ts',
30-
'src/core.ts',
31-
'src/wasm.ts',
32-
],
37+
input: entries,
3338
output: {
3439
dir: 'dist',
3540
format: 'esm',
@@ -49,12 +54,7 @@ export default defineConfig([
4954
],
5055
},
5156
{
52-
input: [
53-
'src/core.ts',
54-
'src/index.ts',
55-
'src/wasm.ts',
56-
'src/types.ts',
57-
],
57+
input: entries,
5858
output: {
5959
dir: 'dist',
6060
format: 'esm',

‎scripts/prepare.ts

+11-8
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@ for (const file of files) {
2020
}
2121

2222
await fs.writeJSON(`./src/vendor/languages/${lang.id}.json`, {
23-
...lang,
24-
samplePath: undefined,
25-
path: undefined,
26-
displayName: undefined,
27-
grammar: content,
28-
})
23+
...content,
24+
name: content.name || lang.id,
25+
scopeName: content.scopeName || lang.scopeName,
26+
displayName: lang.displayName,
27+
aliases: lang.aliases,
28+
embeddedLangs: lang.embeddedLangs,
29+
balancedBracketSelectors: lang.balancedBracketSelectors,
30+
unbalancedBracketSelectors: lang.unbalancedBracketSelectors,
31+
}, { spaces: 2 })
2932
}
3033

3134
const languages = Object.fromEntries(BUNDLED_LANGUAGES.map(i => [i.id, `__()=>import('./languages/${i.id}.json').then(r=>r.default as unknown as LanguageRegistration)__`]))
@@ -36,14 +39,14 @@ await fs.writeFile(
3639
'src/vendor/langs.ts',
3740
`import type { LanguageRegistration } from '../types'
3841
39-
export const languages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
42+
export const bundledLanguages = ${JSON.stringify(languages, null, 2).replace(/"__|__"/g, '')}`,
4043
'utf-8',
4144
)
4245

4346
await fs.writeFile(
4447
'src/vendor/themes.ts',
4548
`import type { ThemeRegisterationRaw } from '../types'
4649
47-
export const themes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
50+
export const bundledThemes = ${JSON.stringify(themes, null, 2).replace(/"__|__"/g, '')}`,
4851
'utf-8',
4952
)

‎src/core.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { IOptions } from 'vscode-oniguruma'
1+
import type { OnigurumaLoadOptions } from './oniguruma'
22
import { createOnigScanner, createOnigString, loadWasm } from './oniguruma'
33
import { Registry } from './registry'
44
import type { CodeToHtmlOptions, LanguageInput, ThemeInput } from './types'
@@ -9,17 +9,17 @@ import { toShikiTheme } from './normalize'
99

1010
export * from './types'
1111

12-
export interface CoreHighlighterOptions {
12+
export interface HighlighterCoreOptions {
1313
themes: ThemeInput[]
1414
langs: LanguageInput[]
15-
loadWasm?: IOptions | (() => Promise<IOptions>)
15+
loadWasm?: OnigurumaLoadOptions | (() => Promise<OnigurumaLoadOptions>)
1616
}
1717

1818
export {
1919
loadWasm,
2020
}
2121

22-
export async function getHighlighter(options: CoreHighlighterOptions) {
22+
export async function getHighlighterCore(options: HighlighterCoreOptions) {
2323
const [
2424
themes,
2525
langs,

‎src/index.ts

+10-10
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
import type { BuiltinLanguages, BuiltinThemes, LanguageInput, ThemeInput } from './types'
2-
import { themes } from './vendor/themes'
3-
import { languages } from './vendor/langs'
4-
import { getHighlighter as getCoreHighlighter } from './core'
5-
import { loadWasm } from './oniguruma'
2+
import { bundledThemes } from './vendor/themes'
3+
import { bundledLanguages } from './vendor/langs'
4+
import { getHighlighterCore } from './core'
65
import { getWasmInlined } from './wasm'
76

8-
export * from './types'
7+
export * from './core'
98

109
export {
11-
loadWasm,
1210
getWasmInlined,
11+
bundledLanguages,
12+
bundledThemes,
1313
}
1414

1515
export interface HighlighterOptions {
@@ -20,18 +20,18 @@ export interface HighlighterOptions {
2020
export async function getHighlighter(options: HighlighterOptions = {}) {
2121
const _themes = (options.themes ?? ['nord']).map((i) => {
2222
if (typeof i === 'string')
23-
return themes[i]
23+
return bundledThemes[i]
2424
return i
2525
}) as ThemeInput[]
2626

27-
const langs = (options.langs ?? Object.keys(languages) as BuiltinLanguages[])
27+
const langs = (options.langs ?? Object.keys(bundledLanguages) as BuiltinLanguages[])
2828
.map((i) => {
2929
if (typeof i === 'string')
30-
return languages[i]
30+
return bundledLanguages[i]
3131
return i
3232
})
3333

34-
return getCoreHighlighter({
34+
return getHighlighterCore({
3535
...options,
3636
themes: _themes,
3737
langs,

‎src/languages.ts

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from './vendor/langs'

‎src/oniguruma/index.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ interface IInstantiatorOptions extends ICommonOptions {
361361
interface IDataOptions extends ICommonOptions {
362362
data: ArrayBufferView | ArrayBuffer | Response
363363
}
364-
export type IOptions = IInstantiatorOptions | IDataOptions
364+
export type OnigurumaLoadOptions = IInstantiatorOptions | IDataOptions
365365

366366
async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string) => void) | undefined): Promise<void> {
367367
onigBinding = await OnigasmModuleFactory({
@@ -379,25 +379,25 @@ async function _loadWasm(loader: WebAssemblyInstantiator, print: ((str: string)
379379
})
380380
}
381381

382-
function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IInstantiatorOptions {
382+
function isInstantiatorOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IInstantiatorOptions {
383383
return (typeof (<IInstantiatorOptions>dataOrOptions).instantiator === 'function')
384384
}
385385

386-
function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is IDataOptions {
386+
function isDataOptionsObject(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is IDataOptions {
387387
return (typeof (<IDataOptions>dataOrOptions).data !== 'undefined')
388388
}
389389

390-
function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | IOptions): dataOrOptions is Response {
390+
function isResponse(dataOrOptions: ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): dataOrOptions is Response {
391391
return (typeof Response !== 'undefined' && dataOrOptions instanceof Response)
392392
}
393393

394394
let initCalled = false
395395
let initPromise: Promise<void> | null = null
396396

397397
export function loadWasm(loader: WebAssemblyInstantiator): Promise<void>
398-
export function loadWasm(options: IOptions): Promise<void>
398+
export function loadWasm(options: OnigurumaLoadOptions): Promise<void>
399399
export function loadWasm(data: ArrayBufferView | ArrayBuffer | Response): Promise<void>
400-
export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | IOptions): Promise<void> {
400+
export function loadWasm(dataOrOptions: WebAssemblyInstantiator | ArrayBufferView | ArrayBuffer | Response | OnigurumaLoadOptions): Promise<void> {
401401
if (initCalled) {
402402
// Already initialized
403403
return initPromise!

‎src/registry.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ export class Registry extends TextMateRegistry {
2222
_themes.map(i => [i.name, i]),
2323
)
2424
this._langMap = Object.fromEntries(
25-
_langs.map(i => [i.id, i]),
25+
_langs.map(i => [i.name, i]),
2626
)
2727
}
2828

@@ -57,7 +57,7 @@ export class Registry extends TextMateRegistry {
5757
}
5858

5959
const g = await this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig)
60-
this._resolvedGrammars[lang.id] = g!
60+
this._resolvedGrammars[lang.name] = g!
6161
if (lang.aliases) {
6262
lang.aliases.forEach((la) => {
6363
this._resolvedGrammars[la] = g!
@@ -83,8 +83,8 @@ export class Registry extends TextMateRegistry {
8383
}
8484

8585
private resolveEmbeddedLanguages(lang: LanguageRegistration) {
86-
if (!this._langGraph.has(lang.id))
87-
this._langGraph.set(lang.id, lang)
86+
if (!this._langGraph.has(lang.name))
87+
this._langGraph.set(lang.name, lang)
8888

8989
if (lang.embeddedLangs) {
9090
for (const embeddedLang of lang.embeddedLangs)

‎src/renderer.ts

-3
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,12 @@ const defaultElements: ElementsOptions = {
66
pre({ className, style, children }) {
77
return `<pre class="${className}" style="${style}" tabindex="0">${children}</pre>`
88
},
9-
109
code({ children }) {
1110
return `<code>${children}</code>`
1211
},
13-
1412
line({ className, children }) {
1513
return `<span class="${className}">${children}</span>`
1614
},
17-
1815
token({ style, children }) {
1916
return `<span style="${style}">${children}</span>`
2017
},

‎src/resolver.ts

+2-12
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@ import type { IOnigLib, RegistryOptions } from 'vscode-textmate'
33
import type { LanguageRegistration } from './types'
44

55
export class Resolver implements RegistryOptions {
6-
public languagesPath: string = 'languages/'
7-
86
private readonly languageMap: { [langIdOrAlias: string]: LanguageRegistration } = {}
97
private readonly scopeToLangMap: { [scope: string]: LanguageRegistration } = {}
108

@@ -31,19 +29,11 @@ export class Resolver implements RegistryOptions {
3129
}
3230

3331
public async loadGrammar(scopeName: string): Promise<any> {
34-
const lang = this.scopeToLangMap[scopeName]
35-
36-
if (!lang)
37-
return null
38-
39-
if (lang.grammar)
40-
return lang.grammar
41-
42-
return null
32+
return this.scopeToLangMap[scopeName]
4333
}
4434

4535
public addLanguage(l: LanguageRegistration) {
46-
this.languageMap[l.id] = l
36+
this.languageMap[l.name] = l
4737
if (l.aliases) {
4838
l.aliases.forEach((a) => {
4939
this.languageMap[a] = l

‎src/themes.ts

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from './vendor/themes'

‎src/types.ts

+9-8
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
import type { IGrammar, IRawTheme } from 'vscode-textmate'
2-
import type { languages } from './vendor/langs'
3-
import type { themes } from './vendor/themes'
1+
import type { IRawGrammar, IRawTheme } from 'vscode-textmate'
2+
import type { bundledLanguages } from './vendor/langs'
3+
import type { bundledThemes } from './vendor/themes'
44
import type { IThemedToken } from './themedTokenizer'
55

6-
export type BuiltinLanguages = keyof typeof languages
7-
export type BuiltinThemes = keyof typeof themes
6+
export type BuiltinLanguages = keyof typeof bundledLanguages
7+
export type BuiltinThemes = keyof typeof bundledThemes
88

99
export type Awaitable<T> = T | Promise<T>
1010
export type MaybeGetter<T> = T | (() => Awaitable<T>)
1111

1212
export type ThemeInput = MaybeGetter<ThemeRegisteration | ThemeRegisterationRaw>
1313
export type LanguageInput = MaybeGetter<LanguageRegistration>
1414

15-
export interface LanguageRegistration {
16-
id: string
15+
export interface LanguageRegistration extends IRawGrammar {
16+
name: string
1717
scopeName: string
1818
displayName?: string
1919
aliases?: string[]
@@ -25,7 +25,6 @@ export interface LanguageRegistration {
2525
embeddedLangs?: string[]
2626
balancedBracketSelectors?: string[]
2727
unbalancedBracketSelectors?: string[]
28-
grammar?: IGrammar
2928
}
3029

3130
export interface CodeToHtmlOptions {
@@ -125,3 +124,5 @@ interface TokenElementProps extends ElementProps {
125124
token: IThemedToken
126125
index: number
127126
}
127+
128+
export {}

‎src/vendor/langs.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { LanguageRegistration } from '../types'
22

3-
export const languages = {
3+
export const bundledLanguages = {
44
'abap': () => import('./languages/abap.json').then(r => r.default as unknown as LanguageRegistration),
55
'actionscript-3': () => import('./languages/actionscript-3.json').then(r => r.default as unknown as LanguageRegistration),
66
'ada': () => import('./languages/ada.json').then(r => r.default as unknown as LanguageRegistration),

‎src/vendor/themes.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { ThemeRegisterationRaw } from '../types'
22

3-
export const themes = {
3+
export const bundledThemes = {
44
'css-variables': () => import('shiki/themes/css-variables.json').then(r => r.default as unknown as ThemeRegisterationRaw),
55
'dark-plus': () => import('shiki/themes/dark-plus.json').then(r => r.default as unknown as ThemeRegisterationRaw),
66
'dracula-soft': () => import('shiki/themes/dracula-soft.json').then(r => r.default as unknown as ThemeRegisterationRaw),

‎test/cf.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { getHighlighter, loadWasm } from '../src/core'
1+
import { getHighlighterCore, loadWasm } from '../src/core'
22

33
import nord from '../dist/themes/nord.mjs'
44
import js from '../dist/languages/javascript.mjs'
@@ -10,7 +10,7 @@ await loadWasm(obj => WebAssembly.instantiate(wasm, obj))
1010

1111
export default {
1212
async fetch() {
13-
const highlighter = await getHighlighter({
13+
const highlighter = await getHighlighterCore({
1414
themes: [nord],
1515
langs: [js],
1616
})

‎test/core.test.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, it } from 'vitest'
2-
import { getHighlighter } from '../src/core'
2+
import { getHighlighterCore } from '../src/core'
33

44
import js from '../dist/languages/javascript.mjs'
55
import nord from '../dist/themes/nord.mjs'
@@ -9,7 +9,7 @@ import onig from '../dist/onig.mjs'
99

1010
describe('should', () => {
1111
it('exported', async () => {
12-
const shiki = await getHighlighter({
12+
const shiki = await getHighlighterCore({
1313
themes: [nord],
1414
langs: [js],
1515
loadWasm: {

0 commit comments

Comments
 (0)