Skip to content

Commit c85d00b

Browse files
rubystargos
rubys
authored and
targos
committed
tools: produce JSON documentation using unified/remark/rehype
PR-URL: #21697 Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com> Reviewed-By: Rich Trott <rtrott@gmail.com> Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
1 parent 40af976 commit c85d00b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+494
-23127
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ deps/npm/node_modules/.bin/
107107
/*.pkg
108108
/SHASUMS*.txt*
109109

110+
# api docs artifacts
111+
tools/doc/node_modules
112+
110113
# test artifacts
111114
tools/remark-cli/node_modules
112115
tools/remark-preset-lint-node/node_modules

‎Makefile

+8-8
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ ifeq ($(OSTYPE),aix)
320320
DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
321321
endif
322322

323-
test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
323+
test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
324324
$(RM) -r test/addons/??_*/
325325
[ -x $(NODE) ] && $(NODE) $< || node $<
326326
touch $@
@@ -633,15 +633,15 @@ available-node = \
633633
run-npm-install = $(PWD)/$(NPM) install --production --no-package-lock
634634
run-npm-ci = $(PWD)/$(NPM) ci
635635

636-
gen-json = tools/doc/generate.js --format=json $< > $@
637-
gen-html = tools/doc/generate.js --node-version=$(FULLVERSION) --format=html \
638-
--analytics=$(DOCS_ANALYTICS) $< > $@
636+
tools/doc/node_modules/js-yaml/package.json:
637+
cd tools/doc && $(call available-node,$(run-npm-install))
639638

640-
out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
641-
$(call available-node, $(gen-json))
639+
gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \
640+
--analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api
642641

643-
out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
644-
$(call available-node, $(gen-html))
642+
out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
643+
tools/doc/html.js tools/doc/json.js
644+
$(call available-node, $(gen-api))
645645

646646
out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
647647
$(call available-node, tools/doc/allhtml.js)

‎doc/api/https.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ changes:
123123
pr-url: https://github.com/nodejs/node/pull/10638
124124
description: The `options` parameter can be a WHATWG `URL` object.
125125
-->
126-
- `url` {string | URL}
126+
* `url` {string | URL}
127127
* `options` {Object | string | URL} Accepts the same `options` as
128128
[`https.request()`][], with the `method` always set to `GET`.
129129
* `callback` {Function}
@@ -174,7 +174,7 @@ changes:
174174
pr-url: https://github.com/nodejs/node/pull/10638
175175
description: The `options` parameter can be a WHATWG `URL` object.
176176
-->
177-
- `url` {string | URL}
177+
* `url` {string | URL}
178178
* `options` {Object | string | URL} Accepts all `options` from
179179
[`http.request()`][], with some differences in default values:
180180
- `protocol` **Default:** `'https:'`

‎test/doctool/test-doctool-html.js

+28-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,34 @@ try {
1111
const assert = require('assert');
1212
const { readFile } = require('fs');
1313
const fixtures = require('../common/fixtures');
14-
const toHTML = require('../../tools/doc/html.js');
14+
const html = require('../../tools/doc/html.js');
15+
const path = require('path');
16+
17+
module.paths.unshift(
18+
path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
19+
const unified = require('unified');
20+
const markdown = require('remark-parse');
21+
const remark2rehype = require('remark-rehype');
22+
const raw = require('rehype-raw');
23+
const htmlStringify = require('rehype-stringify');
24+
25+
function toHTML({ input, filename, nodeVersion, analytics }, cb) {
26+
const content = unified()
27+
.use(markdown)
28+
.use(html.firstHeader)
29+
.use(html.preprocessText)
30+
.use(html.preprocessElements, { filename })
31+
.use(html.buildToc, { filename })
32+
.use(remark2rehype, { allowDangerousHTML: true })
33+
.use(raw)
34+
.use(htmlStringify)
35+
.processSync(input);
36+
37+
html.toHTML(
38+
{ input, content, filename, nodeVersion, analytics },
39+
cb
40+
);
41+
}
1542

1643
// Test data is a list of objects with two properties.
1744
// The file property is the file path.
@@ -80,7 +107,6 @@ testData.forEach(({ file, html, analyticsId }) => {
80107

81108
readFile(file, 'utf8', common.mustCall((err, input) => {
82109
assert.ifError(err);
83-
84110
toHTML(
85111
{
86112
input: input,

‎test/doctool/test-doctool-json.js

+30-14
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,27 @@ try {
1010

1111
const assert = require('assert');
1212
const fs = require('fs');
13+
const path = require('path');
1314
const fixtures = require('../common/fixtures');
1415
const json = require('../../tools/doc/json.js');
1516

17+
module.paths.unshift(
18+
path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
19+
const unified = require('unified');
20+
const markdown = require('remark-parse');
21+
22+
function toJSON(input, filename, cb) {
23+
function nullCompiler() {
24+
this.Compiler = (tree) => tree;
25+
}
26+
27+
unified()
28+
.use(markdown)
29+
.use(json.jsonAPI, { filename })
30+
.use(nullCompiler)
31+
.process(input, cb);
32+
}
33+
1634
// Outputs valid json with the expected fields when given simple markdown
1735
// Test data is a list of objects with two properties.
1836
// The file property is the file path.
@@ -21,15 +39,16 @@ const testData = [
2139
{
2240
file: fixtures.path('sample_document.md'),
2341
json: {
42+
type: 'module',
2443
source: 'foo',
2544
modules: [{
2645
textRaw: 'Sample Markdown',
2746
name: 'sample_markdown',
2847
modules: [{
2948
textRaw: 'Seussian Rhymes',
3049
name: 'seussian_rhymes',
31-
desc: '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>' +
32-
'<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
50+
desc: '<ol>\n<li>fish</li>\n<li>fish</li>\n</ol>\n' +
51+
'<ul>\n<li>Red fish</li>\n<li>Blue fish</li>\n</ul>',
3352
type: 'module',
3453
displayName: 'Seussian Rhymes'
3554
}],
@@ -41,6 +60,7 @@ const testData = [
4160
{
4261
file: fixtures.path('order_of_end_tags_5873.md'),
4362
json: {
63+
type: 'module',
4464
source: 'foo',
4565
modules: [{
4666
textRaw: 'Title',
@@ -55,15 +75,10 @@ const testData = [
5575
signatures: [
5676
{
5777
params: [{
58-
textRaw: '`array` {Array} ',
78+
textRaw: '`array` {Array}',
5979
name: 'array',
6080
type: 'Array'
6181
}]
62-
},
63-
{
64-
params: [{
65-
name: 'array'
66-
}]
6782
}
6883
]
6984
}],
@@ -78,6 +93,7 @@ const testData = [
7893
{
7994
file: fixtures.path('doc_with_yaml.md'),
8095
json: {
96+
type: 'module',
8197
source: 'foo',
8298
modules: [
8399
{
@@ -92,7 +108,7 @@ const testData = [
92108
changes: []
93109
},
94110
desc: '<p>Describe <code>Foobar</code> in more detail ' +
95-
'here.</p>\n',
111+
'here.</p>',
96112
type: 'module',
97113
displayName: 'Foobar'
98114
},
@@ -110,7 +126,7 @@ const testData = [
110126
]
111127
},
112128
desc: '<p>Describe <code>Foobar II</code> in more detail ' +
113-
'here. fg(1)</p>\n',
129+
'here. fg(1)</p>',
114130
type: 'module',
115131
displayName: 'Foobar II'
116132
},
@@ -123,15 +139,15 @@ const testData = [
123139
changes: []
124140
},
125141
desc: '<p>Describe <code>Deprecated thingy</code> in more ' +
126-
'detail here. fg(1p)</p>\n',
142+
'detail here. fg(1p)</p>',
127143
type: 'module',
128144
displayName: 'Deprecated thingy'
129145
},
130146
{
131147
textRaw: 'Something',
132148
name: 'something',
133149
desc: '<!-- This is not a metadata comment -->\n<p>' +
134-
'Describe <code>Something</code> in more detail here.</p>\n',
150+
'Describe <code>Something</code> in more detail here.</p>',
135151
type: 'module',
136152
displayName: 'Something'
137153
}
@@ -147,9 +163,9 @@ const testData = [
147163
testData.forEach((item) => {
148164
fs.readFile(item.file, 'utf8', common.mustCall((err, input) => {
149165
assert.ifError(err);
150-
json(input, 'foo', common.mustCall((err, output) => {
166+
toJSON(input, 'foo', common.mustCall((err, output) => {
151167
assert.ifError(err);
152-
assert.deepStrictEqual(output, item.json);
168+
assert.deepStrictEqual(output.json, item.json);
153169
}));
154170
}));
155171
});

‎tools/doc/generate.js

+39-20
Original file line numberDiff line numberDiff line change
@@ -22,53 +22,72 @@
2222
'use strict';
2323

2424
const fs = require('fs');
25+
const path = require('path');
26+
const unified = require('unified');
27+
const markdown = require('remark-parse');
28+
const remark2rehype = require('remark-rehype');
29+
const raw = require('rehype-raw');
30+
const htmlStringify = require('rehype-stringify');
31+
32+
const html = require('./html');
33+
const json = require('./json');
2534

2635
// Parse the args.
2736
// Don't use nopt or whatever for this. It's simple enough.
2837

2938
const args = process.argv.slice(2);
30-
let format = 'json';
3139
let filename = null;
3240
let nodeVersion = null;
3341
let analytics = null;
42+
let outputDir = null;
3443

3544
args.forEach(function(arg) {
3645
if (!arg.startsWith('--')) {
3746
filename = arg;
38-
} else if (arg.startsWith('--format=')) {
39-
format = arg.replace(/^--format=/, '');
4047
} else if (arg.startsWith('--node-version=')) {
4148
nodeVersion = arg.replace(/^--node-version=/, '');
4249
} else if (arg.startsWith('--analytics=')) {
4350
analytics = arg.replace(/^--analytics=/, '');
51+
} else if (arg.startsWith('--output-directory=')) {
52+
outputDir = arg.replace(/^--output-directory=/, '');
4453
}
4554
});
4655

4756
nodeVersion = nodeVersion || process.version;
4857

4958
if (!filename) {
5059
throw new Error('No input file specified');
60+
} else if (!outputDir) {
61+
throw new Error('No output directory specified');
5162
}
5263

64+
5365
fs.readFile(filename, 'utf8', (er, input) => {
5466
if (er) throw er;
55-
switch (format) {
56-
case 'json':
57-
require('./json.js')(input, filename, (er, obj) => {
58-
if (er) throw er;
59-
console.log(JSON.stringify(obj, null, 2));
60-
});
61-
break;
6267

63-
case 'html':
64-
require('./html')({ input, filename, nodeVersion, analytics },
65-
(err, html) => {
66-
if (err) throw err;
67-
console.log(html);
68-
});
69-
break;
68+
const content = unified()
69+
.use(markdown)
70+
.use(json.jsonAPI, { filename })
71+
.use(html.firstHeader)
72+
.use(html.preprocessText)
73+
.use(html.preprocessElements, { filename })
74+
.use(html.buildToc, { filename })
75+
.use(remark2rehype, { allowDangerousHTML: true })
76+
.use(raw)
77+
.use(htmlStringify)
78+
.processSync(input);
7079

71-
default:
72-
throw new Error(`Invalid format: ${format}`);
73-
}
80+
const basename = path.basename(filename, '.md');
81+
82+
html.toHTML(
83+
{ input, content, filename, nodeVersion, analytics },
84+
(err, html) => {
85+
const target = path.join(outputDir, `${basename}.html`);
86+
if (err) throw err;
87+
fs.writeFileSync(target, html);
88+
}
89+
);
90+
91+
const target = path.join(outputDir, `${basename}.json`);
92+
fs.writeFileSync(target, JSON.stringify(content.json, null, 2));
7493
});

‎tools/doc/html.js

+8-17
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,13 @@ const visit = require('unist-util-visit');
2929
const markdown = require('remark-parse');
3030
const remark2rehype = require('remark-rehype');
3131
const raw = require('rehype-raw');
32-
const html = require('rehype-stringify');
32+
const htmlStringify = require('rehype-stringify');
3333
const path = require('path');
3434
const typeParser = require('./type-parser.js');
3535

36-
module.exports = toHTML;
36+
module.exports = {
37+
toHTML, firstHeader, preprocessText, preprocessElements, buildToc
38+
};
3739

3840
const docPath = path.resolve(__dirname, '..', '..', 'doc');
3941

@@ -54,26 +56,15 @@ const gtocHTML = unified()
5456
.use(remark2rehype, { allowDangerousHTML: true })
5557
.use(raw)
5658
.use(navClasses)
57-
.use(html)
59+
.use(htmlStringify)
5860
.processSync(gtocMD).toString();
5961

6062
const templatePath = path.join(docPath, 'template.html');
6163
const template = fs.readFileSync(templatePath, 'utf8');
6264

63-
function toHTML({ input, filename, nodeVersion, analytics }, cb) {
65+
function toHTML({ input, content, filename, nodeVersion, analytics }, cb) {
6466
filename = path.basename(filename, '.md');
6567

66-
const content = unified()
67-
.use(markdown)
68-
.use(firstHeader)
69-
.use(preprocessText)
70-
.use(preprocessElements, { filename })
71-
.use(buildToc, { filename })
72-
.use(remark2rehype, { allowDangerousHTML: true })
73-
.use(raw)
74-
.use(html)
75-
.processSync(input);
76-
7768
const id = filename.replace(/\W+/g, '-');
7869

7970
let HTML = template.replace('__ID__', id)
@@ -296,7 +287,7 @@ function parseYAML(text) {
296287
.use(markdown)
297288
.use(remark2rehype, { allowDangerousHTML: true })
298289
.use(raw)
299-
.use(html)
290+
.use(htmlStringify)
300291
.processSync(change.description).toString();
301292

302293
result += `<tr><td>${change.version}</td>\n` +
@@ -381,7 +372,7 @@ function buildToc({ filename }) {
381372
.use(markdown)
382373
.use(remark2rehype, { allowDangerousHTML: true })
383374
.use(raw)
384-
.use(html)
375+
.use(htmlStringify)
385376
.processSync(toc).toString();
386377
};
387378
}

0 commit comments

Comments
 (0)