Skip to content

Commit 016749b

Browse files
manekinekkoruyadorno
manekinekko
authored and
ruyadorno
committed
test_runner: add initial TAP parser
Work in progress PR-URL: #43525 Refs: #43344 Reviewed-By: Franziska Hinkelmann <franziska.hinkelmann@gmail.com> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
1 parent f720c58 commit 016749b

19 files changed

+4418
-31
lines changed

‎doc/api/errors.md

+19
Original file line numberDiff line numberDiff line change
@@ -2690,6 +2690,25 @@ An unspecified or non-specific system error has occurred within the Node.js
26902690
process. The error object will have an `err.info` object property with
26912691
additional details.
26922692

2693+
<a id="ERR_TAP_LEXER_ERROR"></a>
2694+
2695+
### `ERR_TAP_LEXER_ERROR`
2696+
2697+
An error representing a failing lexer state.
2698+
2699+
<a id="ERR_TAP_PARSER_ERROR"></a>
2700+
2701+
### `ERR_TAP_PARSER_ERROR`
2702+
2703+
An error representing a failing parser state. Additional information about
2704+
the token causing the error is available via the `cause` property.
2705+
2706+
<a id="ERR_TAP_VALIDATION_ERROR"></a>
2707+
2708+
### `ERR_TAP_VALIDATION_ERROR`
2709+
2710+
This error represents a failed TAP validation.
2711+
26932712
<a id="ERR_TEST_FAILURE"></a>
26942713

26952714
### `ERR_TEST_FAILURE`

‎doc/api/test.md

+2-3
Original file line numberDiff line numberDiff line change
@@ -1042,8 +1042,7 @@ Emitted when [`context.diagnostic`][] is called.
10421042
### Event: `'test:fail'`
10431043

10441044
* `data` {Object}
1045-
* `duration` {number} The test duration.
1046-
* `error` {Error} The failure casing test to fail.
1045+
* `details` {Object} Additional execution metadata.
10471046
* `name` {string} The test name.
10481047
* `testNumber` {number} The ordinal number of the test.
10491048
* `todo` {string|undefined} Present if [`context.todo`][] is called
@@ -1054,7 +1053,7 @@ Emitted when a test fails.
10541053
### Event: `'test:pass'`
10551054

10561055
* `data` {Object}
1057-
* `duration` {number} The test duration.
1056+
* `details` {Object} Additional execution metadata.
10581057
* `name` {string} The test name.
10591058
* `testNumber` {number} The ordinal number of the test.
10601059
* `todo` {string|undefined} Present if [`context.todo`][] is called

‎lib/internal/errors.js

+15
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
15971597
E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
15981598
E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
15991599
E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
1600+
E('ERR_TAP_LEXER_ERROR', function(errorMsg) {
1601+
hideInternalStackFrames(this);
1602+
return errorMsg;
1603+
}, Error);
1604+
E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) {
1605+
hideInternalStackFrames(this);
1606+
this.cause = tokenCausedError;
1607+
const { column, line, start, end } = tokenCausedError.location;
1608+
const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
1609+
return errorMsg + errorDetails;
1610+
}, SyntaxError);
1611+
E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
1612+
hideInternalStackFrames(this);
1613+
return errorMsg;
1614+
}, Error);
16001615
E('ERR_TEST_FAILURE', function(error, failureType) {
16011616
hideInternalStackFrames(this);
16021617
assert(typeof failureType === 'string',

‎lib/internal/test_runner/runner.js

+111-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const {
33
ArrayFrom,
44
ArrayPrototypeFilter,
5+
ArrayPrototypeForEach,
56
ArrayPrototypeIncludes,
67
ArrayPrototypeJoin,
78
ArrayPrototypePush,
@@ -14,6 +15,7 @@ const {
1415
SafePromiseAllSettledReturnVoid,
1516
SafeMap,
1617
SafeSet,
18+
StringPrototypeRepeat,
1719
} = primordials;
1820

1921
const { spawn } = require('child_process');
@@ -31,7 +33,10 @@ const { validateArray, validateBoolean } = require('internal/validators');
3133
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
3234
const { kEmptyObject } = require('internal/util');
3335
const { createTestTree } = require('internal/test_runner/harness');
34-
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
36+
const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
37+
const { TapParser } = require('internal/test_runner/tap_parser');
38+
const { TokenKind } = require('internal/test_runner/tap_lexer');
39+
3540
const {
3641
isSupportedFileType,
3742
doesPathMatchFilter,
@@ -120,11 +125,103 @@ function getRunArgs({ path, inspectPort }) {
120125
return argv;
121126
}
122127

128+
class FileTest extends Test {
129+
#buffer = [];
130+
#handleReportItem({ kind, node, nesting = 0 }) {
131+
const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
132+
133+
const details = (diagnostic) => {
134+
return (
135+
diagnostic && {
136+
__proto__: null,
137+
yaml:
138+
`${indent} ` +
139+
ArrayPrototypeJoin(diagnostic, `\n${indent} `) +
140+
'\n',
141+
}
142+
);
143+
};
144+
145+
switch (kind) {
146+
case TokenKind.TAP_VERSION:
147+
// TODO(manekinekko): handle TAP version coming from the parser.
148+
// this.reporter.version(node.version);
149+
break;
150+
151+
case TokenKind.TAP_PLAN:
152+
this.reporter.plan(indent, node.end - node.start + 1);
153+
break;
154+
155+
case TokenKind.TAP_SUBTEST_POINT:
156+
this.reporter.subtest(indent, node.name);
157+
break;
158+
159+
case TokenKind.TAP_TEST_POINT:
160+
// eslint-disable-next-line no-case-declarations
161+
const { todo, skip, pass } = node.status;
162+
// eslint-disable-next-line no-case-declarations
163+
let directive;
164+
165+
if (skip) {
166+
directive = this.reporter.getSkip(node.reason);
167+
} else if (todo) {
168+
directive = this.reporter.getTodo(node.reason);
169+
} else {
170+
directive = kEmptyObject;
171+
}
172+
173+
if (pass) {
174+
this.reporter.ok(
175+
indent,
176+
node.id,
177+
node.description,
178+
details(node.diagnostics),
179+
directive
180+
);
181+
} else {
182+
this.reporter.fail(
183+
indent,
184+
node.id,
185+
node.description,
186+
details(node.diagnostics),
187+
directive
188+
);
189+
}
190+
break;
191+
192+
case TokenKind.COMMENT:
193+
if (indent === kDefaultIndent) {
194+
// Ignore file top level diagnostics
195+
break;
196+
}
197+
this.reporter.diagnostic(indent, node.comment);
198+
break;
199+
200+
case TokenKind.UNKNOWN:
201+
this.reporter.diagnostic(indent, node.value);
202+
break;
203+
}
204+
}
205+
addToReport(ast) {
206+
if (!this.isClearToSend()) {
207+
ArrayPrototypePush(this.#buffer, ast);
208+
return;
209+
}
210+
this.reportSubtest();
211+
this.#handleReportItem(ast);
212+
}
213+
report() {
214+
this.reportSubtest();
215+
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
216+
super.report();
217+
}
218+
}
219+
123220
const runningProcesses = new SafeMap();
124221
const runningSubtests = new SafeMap();
125222

126223
function runTestFile(path, root, inspectPort, filesWatcher) {
127-
const subtest = root.createSubtest(Test, path, async (t) => {
224+
const subtest = root.createSubtest(FileTest, path, async (t) => {
128225
const args = getRunArgs({ path, inspectPort });
129226
const stdio = ['pipe', 'pipe', 'pipe'];
130227
const env = { ...process.env };
@@ -135,8 +232,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
135232

136233
const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio });
137234
runningProcesses.set(path, child);
138-
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
139-
// instead of just displaying it all if the child fails.
235+
140236
let err;
141237
let stderr = '';
142238

@@ -159,6 +255,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
159255
});
160256
}
161257

258+
const parser = new TapParser();
259+
child.stderr.pipe(parser).on('data', (ast) => {
260+
if (ast.lexeme && isInspectorMessage(ast.lexeme)) {
261+
process.stderr.write(ast.lexeme + '\n');
262+
}
263+
});
264+
265+
child.stdout.pipe(parser).on('data', (ast) => {
266+
subtest.addToReport(ast);
267+
});
268+
162269
const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
163270
once(child, 'exit', { signal: t.signal }),
164271
child.stdout.toArray({ signal: t.signal }),

‎lib/internal/test_runner/tap_checker.js

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
'use strict';
2+
3+
const {
4+
ArrayPrototypeFilter,
5+
ArrayPrototypeFind,
6+
NumberParseInt,
7+
} = primordials;
8+
const {
9+
codes: { ERR_TAP_VALIDATION_ERROR },
10+
} = require('internal/errors');
11+
const { TokenKind } = require('internal/test_runner/tap_lexer');
12+
13+
// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
14+
// See https://testanything.org/tap-version-14-specification.html
15+
class TAPValidationStrategy {
16+
validate(ast) {
17+
this.#validateVersion(ast);
18+
this.#validatePlan(ast);
19+
this.#validateTestPoints(ast);
20+
21+
return true;
22+
}
23+
24+
#validateVersion(ast) {
25+
const entry = ArrayPrototypeFind(
26+
ast,
27+
(node) => node.kind === TokenKind.TAP_VERSION
28+
);
29+
30+
if (!entry) {
31+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP version');
32+
}
33+
34+
const { version } = entry.node;
35+
36+
// TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
37+
if (version !== '14' && version !== '13') {
38+
throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
39+
}
40+
}
41+
42+
#validatePlan(ast) {
43+
const entry = ArrayPrototypeFind(
44+
ast,
45+
(node) => node.kind === TokenKind.TAP_PLAN
46+
);
47+
48+
if (!entry) {
49+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan');
50+
}
51+
52+
const plan = entry.node;
53+
54+
if (!plan.start) {
55+
throw new ERR_TAP_VALIDATION_ERROR('missing plan start');
56+
}
57+
58+
if (!plan.end) {
59+
throw new ERR_TAP_VALIDATION_ERROR('missing plan end');
60+
}
61+
62+
const planStart = NumberParseInt(plan.start, 10);
63+
const planEnd = NumberParseInt(plan.end, 10);
64+
65+
if (planEnd !== 0 && planStart > planEnd) {
66+
throw new ERR_TAP_VALIDATION_ERROR(
67+
`plan start ${planStart} is greater than plan end ${planEnd}`
68+
);
69+
}
70+
}
71+
72+
// TODO(@manekinekko): since we are dealing with a flat AST, we need to
73+
// validate test points grouped by their "nesting" level. This is because a set of
74+
// Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
75+
// https://testanything.org/tap-version-14-specification.html#subtests
76+
#validateTestPoints(ast) {
77+
const bailoutEntry = ArrayPrototypeFind(
78+
ast,
79+
(node) => node.kind === TokenKind.TAP_BAIL_OUT
80+
);
81+
const planEntry = ArrayPrototypeFind(
82+
ast,
83+
(node) => node.kind === TokenKind.TAP_PLAN
84+
);
85+
const testPointEntries = ArrayPrototypeFilter(
86+
ast,
87+
(node) => node.kind === TokenKind.TAP_TEST_POINT
88+
);
89+
90+
const plan = planEntry.node;
91+
92+
const planStart = NumberParseInt(plan.start, 10);
93+
const planEnd = NumberParseInt(plan.end, 10);
94+
95+
if (planEnd === 0 && testPointEntries.length > 0) {
96+
throw new ERR_TAP_VALIDATION_ERROR(
97+
`found ${testPointEntries.length} Test Point${
98+
testPointEntries.length > 1 ? 's' : ''
99+
} but plan is ${planStart}..0`
100+
);
101+
}
102+
103+
if (planEnd > 0) {
104+
if (testPointEntries.length === 0) {
105+
throw new ERR_TAP_VALIDATION_ERROR('missing Test Points');
106+
}
107+
108+
if (!bailoutEntry && testPointEntries.length !== planEnd) {
109+
throw new ERR_TAP_VALIDATION_ERROR(
110+
`test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
111+
);
112+
}
113+
114+
for (let i = 0; i < testPointEntries.length; i++) {
115+
const test = testPointEntries[i].node;
116+
const testId = NumberParseInt(test.id, 10);
117+
118+
if (testId < planStart || testId > planEnd) {
119+
throw new ERR_TAP_VALIDATION_ERROR(
120+
`test ${testId} is out of plan range ${planStart}..${planEnd}`
121+
);
122+
}
123+
}
124+
}
125+
}
126+
}
127+
128+
// TAP14 and TAP13 are compatible with each other
129+
class TAP13ValidationStrategy extends TAPValidationStrategy {}
130+
class TAP14ValidationStrategy extends TAPValidationStrategy {}
131+
132+
class TapChecker {
133+
static TAP13 = '13';
134+
static TAP14 = '14';
135+
136+
constructor({ specs }) {
137+
switch (specs) {
138+
case TapChecker.TAP13:
139+
this.strategy = new TAP13ValidationStrategy();
140+
break;
141+
default:
142+
this.strategy = new TAP14ValidationStrategy();
143+
}
144+
}
145+
146+
check(ast) {
147+
return this.strategy.validate(ast);
148+
}
149+
}
150+
151+
module.exports = {
152+
TapChecker,
153+
TAP14ValidationStrategy,
154+
TAP13ValidationStrategy,
155+
};

0 commit comments

Comments
 (0)