Skip to content

Commit 22ed480

Browse files
manekinekkoMoLow
manekinekko
authored and
MoLow
committed
test_runner: add initial TAP parser
Work in progress PR-URL: nodejs#43525 Refs: nodejs#43344 Reviewed-By: Franziska Hinkelmann <franziska.hinkelmann@gmail.com> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
1 parent 26fb2c3 commit 22ed480

19 files changed

+4418
-31
lines changed

‎doc/api/errors.md

+19
Original file line numberDiff line numberDiff line change
@@ -2648,6 +2648,25 @@ An unspecified or non-specific system error has occurred within the Node.js
26482648
process. The error object will have an `err.info` object property with
26492649
additional details.
26502650

2651+
<a id="ERR_TAP_LEXER_ERROR"></a>
2652+
2653+
### `ERR_TAP_LEXER_ERROR`
2654+
2655+
An error representing a failing lexer state.
2656+
2657+
<a id="ERR_TAP_PARSER_ERROR"></a>
2658+
2659+
### `ERR_TAP_PARSER_ERROR`
2660+
2661+
An error representing a failing parser state. Additional information about
2662+
the token causing the error is available via the `cause` property.
2663+
2664+
<a id="ERR_TAP_VALIDATION_ERROR"></a>
2665+
2666+
### `ERR_TAP_VALIDATION_ERROR`
2667+
2668+
This error represents a failed TAP validation.
2669+
26512670
<a id="ERR_TEST_FAILURE"></a>
26522671

26532672
### `ERR_TEST_FAILURE`

‎doc/api/test.md

+2-3
Original file line numberDiff line numberDiff line change
@@ -1028,8 +1028,7 @@ Emitted when [`context.diagnostic`][] is called.
10281028
### Event: `'test:fail'`
10291029

10301030
* `data` {Object}
1031-
* `duration` {number} The test duration.
1032-
* `error` {Error} The failure casing test to fail.
1031+
* `details` {Object} Additional execution metadata.
10331032
* `name` {string} The test name.
10341033
* `testNumber` {number} The ordinal number of the test.
10351034
* `todo` {string|undefined} Present if [`context.todo`][] is called
@@ -1040,7 +1039,7 @@ Emitted when a test fails.
10401039
### Event: `'test:pass'`
10411040

10421041
* `data` {Object}
1043-
* `duration` {number} The test duration.
1042+
* `details` {Object} Additional execution metadata.
10441043
* `name` {string} The test name.
10451044
* `testNumber` {number} The ordinal number of the test.
10461045
* `todo` {string|undefined} Present if [`context.todo`][] is called

‎lib/internal/errors.js

+15
Original file line numberDiff line numberDiff line change
@@ -1587,6 +1587,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
15871587
E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
15881588
E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
15891589
E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
1590+
E('ERR_TAP_LEXER_ERROR', function(errorMsg) {
1591+
hideInternalStackFrames(this);
1592+
return errorMsg;
1593+
}, Error);
1594+
E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) {
1595+
hideInternalStackFrames(this);
1596+
this.cause = tokenCausedError;
1597+
const { column, line, start, end } = tokenCausedError.location;
1598+
const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
1599+
return errorMsg + errorDetails;
1600+
}, SyntaxError);
1601+
E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
1602+
hideInternalStackFrames(this);
1603+
return errorMsg;
1604+
}, Error);
15901605
E('ERR_TEST_FAILURE', function(error, failureType) {
15911606
hideInternalStackFrames(this);
15921607
assert(typeof failureType === 'string',

‎lib/internal/test_runner/runner.js

+111-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const {
33
ArrayFrom,
44
ArrayPrototypeFilter,
5+
ArrayPrototypeForEach,
56
ArrayPrototypeIncludes,
67
ArrayPrototypeJoin,
78
ArrayPrototypePush,
@@ -13,6 +14,7 @@ const {
1314
SafePromiseAllSettled,
1415
SafeMap,
1516
SafeSet,
17+
StringPrototypeRepeat,
1618
} = primordials;
1719

1820
const { spawn } = require('child_process');
@@ -30,7 +32,10 @@ const { validateArray, validateBoolean } = require('internal/validators');
3032
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
3133
const { kEmptyObject } = require('internal/util');
3234
const { createTestTree } = require('internal/test_runner/harness');
33-
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
35+
const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
36+
const { TapParser } = require('internal/test_runner/tap_parser');
37+
const { TokenKind } = require('internal/test_runner/tap_lexer');
38+
3439
const {
3540
isSupportedFileType,
3641
doesPathMatchFilter,
@@ -118,11 +123,103 @@ function getRunArgs({ path, inspectPort }) {
118123
return argv;
119124
}
120125

126+
class FileTest extends Test {
127+
#buffer = [];
128+
#handleReportItem({ kind, node, nesting = 0 }) {
129+
const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
130+
131+
const details = (diagnostic) => {
132+
return (
133+
diagnostic && {
134+
__proto__: null,
135+
yaml:
136+
`${indent} ` +
137+
ArrayPrototypeJoin(diagnostic, `\n${indent} `) +
138+
'\n',
139+
}
140+
);
141+
};
142+
143+
switch (kind) {
144+
case TokenKind.TAP_VERSION:
145+
// TODO(manekinekko): handle TAP version coming from the parser.
146+
// this.reporter.version(node.version);
147+
break;
148+
149+
case TokenKind.TAP_PLAN:
150+
this.reporter.plan(indent, node.end - node.start + 1);
151+
break;
152+
153+
case TokenKind.TAP_SUBTEST_POINT:
154+
this.reporter.subtest(indent, node.name);
155+
break;
156+
157+
case TokenKind.TAP_TEST_POINT:
158+
// eslint-disable-next-line no-case-declarations
159+
const { todo, skip, pass } = node.status;
160+
// eslint-disable-next-line no-case-declarations
161+
let directive;
162+
163+
if (skip) {
164+
directive = this.reporter.getSkip(node.reason);
165+
} else if (todo) {
166+
directive = this.reporter.getTodo(node.reason);
167+
} else {
168+
directive = kEmptyObject;
169+
}
170+
171+
if (pass) {
172+
this.reporter.ok(
173+
indent,
174+
node.id,
175+
node.description,
176+
details(node.diagnostics),
177+
directive
178+
);
179+
} else {
180+
this.reporter.fail(
181+
indent,
182+
node.id,
183+
node.description,
184+
details(node.diagnostics),
185+
directive
186+
);
187+
}
188+
break;
189+
190+
case TokenKind.COMMENT:
191+
if (indent === kDefaultIndent) {
192+
// Ignore file top level diagnostics
193+
break;
194+
}
195+
this.reporter.diagnostic(indent, node.comment);
196+
break;
197+
198+
case TokenKind.UNKNOWN:
199+
this.reporter.diagnostic(indent, node.value);
200+
break;
201+
}
202+
}
203+
addToReport(ast) {
204+
if (!this.isClearToSend()) {
205+
ArrayPrototypePush(this.#buffer, ast);
206+
return;
207+
}
208+
this.reportSubtest();
209+
this.#handleReportItem(ast);
210+
}
211+
report() {
212+
this.reportSubtest();
213+
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
214+
super.report();
215+
}
216+
}
217+
121218
const runningProcesses = new SafeMap();
122219
const runningSubtests = new SafeMap();
123220

124221
function runTestFile(path, root, inspectPort, filesWatcher) {
125-
const subtest = root.createSubtest(Test, path, async (t) => {
222+
const subtest = root.createSubtest(FileTest, path, async (t) => {
126223
const args = getRunArgs({ path, inspectPort });
127224
const stdio = ['pipe', 'pipe', 'pipe'];
128225
const env = { ...process.env };
@@ -133,8 +230,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
133230

134231
const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio });
135232
runningProcesses.set(path, child);
136-
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
137-
// instead of just displaying it all if the child fails.
233+
138234
let err;
139235
let stderr = '';
140236

@@ -157,6 +253,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
157253
});
158254
}
159255

256+
const parser = new TapParser();
257+
child.stderr.pipe(parser).on('data', (ast) => {
258+
if (ast.lexeme && isInspectorMessage(ast.lexeme)) {
259+
process.stderr.write(ast.lexeme + '\n');
260+
}
261+
});
262+
263+
child.stdout.pipe(parser).on('data', (ast) => {
264+
subtest.addToReport(ast);
265+
});
266+
160267
const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
161268
once(child, 'exit', { signal: t.signal }),
162269
child.stdout.toArray({ signal: t.signal }),

‎lib/internal/test_runner/tap_checker.js

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
'use strict';
2+
3+
const {
4+
ArrayPrototypeFilter,
5+
ArrayPrototypeFind,
6+
NumberParseInt,
7+
} = primordials;
8+
const {
9+
codes: { ERR_TAP_VALIDATION_ERROR },
10+
} = require('internal/errors');
11+
const { TokenKind } = require('internal/test_runner/tap_lexer');
12+
13+
// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
14+
// See https://testanything.org/tap-version-14-specification.html
15+
class TAPValidationStrategy {
16+
validate(ast) {
17+
this.#validateVersion(ast);
18+
this.#validatePlan(ast);
19+
this.#validateTestPoints(ast);
20+
21+
return true;
22+
}
23+
24+
#validateVersion(ast) {
25+
const entry = ArrayPrototypeFind(
26+
ast,
27+
(node) => node.kind === TokenKind.TAP_VERSION
28+
);
29+
30+
if (!entry) {
31+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP version');
32+
}
33+
34+
const { version } = entry.node;
35+
36+
// TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
37+
if (version !== '14' && version !== '13') {
38+
throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
39+
}
40+
}
41+
42+
#validatePlan(ast) {
43+
const entry = ArrayPrototypeFind(
44+
ast,
45+
(node) => node.kind === TokenKind.TAP_PLAN
46+
);
47+
48+
if (!entry) {
49+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan');
50+
}
51+
52+
const plan = entry.node;
53+
54+
if (!plan.start) {
55+
throw new ERR_TAP_VALIDATION_ERROR('missing plan start');
56+
}
57+
58+
if (!plan.end) {
59+
throw new ERR_TAP_VALIDATION_ERROR('missing plan end');
60+
}
61+
62+
const planStart = NumberParseInt(plan.start, 10);
63+
const planEnd = NumberParseInt(plan.end, 10);
64+
65+
if (planEnd !== 0 && planStart > planEnd) {
66+
throw new ERR_TAP_VALIDATION_ERROR(
67+
`plan start ${planStart} is greater than plan end ${planEnd}`
68+
);
69+
}
70+
}
71+
72+
// TODO(@manekinekko): since we are dealing with a flat AST, we need to
73+
// validate test points grouped by their "nesting" level. This is because a set of
74+
// Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
75+
// https://testanything.org/tap-version-14-specification.html#subtests
76+
#validateTestPoints(ast) {
77+
const bailoutEntry = ArrayPrototypeFind(
78+
ast,
79+
(node) => node.kind === TokenKind.TAP_BAIL_OUT
80+
);
81+
const planEntry = ArrayPrototypeFind(
82+
ast,
83+
(node) => node.kind === TokenKind.TAP_PLAN
84+
);
85+
const testPointEntries = ArrayPrototypeFilter(
86+
ast,
87+
(node) => node.kind === TokenKind.TAP_TEST_POINT
88+
);
89+
90+
const plan = planEntry.node;
91+
92+
const planStart = NumberParseInt(plan.start, 10);
93+
const planEnd = NumberParseInt(plan.end, 10);
94+
95+
if (planEnd === 0 && testPointEntries.length > 0) {
96+
throw new ERR_TAP_VALIDATION_ERROR(
97+
`found ${testPointEntries.length} Test Point${
98+
testPointEntries.length > 1 ? 's' : ''
99+
} but plan is ${planStart}..0`
100+
);
101+
}
102+
103+
if (planEnd > 0) {
104+
if (testPointEntries.length === 0) {
105+
throw new ERR_TAP_VALIDATION_ERROR('missing Test Points');
106+
}
107+
108+
if (!bailoutEntry && testPointEntries.length !== planEnd) {
109+
throw new ERR_TAP_VALIDATION_ERROR(
110+
`test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
111+
);
112+
}
113+
114+
for (let i = 0; i < testPointEntries.length; i++) {
115+
const test = testPointEntries[i].node;
116+
const testId = NumberParseInt(test.id, 10);
117+
118+
if (testId < planStart || testId > planEnd) {
119+
throw new ERR_TAP_VALIDATION_ERROR(
120+
`test ${testId} is out of plan range ${planStart}..${planEnd}`
121+
);
122+
}
123+
}
124+
}
125+
}
126+
}
127+
128+
// TAP14 and TAP13 are compatible with each other
129+
class TAP13ValidationStrategy extends TAPValidationStrategy {}
130+
class TAP14ValidationStrategy extends TAPValidationStrategy {}
131+
132+
class TapChecker {
133+
static TAP13 = '13';
134+
static TAP14 = '14';
135+
136+
constructor({ specs }) {
137+
switch (specs) {
138+
case TapChecker.TAP13:
139+
this.strategy = new TAP13ValidationStrategy();
140+
break;
141+
default:
142+
this.strategy = new TAP14ValidationStrategy();
143+
}
144+
}
145+
146+
check(ast) {
147+
return this.strategy.validate(ast);
148+
}
149+
}
150+
151+
module.exports = {
152+
TapChecker,
153+
TAP14ValidationStrategy,
154+
TAP13ValidationStrategy,
155+
};

0 commit comments

Comments
 (0)