Skip to content

Commit 16bedba

Browse files
manekinekkodanielleadams
manekinekko
authored and
danielleadams
committed
test_runner: add initial TAP parser
Work in progress PR-URL: #43525 Refs: #43344 Reviewed-By: Franziska Hinkelmann <franziska.hinkelmann@gmail.com> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
1 parent 2bde576 commit 16bedba

19 files changed

+4418
-31
lines changed

‎doc/api/errors.md

+19
Original file line numberDiff line numberDiff line change
@@ -2682,6 +2682,25 @@ An unspecified or non-specific system error has occurred within the Node.js
26822682
process. The error object will have an `err.info` object property with
26832683
additional details.
26842684

2685+
<a id="ERR_TAP_LEXER_ERROR"></a>
2686+
2687+
### `ERR_TAP_LEXER_ERROR`
2688+
2689+
An error representing a failing lexer state.
2690+
2691+
<a id="ERR_TAP_PARSER_ERROR"></a>
2692+
2693+
### `ERR_TAP_PARSER_ERROR`
2694+
2695+
An error representing a failing parser state. Additional information about
2696+
the token causing the error is available via the `cause` property.
2697+
2698+
<a id="ERR_TAP_VALIDATION_ERROR"></a>
2699+
2700+
### `ERR_TAP_VALIDATION_ERROR`
2701+
2702+
This error represents a failed TAP validation.
2703+
26852704
<a id="ERR_TEST_FAILURE"></a>
26862705

26872706
### `ERR_TEST_FAILURE`

‎doc/api/test.md

+2-3
Original file line numberDiff line numberDiff line change
@@ -1028,8 +1028,7 @@ Emitted when [`context.diagnostic`][] is called.
10281028
### Event: `'test:fail'`
10291029

10301030
* `data` {Object}
1031-
* `duration` {number} The test duration.
1032-
* `error` {Error} The failure casing test to fail.
1031+
* `details` {Object} Additional execution metadata.
10331032
* `name` {string} The test name.
10341033
* `testNumber` {number} The ordinal number of the test.
10351034
* `todo` {string|undefined} Present if [`context.todo`][] is called
@@ -1040,7 +1039,7 @@ Emitted when a test fails.
10401039
### Event: `'test:pass'`
10411040

10421041
* `data` {Object}
1043-
* `duration` {number} The test duration.
1042+
* `details` {Object} Additional execution metadata.
10441043
* `name` {string} The test name.
10451044
* `testNumber` {number} The ordinal number of the test.
10461045
* `todo` {string|undefined} Present if [`context.todo`][] is called

‎lib/internal/errors.js

+15
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
15971597
E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
15981598
E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
15991599
E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
1600+
E('ERR_TAP_LEXER_ERROR', function(errorMsg) {
1601+
hideInternalStackFrames(this);
1602+
return errorMsg;
1603+
}, Error);
1604+
E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) {
1605+
hideInternalStackFrames(this);
1606+
this.cause = tokenCausedError;
1607+
const { column, line, start, end } = tokenCausedError.location;
1608+
const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
1609+
return errorMsg + errorDetails;
1610+
}, SyntaxError);
1611+
E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
1612+
hideInternalStackFrames(this);
1613+
return errorMsg;
1614+
}, Error);
16001615
E('ERR_TEST_FAILURE', function(error, failureType) {
16011616
hideInternalStackFrames(this);
16021617
assert(typeof failureType === 'string',

‎lib/internal/test_runner/runner.js

+111-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const {
33
ArrayFrom,
44
ArrayPrototypeFilter,
5+
ArrayPrototypeForEach,
56
ArrayPrototypeIncludes,
67
ArrayPrototypeJoin,
78
ArrayPrototypePush,
@@ -14,6 +15,7 @@ const {
1415
SafePromiseAllSettledReturnVoid,
1516
SafeMap,
1617
SafeSet,
18+
StringPrototypeRepeat,
1719
} = primordials;
1820

1921
const { spawn } = require('child_process');
@@ -31,7 +33,10 @@ const { validateArray, validateBoolean } = require('internal/validators');
3133
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
3234
const { kEmptyObject } = require('internal/util');
3335
const { createTestTree } = require('internal/test_runner/harness');
34-
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
36+
const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
37+
const { TapParser } = require('internal/test_runner/tap_parser');
38+
const { TokenKind } = require('internal/test_runner/tap_lexer');
39+
3540
const {
3641
isSupportedFileType,
3742
doesPathMatchFilter,
@@ -119,11 +124,103 @@ function getRunArgs({ path, inspectPort }) {
119124
return argv;
120125
}
121126

127+
class FileTest extends Test {
128+
#buffer = [];
129+
#handleReportItem({ kind, node, nesting = 0 }) {
130+
const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
131+
132+
const details = (diagnostic) => {
133+
return (
134+
diagnostic && {
135+
__proto__: null,
136+
yaml:
137+
`${indent} ` +
138+
ArrayPrototypeJoin(diagnostic, `\n${indent} `) +
139+
'\n',
140+
}
141+
);
142+
};
143+
144+
switch (kind) {
145+
case TokenKind.TAP_VERSION:
146+
// TODO(manekinekko): handle TAP version coming from the parser.
147+
// this.reporter.version(node.version);
148+
break;
149+
150+
case TokenKind.TAP_PLAN:
151+
this.reporter.plan(indent, node.end - node.start + 1);
152+
break;
153+
154+
case TokenKind.TAP_SUBTEST_POINT:
155+
this.reporter.subtest(indent, node.name);
156+
break;
157+
158+
case TokenKind.TAP_TEST_POINT:
159+
// eslint-disable-next-line no-case-declarations
160+
const { todo, skip, pass } = node.status;
161+
// eslint-disable-next-line no-case-declarations
162+
let directive;
163+
164+
if (skip) {
165+
directive = this.reporter.getSkip(node.reason);
166+
} else if (todo) {
167+
directive = this.reporter.getTodo(node.reason);
168+
} else {
169+
directive = kEmptyObject;
170+
}
171+
172+
if (pass) {
173+
this.reporter.ok(
174+
indent,
175+
node.id,
176+
node.description,
177+
details(node.diagnostics),
178+
directive
179+
);
180+
} else {
181+
this.reporter.fail(
182+
indent,
183+
node.id,
184+
node.description,
185+
details(node.diagnostics),
186+
directive
187+
);
188+
}
189+
break;
190+
191+
case TokenKind.COMMENT:
192+
if (indent === kDefaultIndent) {
193+
// Ignore file top level diagnostics
194+
break;
195+
}
196+
this.reporter.diagnostic(indent, node.comment);
197+
break;
198+
199+
case TokenKind.UNKNOWN:
200+
this.reporter.diagnostic(indent, node.value);
201+
break;
202+
}
203+
}
204+
addToReport(ast) {
205+
if (!this.isClearToSend()) {
206+
ArrayPrototypePush(this.#buffer, ast);
207+
return;
208+
}
209+
this.reportSubtest();
210+
this.#handleReportItem(ast);
211+
}
212+
report() {
213+
this.reportSubtest();
214+
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
215+
super.report();
216+
}
217+
}
218+
122219
const runningProcesses = new SafeMap();
123220
const runningSubtests = new SafeMap();
124221

125222
function runTestFile(path, root, inspectPort, filesWatcher) {
126-
const subtest = root.createSubtest(Test, path, async (t) => {
223+
const subtest = root.createSubtest(FileTest, path, async (t) => {
127224
const args = getRunArgs({ path, inspectPort });
128225
const stdio = ['pipe', 'pipe', 'pipe'];
129226
const env = { ...process.env };
@@ -134,8 +231,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
134231

135232
const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio });
136233
runningProcesses.set(path, child);
137-
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
138-
// instead of just displaying it all if the child fails.
234+
139235
let err;
140236
let stderr = '';
141237

@@ -158,6 +254,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
158254
});
159255
}
160256

257+
const parser = new TapParser();
258+
child.stderr.pipe(parser).on('data', (ast) => {
259+
if (ast.lexeme && isInspectorMessage(ast.lexeme)) {
260+
process.stderr.write(ast.lexeme + '\n');
261+
}
262+
});
263+
264+
child.stdout.pipe(parser).on('data', (ast) => {
265+
subtest.addToReport(ast);
266+
});
267+
161268
const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
162269
once(child, 'exit', { signal: t.signal }),
163270
child.stdout.toArray({ signal: t.signal }),

‎lib/internal/test_runner/tap_checker.js

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
'use strict';
2+
3+
const {
4+
ArrayPrototypeFilter,
5+
ArrayPrototypeFind,
6+
NumberParseInt,
7+
} = primordials;
8+
const {
9+
codes: { ERR_TAP_VALIDATION_ERROR },
10+
} = require('internal/errors');
11+
const { TokenKind } = require('internal/test_runner/tap_lexer');
12+
13+
// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
14+
// See https://testanything.org/tap-version-14-specification.html
15+
class TAPValidationStrategy {
16+
validate(ast) {
17+
this.#validateVersion(ast);
18+
this.#validatePlan(ast);
19+
this.#validateTestPoints(ast);
20+
21+
return true;
22+
}
23+
24+
#validateVersion(ast) {
25+
const entry = ArrayPrototypeFind(
26+
ast,
27+
(node) => node.kind === TokenKind.TAP_VERSION
28+
);
29+
30+
if (!entry) {
31+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP version');
32+
}
33+
34+
const { version } = entry.node;
35+
36+
// TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
37+
if (version !== '14' && version !== '13') {
38+
throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
39+
}
40+
}
41+
42+
#validatePlan(ast) {
43+
const entry = ArrayPrototypeFind(
44+
ast,
45+
(node) => node.kind === TokenKind.TAP_PLAN
46+
);
47+
48+
if (!entry) {
49+
throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan');
50+
}
51+
52+
const plan = entry.node;
53+
54+
if (!plan.start) {
55+
throw new ERR_TAP_VALIDATION_ERROR('missing plan start');
56+
}
57+
58+
if (!plan.end) {
59+
throw new ERR_TAP_VALIDATION_ERROR('missing plan end');
60+
}
61+
62+
const planStart = NumberParseInt(plan.start, 10);
63+
const planEnd = NumberParseInt(plan.end, 10);
64+
65+
if (planEnd !== 0 && planStart > planEnd) {
66+
throw new ERR_TAP_VALIDATION_ERROR(
67+
`plan start ${planStart} is greater than plan end ${planEnd}`
68+
);
69+
}
70+
}
71+
72+
// TODO(@manekinekko): since we are dealing with a flat AST, we need to
73+
// validate test points grouped by their "nesting" level. This is because a set of
74+
// Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
75+
// https://testanything.org/tap-version-14-specification.html#subtests
76+
#validateTestPoints(ast) {
77+
const bailoutEntry = ArrayPrototypeFind(
78+
ast,
79+
(node) => node.kind === TokenKind.TAP_BAIL_OUT
80+
);
81+
const planEntry = ArrayPrototypeFind(
82+
ast,
83+
(node) => node.kind === TokenKind.TAP_PLAN
84+
);
85+
const testPointEntries = ArrayPrototypeFilter(
86+
ast,
87+
(node) => node.kind === TokenKind.TAP_TEST_POINT
88+
);
89+
90+
const plan = planEntry.node;
91+
92+
const planStart = NumberParseInt(plan.start, 10);
93+
const planEnd = NumberParseInt(plan.end, 10);
94+
95+
if (planEnd === 0 && testPointEntries.length > 0) {
96+
throw new ERR_TAP_VALIDATION_ERROR(
97+
`found ${testPointEntries.length} Test Point${
98+
testPointEntries.length > 1 ? 's' : ''
99+
} but plan is ${planStart}..0`
100+
);
101+
}
102+
103+
if (planEnd > 0) {
104+
if (testPointEntries.length === 0) {
105+
throw new ERR_TAP_VALIDATION_ERROR('missing Test Points');
106+
}
107+
108+
if (!bailoutEntry && testPointEntries.length !== planEnd) {
109+
throw new ERR_TAP_VALIDATION_ERROR(
110+
`test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
111+
);
112+
}
113+
114+
for (let i = 0; i < testPointEntries.length; i++) {
115+
const test = testPointEntries[i].node;
116+
const testId = NumberParseInt(test.id, 10);
117+
118+
if (testId < planStart || testId > planEnd) {
119+
throw new ERR_TAP_VALIDATION_ERROR(
120+
`test ${testId} is out of plan range ${planStart}..${planEnd}`
121+
);
122+
}
123+
}
124+
}
125+
}
126+
}
127+
128+
// TAP14 and TAP13 are compatible with each other
129+
class TAP13ValidationStrategy extends TAPValidationStrategy {}
130+
class TAP14ValidationStrategy extends TAPValidationStrategy {}
131+
132+
class TapChecker {
133+
static TAP13 = '13';
134+
static TAP14 = '14';
135+
136+
constructor({ specs }) {
137+
switch (specs) {
138+
case TapChecker.TAP13:
139+
this.strategy = new TAP13ValidationStrategy();
140+
break;
141+
default:
142+
this.strategy = new TAP14ValidationStrategy();
143+
}
144+
}
145+
146+
check(ast) {
147+
return this.strategy.validate(ast);
148+
}
149+
}
150+
151+
module.exports = {
152+
TapChecker,
153+
TAP14ValidationStrategy,
154+
TAP13ValidationStrategy,
155+
};

0 commit comments

Comments
 (0)