test_runner: report coverage thresholds in test:coverage

Author: avivkeller

doc/api/test.md

@@ -2814,6 +2814,11 @@ are defined, while others are emitted in the order that the tests execute.
         numbers and the number of times they were covered.
         * `line` {number} The line number.
         * `count` {number} The number of times the line was covered.
+    * `thresholds` {Object} An object containing whether or not the coverage for
+      each coverage type.
+      * `function` {boolean} Whether or not the coverage surpassed the `function` threshold.
+      * `branch` {boolean} Whether or not the coverage surpassed the `branch` threshold.
+      * `line` {boolean} Whether or not the coverage surpassed the `line` threshold.
     * `totals` {Object} An object containing a summary of coverage for all
       files.
       * `totalLineCount` {number} The total number of lines.

lib/internal/main/test_runner.js

@@ -31,8 +31,13 @@ if (isUsingInspector() && options.isolation === 'process') {
 options.globPatterns = ArrayPrototypeSlice(process.argv, 1);
 
 debug('test runner configuration:', options);
-run(options).on('test:fail', (data) => {
+const ran = run(options);
+ran.on('test:fail', (data) => {
   if (data.todo === undefined || data.todo === false) {
     process.exitCode = kGenericUserError;
   }
 });
+
+ran.on('test:coverage', ({ summary: { thresholds: { line, branch, function: func } } }) => {
+  if (!line || !branch || !func) process.exitCode = kGenericUserError;
+});

lib/internal/test_runner/harness.js

@@ -250,6 +250,9 @@ function lazyBootstrapRoot() {
         process.exitCode = kGenericUserError;
       }
     });
+    // globalRoot.reporter.on('test:coverage', ({ summary: { thresholds: { line, branch, function: func }}}) => {
+    //   if (!line || !branch || !func) process.exitCode = kGenericUserError;
+    // });
     globalRoot.harness.bootstrapPromise = globalOptions.setup(globalRoot.reporter);
   }
   return globalRoot;

lib/internal/test_runner/test.js

@@ -29,7 +29,6 @@ const {
   SymbolDispose,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
-const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { addAbortListener } = require('internal/events/abort_listener');
 const { queueMicrotask } = require('internal/process/task_queues');
 const { AsyncResource } = require('async_hooks');
@@ -1010,8 +1009,12 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
-        reporter.coverage(nesting, loc, coverage);
-
+        coverage.thresholds = {
+          __proto__: null,
+          line: true,
+          branch: true,
+          function: true,
+        };
         const coverages = [
           { __proto__: null, actual: coverage.totals.coveredLinePercent,
             threshold: this.config.lineCoverage, name: 'line' },
@@ -1026,10 +1029,12 @@ class Test extends AsyncResource {
         for (let i = 0; i < coverages.length; i++) {
           const { threshold, actual, name } = coverages[i];
           if (actual < threshold) {
-            process.exitCode = kGenericUserError;
+            coverage.thresholds[name] = false;
             reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
           }
         }
+
+        reporter.coverage(nesting, loc, coverage);
       }
 
       if (harness.watching) {

test/parallel/test-runner-coverage-thresholds.js

@@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() {
 }
 
 const fixture = fixtures.path('test-runner', 'coverage.js');
-const neededArguments = [
-  '--experimental-test-coverage',
-  '--test-reporter', 'tap',
-];
+const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs');
 
 const coverages = [
   { flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
@@ -56,10 +53,12 @@ const coverages = [
 ];
 
 for (const coverage of coverages) {
-  test(`test passing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=25`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -70,10 +69,27 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test failing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=25`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert(stdout.summary.thresholds[coverage.name]);
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test failing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=99`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -84,9 +100,25 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
+  test(`test failing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=99`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert(!stdout.summary.thresholds[coverage.name]);
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test out-of-range ${coverage.flag} (too high)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=101`,
       fixture,
     ]);
@@ -96,9 +128,10 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
+  test(`test out-of-range ${coverage.flag} (too low)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=-1`,
       fixture,
     ]);

test/parallel/test-runner-coverage.js

@@ -253,6 +253,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as
     const calledTwice = file.functions.find((f) => f.name === 'fnWithControlFlow');
     assert.strictEqual(calledTwice.count, 2);
     assert.strictEqual(calledTwice.line, 35);
+
+    assert.ok(coverage.summary.thresholds.function);
   });
 
   await t.test('reports on branch coverage', () => {
@@ -261,6 +263,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as
 
     const calledTwice = file.branches.find((b) => b.line === 35);
     assert.strictEqual(calledTwice.count, 2);
+
+    assert.ok(coverage.summary.thresholds.branch);
   });
 
   await t.test('reports on line coverage', () => {
@@ -278,6 +282,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as
       const testLine = file.lines.find((l) => l.line === line.line);
       assert.strictEqual(testLine.count, line.count);
     });
+
+    assert.ok(coverage.summary.thresholds.line);
   });
 });