From 6c7bfd3a7e9eb60f74f7c2d1ed1287452653f9c4 Mon Sep 17 00:00:00 2001
From: Rich Trott <rtrott@gmail.com>
Date: Thu, 8 Nov 2018 16:33:42 -0800
Subject: [PATCH] test: move benchmark tests out of main test suite
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Move benchmark tests (which are slow) out of the main test suite. We can
hopefully add them to node-daily-master so that they are still run daily
on CI.
PR-URL: https://github.com/nodejs/node/pull/24265
Reviewed-By: Richard Lau <riclau@uk.ibm.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
---
Makefile | 11 +++++---
test/README.md | 1 +
test/benchmark/benchmark.status | 21 ++++++++++++++
.../test-benchmark-assert.js | 0
.../test-benchmark-async-hooks.js | 0
.../test-benchmark-buffer.js | 0
.../test-benchmark-child-process.js | 0
.../test-benchmark-cluster.js | 0
.../test-benchmark-crypto.js | 0
.../test-benchmark-dgram.js | 0
.../test-benchmark-dns.js | 0
.../test-benchmark-domain.js | 0
.../test-benchmark-es.js | 0
.../test-benchmark-events.js | 0
.../test-benchmark-fs.js | 0
.../test-benchmark-http.js | 0
.../test-benchmark-misc.js | 0
.../test-benchmark-module.js | 0
.../test-benchmark-napi.js | 0
.../test-benchmark-net.js | 0
.../test-benchmark-os.js | 0
.../test-benchmark-path.js | 0
.../test-benchmark-process.js | 0
.../test-benchmark-querystring.js | 0
.../test-benchmark-streams.js | 0
.../test-benchmark-string_decoder.js | 0
.../test-benchmark-timers.js | 0
.../test-benchmark-tls.js | 0
.../test-benchmark-url.js | 0
.../test-benchmark-util.js | 0
.../test-benchmark-v8.js | 0
.../test-benchmark-vm.js | 0
.../test-benchmark-worker.js | 0
.../test-benchmark-zlib.js | 0
test/benchmark/testcfg.py | 6 ++++
test/root.status | 28 -------------------
tools/test.py | 1 +
vcbuild.bat | 3 +-
38 files changed, 38 insertions(+), 33 deletions(-)
create mode 100644 test/benchmark/benchmark.status
rename test/{parallel => benchmark}/test-benchmark-assert.js (100%)
rename test/{sequential => benchmark}/test-benchmark-async-hooks.js (100%)
rename test/{sequential => benchmark}/test-benchmark-buffer.js (100%)
rename test/{sequential => benchmark}/test-benchmark-child-process.js (100%)
rename test/{parallel => benchmark}/test-benchmark-cluster.js (100%)
rename test/{parallel => benchmark}/test-benchmark-crypto.js (100%)
rename test/{sequential => benchmark}/test-benchmark-dgram.js (100%)
rename test/{parallel => benchmark}/test-benchmark-dns.js (100%)
rename test/{parallel => benchmark}/test-benchmark-domain.js (100%)
rename test/{parallel => benchmark}/test-benchmark-es.js (100%)
rename test/{parallel => benchmark}/test-benchmark-events.js (100%)
rename test/{parallel => benchmark}/test-benchmark-fs.js (100%)
rename test/{sequential => benchmark}/test-benchmark-http.js (100%)
rename test/{parallel => benchmark}/test-benchmark-misc.js (100%)
rename test/{parallel => benchmark}/test-benchmark-module.js (100%)
rename test/{sequential => benchmark}/test-benchmark-napi.js (100%)
rename test/{sequential => benchmark}/test-benchmark-net.js (100%)
rename test/{parallel => benchmark}/test-benchmark-os.js (100%)
rename test/{sequential => benchmark}/test-benchmark-path.js (100%)
rename test/{parallel => benchmark}/test-benchmark-process.js (100%)
rename test/{parallel => benchmark}/test-benchmark-querystring.js (100%)
rename test/{parallel => benchmark}/test-benchmark-streams.js (100%)
rename test/{parallel => benchmark}/test-benchmark-string_decoder.js (100%)
rename test/{parallel => benchmark}/test-benchmark-timers.js (100%)
rename test/{sequential => benchmark}/test-benchmark-tls.js (100%)
rename test/{parallel => benchmark}/test-benchmark-url.js (100%)
rename test/{parallel => benchmark}/test-benchmark-util.js (100%)
rename test/{parallel => benchmark}/test-benchmark-v8.js (100%)
rename test/{parallel => benchmark}/test-benchmark-vm.js (100%)
rename test/{sequential => benchmark}/test-benchmark-worker.js (100%)
rename test/{parallel => benchmark}/test-benchmark-zlib.js (100%)
create mode 100644 test/benchmark/testcfg.py
diff --git a/Makefile b/Makefile
index b7a80f57450fe7..d24451f4289414 100644
--- a/Makefile
+++ b/Makefile
@@ -270,7 +270,7 @@ v8:
tools/make-v8.sh $(V8_ARCH).$(BUILDTYPE_LOWER) $(V8_BUILD_OPTIONS)
.PHONY: jstest
-jstest: build-addons build-addons-napi bench-addons-build ## Runs addon tests and JS tests
+jstest: build-addons build-addons-napi ## Runs addon tests and JS tests
$(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) \
--skip-tests=$(CI_SKIP_TESTS) \
$(CI_JS_SUITES) \
@@ -414,7 +414,7 @@ clear-stalled:
echo $${PS_OUT} | xargs kill -9; \
fi
-test-build: | all build-addons build-addons-napi bench-addons-build
+test-build: | all build-addons build-addons-napi
test-build-addons-napi: all build-addons-napi
@@ -455,7 +455,7 @@ test-ci-js: | clear-stalled
.PHONY: test-ci
# Related CI jobs: most CI tests, excluding node-test-commit-arm-fanned
test-ci: LOGLEVEL := info
-test-ci: | clear-stalled build-addons build-addons-napi doc-only bench-addons-build
+test-ci: | clear-stalled build-addons build-addons-napi doc-only
out/Release/cctest --gtest_output=tap:cctest.tap
$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
--mode=$(BUILDTYPE_LOWER) --flaky-tests=$(FLAKY_TESTS) \
@@ -496,7 +496,7 @@ test-debug: test-build
test-message: test-build
$(PYTHON) tools/test.py $(PARALLEL_ARGS) message
-test-simple: | cctest bench-addons-build # Depends on 'all'.
+test-simple: | cctest # Depends on 'all'.
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
test-pummel: all
@@ -509,6 +509,9 @@ test-node-inspect: $(NODE_EXE)
USE_EMBEDDED_NODE_INSPECT=1 $(NODE) tools/test-npm-package \
--install deps/node-inspect test
+test-benchmark: | bench-addons-build
+ $(PYTHON) tools/test.py $(PARALLEL_ARGS) benchmark
+
test-tick-processor: all
$(PYTHON) tools/test.py $(PARALLEL_ARGS) tick-processor
diff --git a/test/README.md b/test/README.md
index 7ef705230983f3..25243d7687d133 100644
--- a/test/README.md
+++ b/test/README.md
@@ -19,6 +19,7 @@ GitHub with the `autocrlf` git config flag set to true.
| `addons` | Yes | Tests for [addon](https://nodejs.org/api/addons.html) functionality along with some tests that require an addon to function properly. |
| `addons-napi` | Yes | Tests for [n-api](https://nodejs.org/api/n-api.html) functionality. |
| `async-hooks` | Yes | Tests for [async_hooks](https://nodejs.org/api/async_hooks.html) functionality. |
+| `benchmark` | No | Test minimal functionality of benchmarks. |
| `cctest` | Yes | C++ tests that are run as part of the build process. |
| `code-cache` | No | Tests for a Node.js binary compiled with V8 code cache. |
| `common` | | Common modules shared among many tests. [Documentation](./common/README.md) |
diff --git a/test/benchmark/benchmark.status b/test/benchmark/benchmark.status
new file mode 100644
index 00000000000000..6a966743aab26b
--- /dev/null
+++ b/test/benchmark/benchmark.status
@@ -0,0 +1,21 @@
+prefix benchmark
+
+# To mark a test as flaky, list the test name in the appropriate section
+# below, without ".js", followed by ": PASS,FLAKY". Example:
+# sample-test : PASS,FLAKY
+
+[true] # This section applies to all platforms
+
+[$system==win32]
+
+[$system==linux]
+
+[$system==macos]
+
+[$system==solaris] # Also applies to SmartOS
+
+[$system==freebsd]
+
+[$system==aix]
+
+[$arch==arm]
diff --git a/test/parallel/test-benchmark-assert.js b/test/benchmark/test-benchmark-assert.js
similarity index 100%
rename from test/parallel/test-benchmark-assert.js
rename to test/benchmark/test-benchmark-assert.js
diff --git a/test/sequential/test-benchmark-async-hooks.js b/test/benchmark/test-benchmark-async-hooks.js
similarity index 100%
rename from test/sequential/test-benchmark-async-hooks.js
rename to test/benchmark/test-benchmark-async-hooks.js
diff --git a/test/sequential/test-benchmark-buffer.js b/test/benchmark/test-benchmark-buffer.js
similarity index 100%
rename from test/sequential/test-benchmark-buffer.js
rename to test/benchmark/test-benchmark-buffer.js
diff --git a/test/sequential/test-benchmark-child-process.js b/test/benchmark/test-benchmark-child-process.js
similarity index 100%
rename from test/sequential/test-benchmark-child-process.js
rename to test/benchmark/test-benchmark-child-process.js
diff --git a/test/parallel/test-benchmark-cluster.js b/test/benchmark/test-benchmark-cluster.js
similarity index 100%
rename from test/parallel/test-benchmark-cluster.js
rename to test/benchmark/test-benchmark-cluster.js
diff --git a/test/parallel/test-benchmark-crypto.js b/test/benchmark/test-benchmark-crypto.js
similarity index 100%
rename from test/parallel/test-benchmark-crypto.js
rename to test/benchmark/test-benchmark-crypto.js
diff --git a/test/sequential/test-benchmark-dgram.js b/test/benchmark/test-benchmark-dgram.js
similarity index 100%
rename from test/sequential/test-benchmark-dgram.js
rename to test/benchmark/test-benchmark-dgram.js
diff --git a/test/parallel/test-benchmark-dns.js b/test/benchmark/test-benchmark-dns.js
similarity index 100%
rename from test/parallel/test-benchmark-dns.js
rename to test/benchmark/test-benchmark-dns.js
diff --git a/test/parallel/test-benchmark-domain.js b/test/benchmark/test-benchmark-domain.js
similarity index 100%
rename from test/parallel/test-benchmark-domain.js
rename to test/benchmark/test-benchmark-domain.js
diff --git a/test/parallel/test-benchmark-es.js b/test/benchmark/test-benchmark-es.js
similarity index 100%
rename from test/parallel/test-benchmark-es.js
rename to test/benchmark/test-benchmark-es.js
diff --git a/test/parallel/test-benchmark-events.js b/test/benchmark/test-benchmark-events.js
similarity index 100%
rename from test/parallel/test-benchmark-events.js
rename to test/benchmark/test-benchmark-events.js
diff --git a/test/parallel/test-benchmark-fs.js b/test/benchmark/test-benchmark-fs.js
similarity index 100%
rename from test/parallel/test-benchmark-fs.js
rename to test/benchmark/test-benchmark-fs.js
diff --git a/test/sequential/test-benchmark-http.js b/test/benchmark/test-benchmark-http.js
similarity index 100%
rename from test/sequential/test-benchmark-http.js
rename to test/benchmark/test-benchmark-http.js
diff --git a/test/parallel/test-benchmark-misc.js b/test/benchmark/test-benchmark-misc.js
similarity index 100%
rename from test/parallel/test-benchmark-misc.js
rename to test/benchmark/test-benchmark-misc.js
diff --git a/test/parallel/test-benchmark-module.js b/test/benchmark/test-benchmark-module.js
similarity index 100%
rename from test/parallel/test-benchmark-module.js
rename to test/benchmark/test-benchmark-module.js
diff --git a/test/sequential/test-benchmark-napi.js b/test/benchmark/test-benchmark-napi.js
similarity index 100%
rename from test/sequential/test-benchmark-napi.js
rename to test/benchmark/test-benchmark-napi.js
diff --git a/test/sequential/test-benchmark-net.js b/test/benchmark/test-benchmark-net.js
similarity index 100%
rename from test/sequential/test-benchmark-net.js
rename to test/benchmark/test-benchmark-net.js
diff --git a/test/parallel/test-benchmark-os.js b/test/benchmark/test-benchmark-os.js
similarity index 100%
rename from test/parallel/test-benchmark-os.js
rename to test/benchmark/test-benchmark-os.js
diff --git a/test/sequential/test-benchmark-path.js b/test/benchmark/test-benchmark-path.js
similarity index 100%
rename from test/sequential/test-benchmark-path.js
rename to test/benchmark/test-benchmark-path.js
diff --git a/test/parallel/test-benchmark-process.js b/test/benchmark/test-benchmark-process.js
similarity index 100%
rename from test/parallel/test-benchmark-process.js
rename to test/benchmark/test-benchmark-process.js
diff --git a/test/parallel/test-benchmark-querystring.js b/test/benchmark/test-benchmark-querystring.js
similarity index 100%
rename from test/parallel/test-benchmark-querystring.js
rename to test/benchmark/test-benchmark-querystring.js
diff --git a/test/parallel/test-benchmark-streams.js b/test/benchmark/test-benchmark-streams.js
similarity index 100%
rename from test/parallel/test-benchmark-streams.js
rename to test/benchmark/test-benchmark-streams.js
diff --git a/test/parallel/test-benchmark-string_decoder.js b/test/benchmark/test-benchmark-string_decoder.js
similarity index 100%
rename from test/parallel/test-benchmark-string_decoder.js
rename to test/benchmark/test-benchmark-string_decoder.js
diff --git a/test/parallel/test-benchmark-timers.js b/test/benchmark/test-benchmark-timers.js
similarity index 100%
rename from test/parallel/test-benchmark-timers.js
rename to test/benchmark/test-benchmark-timers.js
diff --git a/test/sequential/test-benchmark-tls.js b/test/benchmark/test-benchmark-tls.js
similarity index 100%
rename from test/sequential/test-benchmark-tls.js
rename to test/benchmark/test-benchmark-tls.js
diff --git a/test/parallel/test-benchmark-url.js b/test/benchmark/test-benchmark-url.js
similarity index 100%
rename from test/parallel/test-benchmark-url.js
rename to test/benchmark/test-benchmark-url.js
diff --git a/test/parallel/test-benchmark-util.js b/test/benchmark/test-benchmark-util.js
similarity index 100%
rename from test/parallel/test-benchmark-util.js
rename to test/benchmark/test-benchmark-util.js
diff --git a/test/parallel/test-benchmark-v8.js b/test/benchmark/test-benchmark-v8.js
similarity index 100%
rename from test/parallel/test-benchmark-v8.js
rename to test/benchmark/test-benchmark-v8.js
diff --git a/test/parallel/test-benchmark-vm.js b/test/benchmark/test-benchmark-vm.js
similarity index 100%
rename from test/parallel/test-benchmark-vm.js
rename to test/benchmark/test-benchmark-vm.js
diff --git a/test/sequential/test-benchmark-worker.js b/test/benchmark/test-benchmark-worker.js
similarity index 100%
rename from test/sequential/test-benchmark-worker.js
rename to test/benchmark/test-benchmark-worker.js
diff --git a/test/parallel/test-benchmark-zlib.js b/test/benchmark/test-benchmark-zlib.js
similarity index 100%
rename from test/parallel/test-benchmark-zlib.js
rename to test/benchmark/test-benchmark-zlib.js
diff --git a/test/benchmark/testcfg.py b/test/benchmark/testcfg.py
new file mode 100644
index 00000000000000..2c2929f610b851
--- /dev/null
+++ b/test/benchmark/testcfg.py
@@ -0,0 +1,6 @@
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+ return testpy.SimpleTestConfiguration(context, root, 'benchmark')
diff --git a/test/root.status b/test/root.status
index 9ed9004c2169d0..9c40512cb68051 100644
--- a/test/root.status
+++ b/test/root.status
@@ -9,27 +9,6 @@ async-hooks/test-tlswrap: SLOW
async-hooks/test-tlswrap: SLOW
message/eval_messages: SLOW
message/stdin_messages: SLOW
-parallel/test-benchmark-assert: SLOW
-parallel/test-benchmark-cluster: SLOW
-parallel/test-benchmark-crypto: SLOW
-parallel/test-benchmark-dns: SLOW
-parallel/test-benchmark-domain: SLOW
-parallel/test-benchmark-es: SLOW
-parallel/test-benchmark-events: SLOW
-parallel/test-benchmark-fs: SLOW
-parallel/test-benchmark-misc: SLOW
-parallel/test-benchmark-module: SLOW
-parallel/test-benchmark-os: SLOW
-parallel/test-benchmark-process: SLOW
-parallel/test-benchmark-querystring: SLOW
-parallel/test-benchmark-streams: SLOW
-parallel/test-benchmark-string_decoder: SLOW
-parallel/test-benchmark-timers: SLOW
-parallel/test-benchmark-url: SLOW
-parallel/test-benchmark-util: SLOW
-parallel/test-benchmark-v8: SLOW
-parallel/test-benchmark-vm: SLOW
-parallel/test-benchmark-zlib: SLOW
parallel/test-buffer-constructor-node-modules-paths: SLOW
parallel/test-buffer-indexof: SLOW
parallel/test-child-process-spawnsync-input: SLOW
@@ -165,13 +144,6 @@ parallel/test-worker-unsupported-things: SLOW
parallel/test-worker-workerdata-sharedarraybuffer: SLOW
parallel/test-zlib-bytes-read: SLOW
parallel/test-zlib-convenience-methods: SLOW
-sequential/test-benchmark-buffer: SLOW
-sequential/test-benchmark-child-process: SLOW
-sequential/test-benchmark-dgram: SLOW
-sequential/test-benchmark-http: SLOW
-sequential/test-benchmark-net: SLOW
-sequential/test-benchmark-path: SLOW
-sequential/test-benchmark-tls: SLOW
sequential/test-child-process-execsync: SLOW
sequential/test-child-process-exit: SLOW
sequential/test-child-process-pass-fd: SLOW
diff --git a/tools/test.py b/tools/test.py
index cd361196653043..3a464be61da1b3 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -1498,6 +1498,7 @@ def PrintCrashed(code):
IGNORED_SUITES = [
'addons',
'addons-napi',
+ 'benchmark',
'doctool',
'internet',
'pummel',
diff --git a/vcbuild.bat b/vcbuild.bat
index 8627dcb296fba7..81dd6087882d32 100644
--- a/vcbuild.bat
+++ b/vcbuild.bat
@@ -85,6 +85,7 @@ if /i "%1"=="build-addons" set build_addons=1&goto arg-ok
if /i "%1"=="build-addons-napi" set build_addons_napi=1&goto arg-ok
if /i "%1"=="test-addons" set test_args=%test_args% addons&set build_addons=1&goto arg-ok
if /i "%1"=="test-addons-napi" set test_args=%test_args% addons-napi&set build_addons_napi=1&goto arg-ok
+if /i "%1"=="test-benchmark" set test_args=%test_args% benchmark&goto arg-ok
if /i "%1"=="test-simple" set test_args=%test_args% sequential parallel -J&goto arg-ok
if /i "%1"=="test-message" set test_args=%test_args% message&goto arg-ok
if /i "%1"=="test-tick-processor" set test_args=%test_args% tick-processor&goto arg-ok
@@ -632,7 +633,7 @@ del .used_configure_flags
goto exit
:help
-echo vcbuild.bat [debug/release] [msi] [doc] [test/test-ci/test-all/test-addons/test-addons-napi/test-internet/test-pummel/test-simple/test-message/test-tick-processor/test-known-issues/test-node-inspect/test-check-deopts/test-npm/test-async-hooks/test-v8/test-v8-intl/test-v8-benchmarks/test-v8-all] [ignore-flaky] [static/dll] [noprojgen] [projgen] [small-icu/full-icu/without-intl] [nobuild] [nosnapshot] [noetw] [noperfctr] [ltcg] [nopch] [licensetf] [sign] [ia32/x86/x64] [vs2017] [download-all] [enable-vtune] [lint/lint-ci/lint-js/lint-js-ci/lint-md] [lint-md-build] [package] [build-release] [upload] [no-NODE-OPTIONS] [link-module path-to-module] [debug-http2] [debug-nghttp2] [clean] [no-cctest] [openssl-no-asm]
+echo vcbuild.bat [debug/release] [msi] [doc] [test/test-ci/test-all/test-addons/test-addons-napi/test-benchmark/test-internet/test-pummel/test-simple/test-message/test-tick-processor/test-known-issues/test-node-inspect/test-check-deopts/test-npm/test-async-hooks/test-v8/test-v8-intl/test-v8-benchmarks/test-v8-all] [ignore-flaky] [static/dll] [noprojgen] [projgen] [small-icu/full-icu/without-intl] [nobuild] [nosnapshot] [noetw] [noperfctr] [ltcg] [nopch] [licensetf] [sign] [ia32/x86/x64] [vs2017] [download-all] [enable-vtune] [lint/lint-ci/lint-js/lint-js-ci/lint-md] [lint-md-build] [package] [build-release] [upload] [no-NODE-OPTIONS] [link-module path-to-module] [debug-http2] [debug-nghttp2] [clean] [no-cctest] [openssl-no-asm]
echo Examples:
echo vcbuild.bat : builds release build
echo vcbuild.bat debug : builds debug build