Skip to content

Commit 9e35755

Browse files
tlhuntertlhunter
committed
execute all sirun benchmarks in a docker container
1 parent 52db2cf commit 9e35755

File tree

29 files changed

+274
-50
lines changed

29 files changed

+274
-50
lines changed

‎.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
node_modules

‎.gitlab-ci.yml

+3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
stages:
2+
- benchmarks
23
- deploy
34

5+
include: ".gitlab/benchmarks.yml"
6+
47
.common: &common
58
tags: [ "runner:main", "size:large" ]
69

‎.gitlab/benchmarks.yml

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
variables:
2+
BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-js
3+
4+
.benchmarks:
5+
stage: benchmarks
6+
when: on_success
7+
tags: ["runner:apm-k8s-tweaked-metal"]
8+
image: $BASE_CI_IMAGE
9+
interruptible: true
10+
timeout: 1h
11+
script:
12+
- git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
13+
- export REPORTS_DIR="$(pwd)/reports/" && (mkdir "${REPORTS_DIR}" || :)
14+
- git clone --branch dd-trace-js https://github.com/DataDog/relenv-microbenchmarking-platform /platform && cd /platform
15+
- ./steps/install-benchmark-analyzer.sh
16+
- ./steps/install-pr-commenter.sh
17+
- ./steps/capture-hardware-software-info.sh
18+
- ./steps/run-benchmarks.sh
19+
- ./steps/analyze-results.sh
20+
- "./steps/upload-results-to-s3.sh || :"
21+
- "./steps/post-pr-comment.sh || :"
22+
artifacts:
23+
name: "reports"
24+
paths:
25+
- reports/
26+
expire_in: 3 months
27+
variables:
28+
UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
29+
UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
30+
UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
31+
UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA
32+
33+
KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-js
34+
FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
35+
36+
benchmark-v14:
37+
extends: .benchmarks
38+
variables:
39+
MAJOR_VERSION: 14
40+
41+
benchmark-v16:
42+
extends: .benchmarks
43+
variables:
44+
MAJOR_VERSION: 16
45+
46+
benchmark-v18:
47+
extends: .benchmarks
48+
variables:
49+
MAJOR_VERSION: 18

‎benchmark/sirun/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
results.ndjson
2+
meta-temp.json

‎benchmark/sirun/Dockerfile

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
FROM --platform=linux/amd64 debian:buster-slim
2+
3+
RUN apt-get update && apt-get install --no-install-recommends -y \
4+
wget curl ca-certificates valgrind \
5+
git hwinfo jq procps \
6+
software-properties-common build-essential libnss3-dev zlib1g-dev libgdbm-dev libncurses5-dev libssl-dev libffi-dev libreadline-dev libsqlite3-dev libbz2-dev
7+
8+
RUN git clone --depth 1 https://github.com/pyenv/pyenv.git --branch "v2.0.4" --single-branch /pyenv
9+
ENV PYENV_ROOT "/pyenv"
10+
ENV PATH "/pyenv/shims:/pyenv/bin:$PATH"
11+
RUN eval "$(pyenv init -)"
12+
RUN pyenv install 3.9.6 && pyenv global 3.9.6
13+
RUN pip3 install awscli virtualenv setuptools
14+
RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/etc/poetry python3 -
15+
16+
# nvm works much better with bash than sh
17+
SHELL ["/bin/bash", "--login", "-c"]
18+
19+
WORKDIR /app
20+
21+
ENV NVM_DIR /usr/local/nvm
22+
23+
RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v0.1.9/sirun-v0.1.9-x86_64-unknown-linux-gnu.tar.gz \
24+
&& tar -xzf sirun.tar.gz \
25+
&& rm sirun.tar.gz \
26+
&& mv sirun /usr/bin/sirun
27+
28+
RUN mkdir -p /usr/local/nvm \
29+
&& wget -q -O - https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash \
30+
&& . $NVM_DIR/nvm.sh \
31+
&& nvm install --no-progress 14.20.1 \
32+
&& nvm install --no-progress 16.17.1 \
33+
&& nvm install --no-progress 18.10.0 \
34+
&& nvm alias default 18 \
35+
&& nvm use 18

‎benchmark/sirun/README.md

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Benchmarks
2+
3+
This directory contains several different types of benchmarks.
4+
5+
Some of these benchmarks rely on [sirun](https://github.com/DataDog/sirun/) for execution.
6+
7+
## Running Benchmarks via Docker
8+
9+
Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun is performed automatically.
10+
11+
In order to run benchmarks using Docker, issue the following commands from the root of the project:
12+
13+
```sh
14+
# Build the Docker Image
15+
$ docker build -t dd-trace-benchmark -f benchmark/sirun/Dockerfile .
16+
17+
# Run the Docker Container
18+
$ docker run -it -v "$(pwd)":/app --platform=linux/amd64 dd-trace-benchmark bash
19+
cd /app/benchmark/sirun
20+
./runall.sh
21+
cat results.ndjson
22+
```
23+
24+
The `--platform` flag is required when running benchmarks on a non-x86 system, such as a macOS computer with the M1 chip.
25+
26+
`-v "$(pwd)":/app` mounts dd-trace-js source code root directory to /app inside container.

‎benchmark/sirun/appsec/common.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

33
module.exports = {
4-
port: 3030,
4+
port: 3032,
55
reqs: 1000
66
}

‎benchmark/sirun/appsec/meta.json

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
{
22
"name": "appsec",
3-
"cachegrind": true,
4-
"iterations": 10,
3+
"cachegrind": false,
4+
"iterations": 24,
55
"variants": {
66
"control": {
77
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
88
"run": "node server.js",
9+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"",
910
"env": {
1011
"DD_APPSEC_ENABLED": "0"
1112
}
1213
},
1314
"appsec-enabled": {
1415
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
1516
"run": "node server.js",
17+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"",
1618
"baseline": "control",
1719
"env": {
1820
"DD_APPSEC_ENABLED": "1"
@@ -21,6 +23,7 @@
2123
"control-with-attacks": {
2224
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
2325
"run": "node server.js",
26+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"",
2427
"env": {
2528
"DD_APPSEC_ENABLED": "0",
2629
"ATTACK_UA": "1",
@@ -31,6 +34,7 @@
3134
"appsec-enabled-with-attacks": {
3235
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
3336
"run": "node server.js",
37+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"",
3438
"baseline": "control-with-attacks",
3539
"env": {
3640
"DD_APPSEC_ENABLED": "1",

‎benchmark/sirun/async_hooks/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "async_hooks",
33
"run": "node -r ../monitor .",
4-
"cachegrind": true,
5-
"iterations": 5,
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node -r ../monitor .\"",
5+
"cachegrind": false,
6+
"iterations": 24,
67
"variants": {
78
"no-hooks": { "env": { "ASYNC_HOOKS": "" } },
89
"init-only": {

‎benchmark/sirun/encoding/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,6 @@ for (let parent = null, i = 0; i < 30; i++) {
4242

4343
const encoder = new AgentEncoder(writer)
4444

45-
for (let j = 0; j < 10000; j++) {
45+
for (let j = 0; j < 5000; j++) {
4646
encoder.encode(trace)
4747
}

‎benchmark/sirun/encoding/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "encoders",
33
"run": "node index.js",
4-
"cachegrind": true,
5-
"iterations": 10,
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
5+
"cachegrind": false,
6+
"iterations": 22,
67
"variants": {
78
"0.4": { "env": { "ENCODER_VERSION": "0.4" } },
89
"0.5": { "env": { "ENCODER_VERSION": "0.5" } }

‎benchmark/sirun/exporting-pipeline/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
"name": "exporting-pipeline",
33
"setup": "bash -c \"nohup node ../../e2e/fake-agent.js >/dev/null 2>&1 &\"",
44
"run": "node index.js",
5-
"iterations": 10,
6-
"cachegrind": true,
5+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
6+
"iterations": 22,
7+
"cachegrind": false,
78
"variants": {
89
"0.4": {
910
"env": {

‎benchmark/sirun/log/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "log",
33
"run": "node index.js",
4-
"cachegrind": true,
5-
"iterations": 10,
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
5+
"cachegrind": false,
6+
"iterations": 190,
67
"variants": {
78
"without-log": { "env": { "DD_TRACE_DEBUG": "false" } },
89
"skip-log": { "env": { "WITH_LEVEL": "debug", "DD_TRACE_LOG_LEVEL": "error" } },

‎benchmark/sirun/plugin-bluebird/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "plugin-bluebird",
33
"run": "node index.js",
4-
"cachegrind": true,
5-
"iterations": 10,
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
5+
"cachegrind": false,
6+
"iterations": 250,
67
"variants": {
78
"control": {
89
"env": {

‎benchmark/sirun/plugin-dns/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ if (Number(process.env.USE_TRACER)) {
77
const dns = require('dns')
88

99
function testRun (count) {
10-
if (++count === 10000) return
10+
if (++count === 1000) return
1111
dns.lookup('localhost', () => testRun(count))
1212
}
1313

‎benchmark/sirun/plugin-dns/meta.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "plugin-dns",
33
"run": "node index.js",
4-
"cachegrind": true,
5-
"iterations": 10,
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
5+
"cachegrind": false,
6+
"iterations": 40,
67
"variants": {
78
"control": {
89
"env": { "USE_TRACER": "0" }

‎benchmark/sirun/plugin-graphql/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,6 @@ const source = `
4747

4848
const variableValues = { who: 'world' }
4949

50-
for (let i = 0; i < 5; i++) {
50+
for (let i = 0; i < 6; i++) {
5151
graphql.graphql({ schema, source, variableValues })
5252
}

‎benchmark/sirun/plugin-graphql/meta.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
{
22
"name": "plugin-graphql",
33
"run": "node index.js",
4+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
45
"cachegrind": false,
5-
"iterations": 3,
6+
"iterations": 2,
67
"variants": {
78
"control": {},
89
"with-async-hooks": {
@@ -12,7 +13,8 @@
1213
"with-depth-off": {
1314
"baseline": "control",
1415
"env": { "WITH_TRACER": "1", "WITH_DEPTH": "0"}
15-
},"with-depth-on-max": {
16+
},
17+
"with-depth-on-max": {
1618
"baseline": "control",
1719
"env": { "WITH_TRACER": "1", "WITH_DEPTH": "4"}
1820
},

‎benchmark/sirun/plugin-http/common.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

33
module.exports = {
4-
port: 3030,
5-
reqs: 1000
4+
port: 3031,
5+
reqs: 700
66
}

‎benchmark/sirun/plugin-http/meta.json

+18-8
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,51 @@
11
{
22
"name": "plugin-http",
3-
"cachegrind": true,
4-
"iterations": 10,
3+
"cachegrind": false,
4+
"iterations": 20,
55
"variants": {
66
"client-control": {
77
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
8+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
89
"run": "node client.js",
10+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\"",
911
"env": {
1012
"CLIENT_USE_TRACER": "0"
1113
}
1214
},
1315
"client-with-tracer": {
1416
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
17+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
1518
"run": "node client.js",
19+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\"",
1620
"baseline": "client-control",
1721
"env": {
1822
"CLIENT_USE_TRACER": "1"
1923
}
2024
},
2125
"server-control": {
22-
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
23-
"run": "node server.js",
26+
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
27+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
28+
"run": "node client.js",
29+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\"",
2430
"env": {
2531
"SERVER_USE_TRACER": "0"
2632
}
2733
},
2834
"server-with-tracer": {
29-
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
30-
"run": "node server.js",
35+
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
36+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
37+
"run": "node client.js",
38+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\"",
3139
"baseline": "server-control",
3240
"env": {
3341
"SERVER_USE_TRACER": "1"
3442
}
3543
},
3644
"server-querystring-obfuscation": {
37-
"setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"",
38-
"run": "node server.js",
45+
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
46+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
47+
"run": "node client.js",
48+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\"",
3949
"baseline": "server-with-tracer",
4050
"env": {
4151
"SERVER_USE_TRACER": "1",

‎benchmark/sirun/plugin-net/common.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

33
module.exports = {
4-
port: 3030,
4+
port: 3033,
55
reqs: 1000
66
}

‎benchmark/sirun/plugin-net/meta.json

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
{
22
"name": "net",
3-
"cachegrind": true,
4-
"iterations": 10,
3+
"cachegrind": false,
4+
"iterations": 50,
55
"setup": "bash -c \"nohup node server.js >/dev/null 2>&1 &\"",
6+
"setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node server.js >/dev/null 2>&1 &\"",
67
"variants": {
78
"control": {
8-
"run": "node client.js"
9+
"run": "node client.js",
10+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node client.js\""
911
},
1012
"with-tracer": {
1113
"run": "node -r ../../../init.js client.js",
14+
"run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node -r ../../../init.js client.js\"",
1215
"baseline": "control"
1316
}
1417
}

0 commit comments

Comments
 (0)