Skip to content

Commit 810ef7f

Browse files
leahwiczleahwicz
authored
Adding perf testing GHA (dbt-labs#5851)
* Adding perf testing GHA * Fixing tigger syntax * Fixing PR creation issue * Updating testing var * Remove unneeded branch names * Fixing branch naming convention * Standardizing branch name to var * Consolidating PR jobs * Updating inputs and making more readable * Splitting steps up * Making some updates here to simplify and update * Remove tab * Cleaned up testing TODOs before committing * Fixing spacing * Fixing spacing issue
1 parent fd73066 commit 810ef7f

File tree

1 file changed

+265
-0
lines changed

1 file changed

+265
-0
lines changed

‎.github/workflows/model_performance.yml

+265
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
# **what?**
2+
# This workflow models the performance characteristics of a point in time in dbt.
3+
# It runs specific dbt commands on committed projects multiple times to create and
4+
# commit information about the distribution to the current branch. For more information
5+
# see the readme in the performance module at /performance/README.md.
6+
#
7+
# **why?**
8+
# When developing new features, we can take quick performance samples and compare
9+
# them against the commited baseline measurements produced by this workflow to detect
10+
# some performance regressions at development time before they reach users.
11+
#
12+
# **when?**
13+
# This is only run once directly after each release (for non-prereleases). If for some
14+
# reason the results of a run are not satisfactory, it can also be triggered manually.
15+
16+
name: Model Performance Characteristics
17+
18+
on:
19+
# runs after non-prereleases are published.
20+
release:
21+
types: [released]
22+
# run manually from the actions tab
23+
workflow_dispatch:
24+
inputs:
25+
release_id:
26+
description: 'dbt version to model (must be non-prerelease in Pypi)'
27+
type: string
28+
required: true
29+
30+
env:
31+
RUNNER_CACHE_PATH: performance/runner/target/release/runner
32+
33+
# both jobs need to write
34+
permissions:
35+
contents: write
36+
pull-requests: write
37+
38+
jobs:
39+
set-variables:
40+
name: Setting Variables
41+
runs-on: ubuntu-latest
42+
outputs:
43+
cache_key: ${{ steps.variables.outputs.cache_key }}
44+
release_id: ${{ steps.semver.outputs.base-version }}
45+
release_branch: ${{ steps.variables.outputs.release_branch }}
46+
steps:
47+
48+
# explicitly checkout the performance runner from main regardless of which
49+
# version we are modeling.
50+
- name: Checkout
51+
uses: actions/checkout@v3
52+
with:
53+
ref: main
54+
55+
- name: Parse version into parts
56+
id: semver
57+
uses: dbt-labs/actions/parse-semver@v1
58+
with:
59+
version: ${{ github.event.inputs.release_id || github.event.release.tag_name }}
60+
61+
# collect all the variables that need to be used in subsequent jobs
62+
- name: Set variables
63+
id: variables
64+
run: |
65+
# create a cache key that will be used in the next job. without this the
66+
# next job would have to checkout from main and hash the files itself.
67+
echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT
68+
69+
branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest"
70+
echo "release_branch=$branch_name" >> $GITHUB_OUTPUT
71+
echo "release branch is inferred to be ${branch_name}"
72+
73+
latest-runner:
74+
name: Build or Fetch Runner
75+
runs-on: ubuntu-latest
76+
needs: [set-variables]
77+
env:
78+
RUSTFLAGS: "-D warnings"
79+
steps:
80+
- name: '[DEBUG] print variables'
81+
run: |
82+
echo "all variables defined in set-variables"
83+
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
84+
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
85+
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
86+
87+
# explicitly checkout the performance runner from main regardless of which
88+
# version we are modeling.
89+
- name: Checkout
90+
uses: actions/checkout@v3
91+
with:
92+
ref: main
93+
94+
# attempts to access a previously cached runner
95+
- uses: actions/cache@v3
96+
id: cache
97+
with:
98+
path: ${{ env.RUNNER_CACHE_PATH }}
99+
key: ${{ needs.set-variables.outputs.cache_key }}
100+
101+
- name: Fetch Rust Toolchain
102+
if: steps.cache.outputs.cache-hit != 'true'
103+
uses: actions-rs/toolchain@v1
104+
with:
105+
profile: minimal
106+
toolchain: stable
107+
override: true
108+
109+
- name: Add fmt
110+
if: steps.cache.outputs.cache-hit != 'true'
111+
run: rustup component add rustfmt
112+
113+
- name: Cargo fmt
114+
if: steps.cache.outputs.cache-hit != 'true'
115+
uses: actions-rs/cargo@v1
116+
with:
117+
command: fmt
118+
args: --manifest-path performance/runner/Cargo.toml --all -- --check
119+
120+
- name: Test
121+
if: steps.cache.outputs.cache-hit != 'true'
122+
uses: actions-rs/cargo@v1
123+
with:
124+
command: test
125+
args: --manifest-path performance/runner/Cargo.toml
126+
127+
- name: Build (optimized)
128+
if: steps.cache.outputs.cache-hit != 'true'
129+
uses: actions-rs/cargo@v1
130+
with:
131+
command: build
132+
args: --release --manifest-path performance/runner/Cargo.toml
133+
# the cache action automatically caches this binary at the end of the job
134+
135+
model:
136+
# depends on `latest-runner` as a separate job so that failures in this job do not prevent
137+
# a successfully tested and built binary from being cached.
138+
needs: [set-variables, latest-runner]
139+
name: Model a release
140+
runs-on: ubuntu-latest
141+
steps:
142+
143+
- name: '[DEBUG] print variables'
144+
run: |
145+
echo "all variables defined in set-variables"
146+
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
147+
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
148+
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
149+
150+
- name: Setup Python
151+
uses: actions/setup-python@v4
152+
with:
153+
python-version: "3.8"
154+
155+
- name: Install dbt
156+
run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }}
157+
158+
- name: Install Hyperfine
159+
run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
160+
161+
# explicitly checkout main to get the latest project definitions
162+
- name: Checkout
163+
uses: actions/checkout@v3
164+
with:
165+
ref: main
166+
167+
# this was built in the previous job so it will be there.
168+
- name: Fetch Runner
169+
uses: actions/cache@v3
170+
id: cache
171+
with:
172+
path: ${{ env.RUNNER_CACHE_PATH }}
173+
key: ${{ needs.set-variables.outputs.cache_key }}
174+
175+
- name: Move Runner
176+
run: mv performance/runner/target/release/runner performance/app
177+
178+
- name: Change Runner Permissions
179+
run: chmod +x ./performance/app
180+
181+
- name: '[DEBUG] ls baseline directory before run'
182+
run: ls -R performance/baselines/
183+
184+
# `${{ github.workspace }}` is used to pass the absolute path
185+
- name: Create directories
186+
run: |
187+
mkdir ${{ github.workspace }}/performance/tmp/
188+
mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/
189+
190+
# Run modeling with taking 20 samples
191+
- name: Run Measurement
192+
run: |
193+
performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 20
194+
195+
- name: '[DEBUG] ls baseline directory after run'
196+
run: ls -R performance/baselines/
197+
198+
- uses: actions/upload-artifact@v3
199+
with:
200+
name: baseline
201+
path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/
202+
203+
create-pr:
204+
name: Open PR for ${{ matrix.base-branch }}
205+
206+
# depends on `model` as a separate job so that the baseline can be committed to more than one branch
207+
# i.e. release branch and main
208+
needs: [set-variables, latest-runner, model]
209+
runs-on: ubuntu-latest
210+
211+
strategy:
212+
matrix:
213+
include:
214+
- base-branch: refs/heads/main
215+
target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
216+
- base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }}
217+
target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
218+
219+
steps:
220+
- name: '[DEBUG] print variables'
221+
run: |
222+
echo "all variables defined in set-variables"
223+
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
224+
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
225+
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
226+
227+
- name: Checkout
228+
uses: actions/checkout@v3
229+
with:
230+
ref: ${{ matrix.base-branch }}
231+
232+
- name: Create PR branch
233+
run: |
234+
git checkout -b ${{ matrix.target-branch }}
235+
git push origin ${{ matrix.target-branch }}
236+
git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }}
237+
238+
- uses: actions/download-artifact@v3
239+
with:
240+
name: baseline
241+
path: performance/baselines/${{ needs.set-variables.outputs.release_id }}
242+
243+
- name: '[DEBUG] ls baselines after artifact download'
244+
run: ls -R performance/baselines/
245+
246+
- name: Commit baseline
247+
uses: EndBug/add-and-commit@v9
248+
with:
249+
add: 'performance/baselines/*'
250+
author_name: 'Github Build Bot'
251+
author_email: 'buildbot@fishtownanalytics.com'
252+
message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}'
253+
push: 'origin origin/${{ matrix.target-branch }}'
254+
255+
- name: Create Pull Request
256+
uses: peter-evans/create-pull-request@v5
257+
with:
258+
author: 'Github Build Bot <buildbot@fishtownanalytics.com>'
259+
base: ${{ matrix.base-branch }}
260+
branch: '${{ matrix.target-branch }}'
261+
title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}'
262+
body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}'
263+
labels: |
264+
Skip Changelog
265+
Performance

0 commit comments

Comments
 (0)