Skip to content

Commit fea2a82

Browse files
ScottToddScottTodd
committed
[infra] Run parameterized ONNX model tests across CPU, Vulkan, and HIP.
1 parent e553425 commit fea2a82

File tree

4 files changed

+103
-5
lines changed

4 files changed

+103
-5
lines changed

‎.github/workflows/pkgci_test_onnx.yml

+16-5
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,22 @@ jobs:
130130
include:
131131
# CPU
132132
- name: cpu_llvm_task
133+
config-file: onnx_models_cpu_llvm_task.json
133134
runs-on: ubuntu-24.04
134135

135-
# TODO(scotttodd): test other backends (parameterize the test suite)
136+
# AMD GPU
137+
- name: amdgpu_rocm_rdna3
138+
config-file: onnx_models_gpu_rocm_rdna3.json
139+
runs-on: nodai-amdgpu-w7900-x86-64
140+
- name: amdgpu_vulkan
141+
config-file: onnx_models_gpu_vulkan.json
142+
runs-on: nodai-amdgpu-w7900-x86-64
143+
144+
# NVIDIA GPU
145+
# TODO(#18238): migrate to new runner cluster
136146
env:
137147
VENV_DIR: ${{ github.workspace }}/venv
148+
CONFIG_FILE_PATH: tests/external/iree-test-suites/onnx_models/${{ matrix.config-file }}
138149
steps:
139150
- name: Checking out IREE repository
140151
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -157,8 +168,8 @@ jobs:
157168
- name: Checkout test suites repository
158169
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
159170
with:
160-
repository: iree-org/iree-test-suites
161-
ref: 8e6af9e75d874ef8c9f8ff55f12cb38157dd55eb
171+
repository: ScottTodd/iree-test-suites # DO NOT SUBMIT - merge https://github.com/iree-org/iree-test-suites/pull/65
172+
ref: f0aa54520573bc0627a92676fce329c15966d093
162173
path: iree-test-suites
163174
- name: Install ONNX models test suite requirements
164175
run: |
@@ -170,6 +181,6 @@ jobs:
170181
pytest iree-test-suites/onnx_models/ \
171182
-rA \
172183
--log-cli-level=info \
173-
--override-ini=xfail_strict=false \
174184
--timeout=120 \
175-
--durations=0
185+
--durations=0 \
186+
--test-config-file=${CONFIG_FILE_PATH}

‎tests/external/iree-test-suites/onnx_models/onnx_models_cpu_llvm_task.json

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"config_name": "cpu_llvm_task",
3+
"iree_compile_flags": [
4+
"--iree-hal-target-backends=llvm-cpu",
5+
"--iree-llvmcpu-target-cpu=host"
6+
],
7+
"iree_run_module_flags": [
8+
"--device=local-task"
9+
],
10+
"tests_and_expected_outcomes": {
11+
"default": "skip",
12+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_age_gender_age_googlenet": "pass",
13+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_age_gender_gender_googlenet": "pass",
14+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
15+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
16+
"tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
17+
"tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
18+
"tests/model_zoo/validated/vision/classification_models_test.py::test_densenet_121": "pass",
19+
"tests/model_zoo/validated/vision/classification_models_test.py::test_googlenet": "pass",
20+
"tests/model_zoo/validated/vision/classification_models_test.py::test_inception_v2": "pass",
21+
"tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
22+
"tests/model_zoo/validated/vision/classification_models_test.py::test_mobilenet": "pass",
23+
"tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v1": "pass",
24+
"tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v2": "pass",
25+
"tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet": "pass",
26+
"tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
27+
"tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
28+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
29+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov2": "pass",
30+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
31+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov2_coco": "pass",
32+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass",
33+
"tests/model_zoo/validated/vision/super_resolution_models_test.py::test_fast_neural_style": "pass"
34+
}
35+
}

‎tests/external/iree-test-suites/onnx_models/onnx_models_gpu_rocm_rdna3.json

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"config_name": "gpu_rocm_rdna3",
3+
"iree_compile_flags": [
4+
"--iree-hal-target-backends=rocm",
5+
"--iree-hip-target=gfx1100"
6+
],
7+
"iree_run_module_flags": [
8+
"--device=hip"
9+
],
10+
"tests_and_expected_outcomes": {
11+
"default": "skip",
12+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
13+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
14+
"tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
15+
"tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
16+
"tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
17+
"tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet": "pass",
18+
"tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
19+
"tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
20+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
21+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
22+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass",
23+
"tests/model_zoo/validated/vision/super_resolution_models_test.py::test_fast_neural_style": "pass"
24+
}
25+
}

‎tests/external/iree-test-suites/onnx_models/onnx_models_gpu_vulkan.json

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"config_name": "gpu_vulkan",
3+
"iree_compile_flags": [
4+
"--iree-hal-target-backends=vulkan-spirv"
5+
],
6+
"iree_run_module_flags": [
7+
"--device=vulkan"
8+
],
9+
"tests_and_expected_outcomes": {
10+
"default": "skip",
11+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
12+
"tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
13+
"tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
14+
"tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
15+
"tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
16+
"tests/model_zoo/validated/vision/classification_models_test.py::test_mobilenet": "pass",
17+
"tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v1": "pass",
18+
"tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v2": "pass",
19+
"tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
20+
"tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
21+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
22+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov2": "pass",
23+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
24+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov2_coco": "pass",
25+
"tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass"
26+
}
27+
}

0 commit comments

Comments
 (0)