Skip to content

Commit 64d8468

Browse files
[ci] retry examples tests on connection errors (#3319)
### Changes Retry run example on connection errors ### Tests https://github.com/openvinotoolkit/nncf/actions/runs/13585977596
1 parent 8115aed commit 64d8468

File tree

1 file changed

+32
-2
lines changed

1 file changed

+32
-2
lines changed

tests/cross_fw/examples/test_examples.py

+32-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import os
1313
import subprocess
1414
import sys
15+
import time
1516
from pathlib import Path
1617
from typing import Any, Dict, List
1718

@@ -38,13 +39,30 @@
3839
MODEL_SIZE_METRICS = "model_size_metrics"
3940
PERFORMANCE_METRICS = "performance_metrics"
4041

42+
NUM_RETRY_ON_CONNECTION_ERROR = 2
43+
RETRY_TIMEOUT = 60
44+
4145

4246
def example_test_cases():
4347
example_scope = load_json(EXAMPLE_SCOPE_PATH)
4448
for example_name, example_params in example_scope.items():
4549
yield pytest.param(example_name, example_params, id=example_name)
4650

4751

52+
def _is_connection_error(txt: str) -> bool:
53+
error_list = [
54+
"ReadTimeoutError",
55+
"HTTPError",
56+
"URL fetch failure",
57+
]
58+
for line in txt.split()[::-1]:
59+
if any(e in line for e in error_list):
60+
print("-------------------------------")
61+
print(f"Detect connection error: {line}")
62+
return True
63+
return False
64+
65+
4866
@pytest.mark.parametrize("example_name, example_params", example_test_cases())
4967
def test_examples(
5068
tmp_path: Path,
@@ -93,8 +111,20 @@ def test_examples(
93111
run_cmd_line = f"{python_executable_with_venv} {run_example_py} --name {example_name} --output {metrics_file_path}"
94112
if data is not None:
95113
run_cmd_line += f" --data {data}"
96-
cmd = Command(run_cmd_line, cwd=PROJECT_ROOT, env=env)
97-
cmd.run()
114+
115+
retry_count = 0
116+
while True:
117+
cmd = Command(run_cmd_line, cwd=PROJECT_ROOT, env=env)
118+
try:
119+
ret = cmd.run()
120+
if ret == 0:
121+
break
122+
except Exception as e:
123+
if retry_count >= NUM_RETRY_ON_CONNECTION_ERROR or not _is_connection_error(str(e)):
124+
raise e
125+
retry_count += 1
126+
print(f"Retry {retry_count} after {RETRY_TIMEOUT} seconds")
127+
time.sleep(RETRY_TIMEOUT)
98128

99129
measured_metrics = load_json(metrics_file_path)
100130
print(measured_metrics)

0 commit comments

Comments
 (0)