Skip to content

Commit 1d7ab84

Browse files
iagaponenkoiagaponenko
committedFeb 4, 2025·
Merge branch 'tickets/DM-45546'
2 parents d20cd33 + bb2cacf commit 1d7ab84

File tree

13 files changed

+2108
-32
lines changed

13 files changed

+2108
-32
lines changed
 

‎.github/workflows/ci.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
- name: Launch unit tests
3535
run: |
3636
./code-checks.sh -u
37-
- uses: actions/upload-artifact@v2
37+
- uses: actions/upload-artifact@v4
3838
with:
3939
name: image
4040
path: artifacts
@@ -44,7 +44,7 @@ jobs:
4444
needs: build
4545
steps:
4646
- name: Download image
47-
uses: actions/download-artifact@v3
47+
uses: actions/download-artifact@v4
4848
with:
4949
name: image
5050
path: artifacts
@@ -75,7 +75,7 @@ jobs:
7575
sudo apt-get install apparmor-profiles
7676
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
7777
- name: Download qserv-ingest image
78-
uses: actions/download-artifact@v3
78+
uses: actions/download-artifact@v4
7979
with:
8080
name: image
8181
path: artifacts
@@ -117,7 +117,7 @@ jobs:
117117
needs: integration
118118
steps:
119119
- name: Download qserv-ingest image
120-
uses: actions/download-artifact@v3
120+
uses: actions/download-artifact@v4
121121
with:
122122
name: image
123123
path: artifacts

‎env.example.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Set variable below to empty string in order to install
22
# current development version
3-
INGEST_RELEASE='2024.4.1-rc1'
3+
INGEST_RELEASE='2024.8.1-rc4'
44
TAG=${INGEST_RELEASE:-$(git -C $DIR describe --dirty --always)}
55

66
# Select dataset to load

‎manifests/test/configmap/dbbench.ini

+1,000
Large diffs are not rendered by default.

‎manifests/test/configmap/ingest.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
version: 15
2+
ingest:
3+
http:
4+
# Timeout for POST and PUT queries in seconds
5+
write_timeout: 7200
6+
# Timeout for READ queries in seconds
7+
read_timeout: 7200
8+
input:
9+
# Servers which provides input data
10+
# TODO Add support for webdav protocol
11+
# Use file:// as first element in list when using local data
12+
servers:
13+
- http://wsa.roe.ac.uk
14+
# Path on server where input data is available
15+
path: gaiaFull/
16+
qserv:
17+
18+
# URL of Qserv services
19+
# Proxy URL
20+
query_url: "mysql://qsmaster:@qserv-czar:4040"
21+
# Ingest database URL
22+
queue_url: "mysql://qsingest:@qserv-ingest-db-0.qserv-ingest-db/qservIngest"
23+
# Replication service URL
24+
replication_url: http://qserv-repl-ctl-0.qserv-repl-ctl:8080
25+
ingestservice:
26+
auto_build_secondary_index: 1
27+
async_proc_limit: 4
28+
low_speed_limit: 10
29+
low_speed_time: 3600

‎manifests/test/configmap/kustomization.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
bases:
2+
- ../../base/configmap
3+
4+
configMapGenerator:
5+
- name: config-ingest
6+
behavior: replace
7+
files:
8+
- ingest.yaml
9+
- dbbench.ini
10+
generatorOptions:
11+
labels:
12+
kustomize.generated.resources: qserv.ingest
13+
disableNameSuffixHash: true

‎manifests/test/uk-dac-gaiafull/configmap/dbbench.ini

+1,000
Large diffs are not rendered by default.

‎manifests/test/uk-dac-gaiafull/configmap/ingest.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
version: 15
2+
ingest:
3+
http:
4+
# Timeout for POST and PUT queries in seconds
5+
write_timeout: 7200
6+
# Timeout for READ queries in seconds
7+
read_timeout: 7200
8+
input:
9+
# Servers which provides input data
10+
# TODO Add support for webdav protocol
11+
# Use file:// as first element in list when using local data
12+
servers:
13+
- http://wsa.roe.ac.uk
14+
# Path on server where input data is available
15+
path: gaiaFull/
16+
qserv:
17+
18+
# URL of Qserv services
19+
# Proxy URL
20+
query_url: "mysql://qsmaster:@qserv-czar:4040"
21+
# Ingest database URL
22+
queue_url: "mysql://qsingest:@qserv-ingest-db-0.qserv-ingest-db/qservIngest"
23+
# Replication service URL
24+
replication_url: http://qserv-repl-ctl-0.qserv-repl-ctl:8080
25+
ingestservice:
26+
auto_build_secondary_index: 1
27+
async_proc_limit: 4
28+
low_speed_limit: 10
29+
low_speed_time: 3600

‎manifests/test/uk-dac-gaiafull/configmap/kustomization.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
bases:
2+
- ../../base/configmap
3+
4+
configMapGenerator:
5+
- name: config-ingest
6+
behavior: replace
7+
files:
8+
- ingest.yaml
9+
- dbbench.ini
10+
generatorOptions:
11+
labels:
12+
kustomize.generated.resources: qserv.ingest
13+
disableNameSuffixHash: true

‎rootfs/ingest/python/qserv/http.py

+3
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ def __init__(
152152

153153
def is_reachable(self, url: str) -> bool:
154154
"""Check if a given http URL is reachable through the network."""
155+
_LOG.debug("Checking if %s is reachable", url)
155156
try:
156157
self.http.head(url)
157158
except requests.exceptions.ConnectionError as e:
@@ -230,6 +231,8 @@ def post(
230231
"""
231232
if payload is None:
232233
payload = dict()
234+
# Set version if it does not exists
235+
payload["version"] = payload.get("version", version.REPL_SERVICE_VERSION)
233236
if auth is True:
234237
payload["auth_key"] = self.authKey
235238
timeouts: Union[float, Tuple[float, float], Tuple[float, None]]

‎rootfs/ingest/python/qserv/ingestconfig.py

+3-6
Original file line numberDiff line numberDiff line change
@@ -112,19 +112,16 @@ def _check_version(self, yaml: dict) -> None:
112112
if "version" in yaml:
113113
fileversion = yaml["version"]
114114

115-
if fileversion is None or not (
116-
_MIN_SUPPORTED_VERSION <= fileversion <= version.INGEST_CONFIG_VERSION
117-
):
115+
if fileversion is None or fileversion < _MIN_SUPPORTED_VERSION:
118116
_LOG.critical(
119117
"The ingest configuration file (ingest.yaml) version "
120118
"is not in the range supported by qserv-ingest "
121-
"(is %s, expected between %s and %s)",
119+
"(is %s, expected at least %s)",
122120
fileversion,
123121
_MIN_SUPPORTED_VERSION,
124-
version.REPL_SERVICE_VERSION,
125122
)
126123
sys.exit(1)
127-
_LOG.info("Ingest configuration file version: %s", version.REPL_SERVICE_VERSION)
124+
_LOG.info("Ingest configuration file version: %s", fileversion)
128125

129126

130127
@dataclass

‎rootfs/ingest/python/qserv/metadata.py

+4-6
Original file line numberDiff line numberDiff line change
@@ -330,23 +330,21 @@ def json_db(self) -> Dict[str, Any]:
330330
return self._json_db
331331

332332
def _check_version(self) -> None:
333-
"""Check metadata file version
334-
and exit if its value is not supported"""
333+
"""Check metadata file version and exit if its value is not supported"""
335334
fileversion = None
336335
if "version" in self.metadata:
337336
fileversion = self.metadata["version"]
338337

339-
if fileversion is None or not (_MIN_SUPPORTED_VERSION <= fileversion <= version.REPL_SERVICE_VERSION):
338+
if fileversion is None or fileversion < _MIN_SUPPORTED_VERSION:
340339
_LOG.critical(
341340
"The metadata file (%s) version is not in the range supported by qserv-ingest "
342-
"(is %s, expected between %s and %s)",
341+
"(is %s, expected at least %s)",
343342
self.metadata_url,
344343
fileversion,
345344
_MIN_SUPPORTED_VERSION,
346-
version.REPL_SERVICE_VERSION,
347345
)
348346
sys.exit(1)
349-
_LOG.info("Metadata file version: %s", version.REPL_SERVICE_VERSION)
347+
_LOG.info("Metadata file version: %s", fileversion)
350348

351349
@property
352350
def table_contribs_spec(self) -> Generator[TableContributionsSpec, None, None]:

‎rootfs/ingest/python/qserv/replicationclient.py

+4-13
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ def __init__(
6464

6565
self.repl_url = util.trailing_slash(repl_url)
6666
self.http = Http(timeout_read_sec, timeout_write_sec, auth_path)
67-
self._check_version()
67+
self._report_version()
6868
self.index_url = urllib.parse.urljoin(self.repl_url, "replication/sql/index")
6969

7070
def abort_all_transactions(self, database: str) -> None:
@@ -102,20 +102,11 @@ def close_transaction(self, database: str, transaction_id: int, success: bool) -
102102
for trans in responseJson["databases"][database]["transactions"]:
103103
_LOG.debug("Close transaction (id: %s state: %s)", trans["id"], trans["state"])
104104

105-
def _check_version(self) -> None:
106-
"""Check replication service version and exit if it is not
107-
the expected one
108-
"""
105+
def _report_version(self) -> None:
106+
"""Get and report replication service version"""
109107
url = urllib.parse.urljoin(self.repl_url, "meta/version")
110108
responseJson = self.http.get(url)
111-
if responseJson["version"] != version.REPL_SERVICE_VERSION:
112-
_LOG.critical(
113-
"Invalid replication server version (is %s, expected %s)",
114-
responseJson["version"],
115-
version.REPL_SERVICE_VERSION,
116-
)
117-
sys.exit(1)
118-
_LOG.info("Replication service version: v%s", version.REPL_SERVICE_VERSION)
109+
_LOG.info("Replication service version: %s, this application's version: %s", responseJson["version"], version.REPL_SERVICE_VERSION)
119110

120111
def database_config(self, database: str, ingest_service_config: IngestServiceConfig) -> None:
121112
"""Set replication system configuration for a given database https://co

‎rootfs/ingest/python/qserv/version.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,8 @@
4141
# ---------------------------------
4242
_LOG = logging.getLogger(__name__)
4343

44-
REPL_SERVICE_VERSION = 30
45-
INGEST_CONFIG_VERSION = 15
44+
# The version number of the Qserv Replication/Ingest service this application
45+
# has been written for. The number is sent to the REST services in all
46+
# requests. This mechanism is meant to check compatibility between the application
47+
# and the services.
48+
REPL_SERVICE_VERSION = 16

0 commit comments

Comments
 (0)
Please sign in to comment.