From 67c416fb06fae13d75d52a521c2a0feade633229 Mon Sep 17 00:00:00 2001 From: Adam Parker <105722748+adam-parker1@users.noreply.github.com> Date: Thu, 16 Jan 2025 08:17:40 +0000 Subject: [PATCH] Add workflows to validate and formatting fixes (#28) * Fix diagnostic schema and example * feat: add custom camel case validator to git action * fix: add multiple file to json lint schema validation * fix: typo error in workflow file * fix: typo bug in workflow * fix: typo bug in workflow * fix: typo bug in workflow * fix: typo bug in workflow * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * fix: change json file validator package * fix: change json file validator package * fix: change json file validator package * fix: errors * Revert "fix: change json file validator package" This reverts commit 29a8eb4181b1ac06fe4eee36b80fac4486b3aa99. * fix: remote pull * feat: add new json validator package * fix: Error in worlflow file * fix: sample.schema.json validation error * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Update medata-ci-workflows.yml * Pull: from remote main to use cardinality validator * fix: changed schema-based validator package to cardinalby * fix: faclity json has wrong facility-id value * fix: error in custom validation python file * fix: file path error in custom validation python file * Revert "fix: error in custom validation python file" This reverts commit dc340655f736a9e102542d54b08cef23a250bba5. * fix: revert to python bug fix * fix: json file argument * fix: json file argument * fix: Error and regex pattern in calidate_camel_case.py * fix: convert args for python file into list * fix: convert args for python file into list * fix: convert args for python file into list * fix: convert args for python file into list * fix: convert args for python file into list * fix: convert args for python file into list * fix: changed python file to accept single args * fix: changed python file to accept single args * feat: Add snack case to check camelCase validator * fix: adjusted the rege4x pattern to match more than 1 upper after lower case * fix: adjusted the rege4x pattern to match more than 1 upper after lower case * fix: returned adjusted case * feat: added exit to python code if validation fails * feat: added exit to python code if validation fails * feat: added exit to python code if validation fails * feat: added exit to python code if validation fails * fix: changed error message * fix: returned campaign.schema.json file to normal * fix: returned campaign.schema.json file to normal --------- Co-authored-by: Nathan Cummings Co-authored-by: Nathan Cummings Co-authored-by: khalid Co-authored-by: Khalid Lawal Abiola <81440160+khalsz@users.noreply.github.com> --- .github/workflows/medata-ci-workflows.yml | 49 ++++++++++++++++ scicat-schema/scicat.sample.schema.json | 1 + ukaea-schema/examples/hive/experiment.json | 53 ++++++++++++----- .../facility/hive/diagnostic.schema.json | 2 + validate_camel_case.py | 57 +++++++++++++++++++ 5 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/medata-ci-workflows.yml create mode 100644 validate_camel_case.py diff --git a/.github/workflows/medata-ci-workflows.yml b/.github/workflows/medata-ci-workflows.yml new file mode 100644 index 0000000..0536702 --- /dev/null +++ b/.github/workflows/medata-ci-workflows.yml @@ -0,0 +1,49 @@ +name: Metadata CI Workflow +on: + push: + branches: + - main + pull_request: + branches: + - main + + +jobs: + validate-json: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: validate-json-files + id: json-validate + uses: GrantBirki/json-yaml-validate@v3 + with: + comment: "true" + + - name: Validate campaign JSON + uses: cardinalby/schema-validator-action@v3 + with: + schema: ukaea-schema/facility/hive/campaign.schema.json + file: ./ukaea-schema/examples/hive/campaign.json + - name: Validate experiment JSON + uses: cardinalby/schema-validator-action@v3 + with: + schema: ./ukaea-schema/facility/hive/campaign.schema.json + file: ./ukaea-schema/examples/hive/campaign.json + + # - name: Validate facility JSON + # uses: cardinalby/schema-validator-action@v3 + # with: + # schema: ./ukaea-schema/facility/facility.schema.json + # file: ./ukaea-schema/examples/facility.json + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: custom-camel-case-validation + run: | + readarray -d '' json_file_paths < <(find . -name *.json -print0) + echo ${json_file_paths[@]} + for json_file in ${json_file_paths[@]}; do + python validate_camel_case.py $json_file + done + diff --git a/scicat-schema/scicat.sample.schema.json b/scicat-schema/scicat.sample.schema.json index e69de29..9082d25 100644 --- a/scicat-schema/scicat.sample.schema.json +++ b/scicat-schema/scicat.sample.schema.json @@ -0,0 +1 @@ +{"key": "value"} \ No newline at end of file diff --git a/ukaea-schema/examples/hive/experiment.json b/ukaea-schema/examples/hive/experiment.json index cd85f5d..2bd3a4b 100644 --- a/ukaea-schema/examples/hive/experiment.json +++ b/ukaea-schema/examples/hive/experiment.json @@ -1,18 +1,18 @@ { - "campaignID": "HIVE-20230901", - "experimentID": "HIVE-E-20230905", + "campaignID": "HIVE-20230901", + "experimentID": "HIVE-E-20230905", "leadInvestigator": { "firstName": "Alice", "lastName": "Johnson", "email": "alice.johnson@example.com" }, "customer": "ACME Corp", - "experimentStart": "2023-09-05T10:00:00Z", - "experimentEnd": "2023-09-05T16:00:00Z", - "experimentType": "Induction", + "experimentStart": "2023-09-05T10:00:00Z", + "experimentEnd": "2023-09-05T16:00:00Z", + "experimentType": "Induction", "sampleCooling": true, "pulse": { - "pulseID": "123e4567-e89b-12d3-a456-426614174000", + "pulseID": "123e4567-e89b-12d3-a456-426614174000", "firstOperator": { "firstName": "Bob", "lastName": "Smith", @@ -23,9 +23,9 @@ "lastName": "Davis", "email": "carol.davis@example.com" }, - "pulseStart": "2023-09-05T12:00:00Z", - "pulseDuration": 120, - "dataCaptureStart": "2023-09-05T12:00:10Z", + "pulseStart": "2023-09-05T12:00:00Z", + "pulseDuration": 120, + "dataCaptureStart": "2023-09-05T12:00:10Z", "operatorComment": "Pulse completed without issues. Slight delay in data capture start.", "pulseQuality": "Success", "coolantInformation": { @@ -62,6 +62,8 @@ }, "diagnostics": [ { + "diagnosticName": "TC01", + "diagnosticType": "Thermocouple", "port": { "portID": "Port_1", "portDescription": "Front Port", @@ -71,13 +73,19 @@ "status": "Active", "attachment": "spot weld with inert gas shield", "tcType": "K", - "location": [18.065, 2.355, 23.0], + "location": [ + 18.065, + 2.355, + 23.0 + ], "areaType": "Circular", "circleDiameter": 1.31, "noiseFloor": "not captured - assume standard resolution" } }, { + "diagnosticName": "Cam01", + "diagnosticType": "Camera", "port": { "portID": "Port_6", "portDescription": "Bottom Port", @@ -100,7 +108,10 @@ "patternSpeckle": "White speckles", "approxFeatureSize": 50.0, "calTargetMake": "OptiCal", - "calTargetDims": [100, 100], + "calTargetDims": [ + 100, + 100 + ], "calTargetSpacing": 0.5 }, "cameraSetup": { @@ -110,7 +121,10 @@ "make": "Canon", "model": "EOS 5D Mark IV", "serialNumber": "CN12345678", - "resolution": [6720, 4480] + "resolution": [ + 6720, + 4480 + ] }, "lensInformation": { "make": "Canon", @@ -118,7 +132,10 @@ "serialNumber": "LN98765432", "focalLength": 50, "aperture": "f/1.8", - "fieldOfView": [6720, 4480] + "fieldOfView": [ + 6720, + 4480 + ] }, "captureSettings": { "imageAcquisitionRate": 30, @@ -131,7 +148,10 @@ "make": "Nikon", "model": "D850", "serialNumber": "NK87654321", - "resolution": [8256, 5504] + "resolution": [ + 8256, + 5504 + ] }, "lensInformation": { "make": "Nikon", @@ -139,7 +159,10 @@ "serialNumber": "LN12345678", "focalLength": 70, "aperture": "f/2.8", - "fieldOfView": [8256, 5504] + "fieldOfView": [ + 8256, + 5504 + ] }, "captureSettings": { "imageAcquisitionRate": 25, diff --git a/ukaea-schema/facility/hive/diagnostic.schema.json b/ukaea-schema/facility/hive/diagnostic.schema.json index 8bfad1e..9596bbd 100644 --- a/ukaea-schema/facility/hive/diagnostic.schema.json +++ b/ukaea-schema/facility/hive/diagnostic.schema.json @@ -4,6 +4,8 @@ "description": "Metadata schema for a HIVE diagnostic", "type": "object", "required": [ + "diagnosticName", + "diagnosticType", "port", "diagnostic" ], diff --git a/validate_camel_case.py b/validate_camel_case.py new file mode 100644 index 0000000..5f78062 --- /dev/null +++ b/validate_camel_case.py @@ -0,0 +1,57 @@ +import re +import json +import argparse +from pathlib import Path +import sys + + + # regex to match camel case +def match_camel_case(s): + # match camelCase format using regular expression + return bool(re.match(r'^[$a-z]+([.A-Z0-9]+[a-z]*)*$', s)) + + +def validate_camel_case(data): + # validate camelCase in json key + for key, value in data.items(): + if not match_camel_case(key): + return False, f"Invalid key format (not CamelCase): {key}" + + # check if json value is a dictionary + if isinstance(value, dict): + is_valid, message = validate_camel_case(value) # recursive check nested dictionary + if not is_valid: + return False, f"{message}" + return True, None + + +def val_json_file(json_file): + # initialize empty list to store error result for invalid json file validation + try: + with open(Path(json_file), 'r') as jf: + data = json.load(jf) + is_valid, message = validate_camel_case(data) + if not is_valid: + # append result of json file with non cameCase keys + print(f"json file: {json_file} contains {message}") + sys.exit(1) + else: + return(f"json file: {json_file} contains valid camelCase keys") + except json.JSONDecodeError as e: + # append result of invalid json file + print(f"Error in file {json_file}, Invalid JSON format {e}") + sys.exit(1) + + + + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="all json files in the repository") + parser.add_argument("json_file_paths", type=str, + help="pass the paths to all json files in the repository") + arg = parser.parse_args() + + validation_result = val_json_file(arg.json_file_paths) + print(validation_result) \ No newline at end of file