From 059f680ce1d256e02acecd9e8464715e58bc2307 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Wed, 24 Jul 2024 19:34:55 -0300 Subject: [PATCH 01/11] =?UTF-8?q?refactor:=20mover=20asn1=20bu=20dentro=20?= =?UTF-8?q?do=20diret=C3=B3rio=20static?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/bu_service/app/services/bu_service.py | 2 +- backend/bu_service/app/services/{ => static}/bu.asn1 | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/bu_service/app/services/{ => static}/bu.asn1 (100%) diff --git a/backend/bu_service/app/services/bu_service.py b/backend/bu_service/app/services/bu_service.py index d7ceb3a3..5cc1dc57 100644 --- a/backend/bu_service/app/services/bu_service.py +++ b/backend/bu_service/app/services/bu_service.py @@ -12,7 +12,7 @@ TREE_NAME_PREFIX = os.getenv('TREE_NAME_PREFIX') TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') -conv = asn1tools.compile_files("app/services/bu.asn1") +conv = asn1tools.compile_files("app/services/static/bu.asn1") county_codes = json.load(open("app/services/static/county_codes.json", encoding='utf-8')) diff --git a/backend/bu_service/app/services/bu.asn1 b/backend/bu_service/app/services/static/bu.asn1 similarity index 100% rename from backend/bu_service/app/services/bu.asn1 rename to backend/bu_service/app/services/static/bu.asn1 From 56e5b847edeb3f361205d70a3335591047ca4513 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:46:45 -0300 Subject: [PATCH 02/11] =?UTF-8?q?fix:=20ignorar=20assets=20adicionados=20p?= =?UTF-8?q?elo=20usu=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bu_utils/.gitignore | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bu_utils/.gitignore b/bu_utils/.gitignore index b264d33f..03f45090 100644 --- a/bu_utils/.gitignore +++ b/bu_utils/.gitignore @@ -1,8 +1,9 @@ -assets/bus/* -assets/bus_1/* +assets/** results* .idea/* __pycache__ **/__pycache/** -!results/example_bus_consolidated.json \ No newline at end of file +!results/example_bus_consolidated.json +!assets/mocked_bus/* +!assets/specification_files/* From 38bea38aea300c6b8134ef624d0d51961cbb177c Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:47:48 -0300 Subject: [PATCH 03/11] =?UTF-8?q?refactor:=20mudar=20m=C3=A9todo=20save=20?= =?UTF-8?q?do=20reposit=C3=B3rio=20de=20BUs=20para=20salvar=20caso=20n?= =?UTF-8?q?=C3=A3o=20exista,=20atualizar=20caso=20exista?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/database/repositories/bu_repository.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/bu_service/app/database/repositories/bu_repository.py b/backend/bu_service/app/database/repositories/bu_repository.py index 86ec4e1f..b5186405 100644 --- a/backend/bu_service/app/database/repositories/bu_repository.py +++ b/backend/bu_service/app/database/repositories/bu_repository.py @@ -4,5 +4,16 @@ collection = get_db()["bus"] +def find_one(election_id: int, zone: int, section: int): + data = collection.find_one({"eleicoes": election_id, "zona": zone, "secao": section}) + if data is None: + return None + return BuModel(**data) + + def save(bu: BuModel): - collection.insert_one(bu.dict()) + collection.update_one( + {"eleicoes": bu.eleicoes, "zona": bu.zona, "secao": bu.secao}, + {"$set": bu.model_dump()}, + upsert=True + ) From 77506a52d63f2322dc88703270c9d1a722608b66 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:49:54 -0300 Subject: [PATCH 04/11] =?UTF-8?q?feat:=20adicionar=20m=C3=A9todo=20que=20i?= =?UTF-8?q?nsere=20folha=20criando=20=C3=A1rvore=20caso=20n=C3=A3o=20exist?= =?UTF-8?q?a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/adapter/tlmanager_adapter.py | 4 +++- .../app/controllers/bu_controller.py | 5 +++-- .../app/services/tlmanager_service.py | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 backend/bu_service/app/services/tlmanager_service.py diff --git a/backend/bu_service/app/adapter/tlmanager_adapter.py b/backend/bu_service/app/adapter/tlmanager_adapter.py index 8e6ce244..95ff9913 100644 --- a/backend/bu_service/app/adapter/tlmanager_adapter.py +++ b/backend/bu_service/app/adapter/tlmanager_adapter.py @@ -1,11 +1,13 @@ -import requests import os + import dotenv +import requests dotenv.load_dotenv() TL_MANAGER_URL = os.getenv('TL_MANAGER_URL') + def get_trees(): return requests.get(f'{TL_MANAGER_URL}/').json() diff --git a/backend/bu_service/app/controllers/bu_controller.py b/backend/bu_service/app/controllers/bu_controller.py index 2232faa6..6a2a2622 100644 --- a/backend/bu_service/app/controllers/bu_controller.py +++ b/backend/bu_service/app/controllers/bu_controller.py @@ -1,4 +1,5 @@ import logging + from fastapi import APIRouter, UploadFile, File from app.services import bu_service @@ -12,5 +13,5 @@ async def create(file: UploadFile = File(...)): file_content = await file.read() logging.info(f"Received file {file_name}") - bu_service.insert(file_content, file_name) - return {"status": "ok"} + bu_parsed = bu_service.insert(file_content, file_name) + return {"status": "ok", "bu": bu_parsed.dict(exclude={"bu"})} diff --git a/backend/bu_service/app/services/tlmanager_service.py b/backend/bu_service/app/services/tlmanager_service.py new file mode 100644 index 00000000..1c8c811d --- /dev/null +++ b/backend/bu_service/app/services/tlmanager_service.py @@ -0,0 +1,22 @@ +import logging + +from app.adapter.tlmanager_adapter import * + +TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') + + +def insert_creating_tree_if_not_exists(tree_name, data, commitment_size=TREE_DEFAULT_COMMITMENT_SIZE): + insert_leaf_response = insert_leaf(tree_name, data) + logging.info(insert_leaf_response.json()) + if insert_leaf_response.status_code != 200 and insert_leaf_response.json().get('message') == "Tree does not exist": + logging.info(f"Creating tree with name {tree_name}") + create_tree(tree_name, commitment_size) + + logging.info(f"Trying to insert leaf for tree {tree_name} again") + insert_leaf_response = insert_leaf(tree_name, data) + logging.info(insert_leaf_response.json()) + + if insert_leaf_response.status_code != 200: + raise Exception("Failed to insert leaf") + + return insert_leaf_response From 1d49252396187a8bd4b6984f475b529f926ff595 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:50:39 -0300 Subject: [PATCH 05/11] =?UTF-8?q?feat:=20adicionar=20arquivo=20para=20trat?= =?UTF-8?q?ativa=20de=20vari=C3=A1veis=20de=20ambiente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/bu_service/.env | 3 ++- backend/bu_service/app/config/environment.py | 27 ++++++++++++++++++++ backend/bu_service/main.py | 5 ++-- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 backend/bu_service/app/config/environment.py diff --git a/backend/bu_service/.env b/backend/bu_service/.env index a3eb65cf..801a32d1 100644 --- a/backend/bu_service/.env +++ b/backend/bu_service/.env @@ -1,4 +1,5 @@ -TREE_NAME_PREFIX=eleicao_ +BU_TREE_NAME_TEMPLATE=bus_eleicao_${ELECTION_ID} +METADATA_TREE_NAME_TEMPLATE=${METADATA_NAME}_eleicao_${ELECTION_ID} TREE_DEFAULT_COMMITMENT_SIZE=8 TL_MANAGER_URL=http://127.0.0.1:8000 MONGO_URL=mongodb://localhost:27017 \ No newline at end of file diff --git a/backend/bu_service/app/config/environment.py b/backend/bu_service/app/config/environment.py new file mode 100644 index 00000000..ecd2941c --- /dev/null +++ b/backend/bu_service/app/config/environment.py @@ -0,0 +1,27 @@ +import logging +import os + +from dotenv import load_dotenv + +load_dotenv() + +BU_TREE_NAME_TEMPLATE = os.getenv('BU_TREE_NAME_TEMPLATE') +TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') +METADATA_TREE_NAME_TEMPLATE = os.getenv('METADATA_TREE_NAME_TEMPLATE') +TL_MANAGER_URL = os.getenv('TL_MANAGER_URL') +MONGO_URL = os.getenv('MONGO_URL') + +if '${ELECTION_ID}' not in BU_TREE_NAME_TEMPLATE: + raise Exception("BU_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}") + +if '${METADATA_NAME}' not in METADATA_TREE_NAME_TEMPLATE: + raise Exception("METADATA_TREE_NAME_TEMPLATE must contain ${METADATA_NAME}") + +if '${ELECTION_ID}' not in METADATA_TREE_NAME_TEMPLATE: + raise Exception("METADATA_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}") + +logging.debug("BU_TREE_NAME_TEMPLATE: %s", BU_TREE_NAME_TEMPLATE) +logging.debug("METADATA_TREE_NAME_TEMPLATE: %s", METADATA_TREE_NAME_TEMPLATE) +logging.debug("TL_MANAGER_URL: %s", TL_MANAGER_URL) +logging.debug("MONGO_URL: %s", MONGO_URL) +logging.debug("TREE_DEFAULT_COMMITMENT_SIZE: %s", TREE_DEFAULT_COMMITMENT_SIZE) diff --git a/backend/bu_service/main.py b/backend/bu_service/main.py index 9bb46cb5..dd89e47f 100644 --- a/backend/bu_service/main.py +++ b/backend/bu_service/main.py @@ -1,8 +1,7 @@ -import dotenv -import logging from fastapi import FastAPI -from app.controllers import tree_controller, bu_controller +from app.config.environment import * +from app.controllers import tree_controller, bu_controller, metadata_controller dotenv.load_dotenv() logging.basicConfig(level=logging.INFO) From 202ff465c3a7a31b6ef19465edfd67a5a3ad77e7 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:52:04 -0300 Subject: [PATCH 06/11] =?UTF-8?q?refactor:=20refatorar=20Service=20de=20bu?= =?UTF-8?q?s=20para=20utilizar=20o=20Service=20de=20=C3=A1rvores=20e=20ret?= =?UTF-8?q?ornar=20BU=20salvo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/bu_service/app/services/bu_service.py | 37 ++++++------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/backend/bu_service/app/services/bu_service.py b/backend/bu_service/app/services/bu_service.py index 5cc1dc57..0977b5ca 100644 --- a/backend/bu_service/app/services/bu_service.py +++ b/backend/bu_service/app/services/bu_service.py @@ -1,16 +1,13 @@ +import base64 +import json import logging import asn1tools -import json -import base64 -import os +from app.config.environment import BU_TREE_NAME_TEMPLATE from app.database.models.bu_model import BuModel, MerkleTreeInfo from app.database.repositories.bu_repository import save -from app.adapter.tlmanager_adapter import * - -TREE_NAME_PREFIX = os.getenv('TREE_NAME_PREFIX') -TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') +from app.services.tlmanager_service import insert_creating_tree_if_not_exists conv = asn1tools.compile_files("app/services/static/bu.asn1") county_codes = json.load(open("app/services/static/county_codes.json", encoding='utf-8')) @@ -20,7 +17,6 @@ class DictWithBytesToJsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, (bytes, bytearray)): return obj.hex() - # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, obj) @@ -73,27 +69,16 @@ def insert(file_content: bytes, filename: str): bu_parsed = parse_bu(bu_decoded, file_content, filename) logging.debug(bu_parsed) - for eleicao in bu_parsed.eleicoes: - logging.info(f"Inserting leaf for election {eleicao}") - response = insert_leaf(f'{TREE_NAME_PREFIX}{eleicao}', base64.b64encode(bu_parsed.bu).decode('utf-8')) - logging.info(response.json()) - if response.status_code != 200 and response.json().get('message') == "Tree does not exist": - logging.info(f"Creating tree for election {eleicao}") - create_tree(f'{TREE_NAME_PREFIX}{eleicao}', TREE_DEFAULT_COMMITMENT_SIZE) - logging.info(response.json()) - - logging.info(f"Trying to insert leaf for election {eleicao} again") - response = insert_leaf(f'{TREE_NAME_PREFIX}{eleicao}', base64.b64encode(bu_parsed.bu).decode('utf-8')) - logging.info(response.json()) - if response.status_code != 200: - raise Exception("Failed to insert leaf") - bu_parsed.merkletree_info[str(eleicao)] = MerkleTreeInfo( - tree_name=f'{TREE_NAME_PREFIX}{eleicao}', + for id_eleicao in bu_parsed.eleicoes: + tree_name = BU_TREE_NAME_TEMPLATE.replace('${ELECTION_ID}', str(id_eleicao)) + logging.info(f"Inserting leaf for election {id_eleicao}") + response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(bu_parsed.bu).decode('utf-8')) + bu_parsed.merkletree_info[str(id_eleicao)] = MerkleTreeInfo( + tree_name=tree_name, index=response.json()['index'], hash=response.json()['value']) logging.debug("Saving BU to database") logging.debug(bu_parsed) save(bu_parsed) - - + return bu_parsed From 9157f6c9978dc30918de965babe0f5cda5e2d9f2 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 25 Jul 2024 17:52:49 -0300 Subject: [PATCH 07/11] feat: adicionar controller de metadados de BU --- .../app/controllers/metadata_controller.py | 32 +++++++++++++++ .../app/database/models/bu_model.py | 9 ++++ .../app/services/metadata_service.py | 41 +++++++++++++++++++ backend/bu_service/main.py | 5 ++- 4 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 backend/bu_service/app/controllers/metadata_controller.py create mode 100644 backend/bu_service/app/services/metadata_service.py diff --git a/backend/bu_service/app/controllers/metadata_controller.py b/backend/bu_service/app/controllers/metadata_controller.py new file mode 100644 index 00000000..78b6f80c --- /dev/null +++ b/backend/bu_service/app/controllers/metadata_controller.py @@ -0,0 +1,32 @@ +import logging + +from fastapi import APIRouter, UploadFile, File, Form + +from app.services import metadata_service + +router = APIRouter() + + +@router.post("/create") +async def create( + metadata_name: str = Form(...), + id_eleicao: int = Form(...), + zona: int = Form(...), + secao: int = Form(...), + file: UploadFile = File(...) +): + file_name = file.filename + file_content = await file.read() + + logging.info(f"Received file {file_name}") + + try: + bu = metadata_service.insert(file_content, file_name, metadata_name, id_eleicao, zona, secao) + except Exception as e: + logging.error(e) + return {"status": "error", "message": str(e)} + + for metadata in bu.metadata: + del metadata.data + + return {"status": "ok", "bu": bu.dict(exclude={"bu"})} diff --git a/backend/bu_service/app/database/models/bu_model.py b/backend/bu_service/app/database/models/bu_model.py index fd69536d..9518de39 100644 --- a/backend/bu_service/app/database/models/bu_model.py +++ b/backend/bu_service/app/database/models/bu_model.py @@ -1,4 +1,5 @@ from typing import Optional + from pydantic import BaseModel @@ -8,6 +9,13 @@ class MerkleTreeInfo(BaseModel): hash: str +class Metadata(BaseModel): + data_name: str + file_name: str + data: bytes + merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} + + class BuModel(BaseModel): eleicoes: list[int] filename: str @@ -18,3 +26,4 @@ class BuModel(BaseModel): bu_json: str bu: bytes merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} + metadata: list[Metadata] = [] diff --git a/backend/bu_service/app/services/metadata_service.py b/backend/bu_service/app/services/metadata_service.py new file mode 100644 index 00000000..cf7cef4b --- /dev/null +++ b/backend/bu_service/app/services/metadata_service.py @@ -0,0 +1,41 @@ +import base64 +import logging + +from app.adapter.tlmanager_adapter import * +from app.database.models.bu_model import MerkleTreeInfo, Metadata +from app.database.repositories.bu_repository import save, find_one +from app.services.tlmanager_service import insert_creating_tree_if_not_exists + +METADATA_TREE_NAME_TEMPLATE = os.getenv('METADATA_TREE_NAME_TEMPLATE') +TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') + + +def insert(file_content: bytes, filename: str, metadata_name: str, id_eleicao: int, zona: int, secao: int): + logging.debug("Retrieving BU with election id %d zone %d and section %d", id_eleicao, zona, secao) + bu = find_one(id_eleicao, zona, secao) + + if bu is None: + raise Exception("BU not found") + + responses = {} + for id_eleicao in bu.eleicoes: + tree_name = METADATA_TREE_NAME_TEMPLATE.replace('${METADATA_NAME}', metadata_name).replace('${ELECTION_ID}', + str(id_eleicao)) + logging.info(f"Inserting {filename} leaf for election {id_eleicao}") + response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) + responses[str(id_eleicao)] = MerkleTreeInfo( + tree_name=tree_name, + index=response.json()['index'], + hash=response.json()['value'] + ) + + bu.metadata.append(Metadata( + data_name=metadata_name, + file_name=filename, + data=file_content, + merkletree_info=responses)) + + logging.debug("Saving metadata to database") + save(bu) + + return bu diff --git a/backend/bu_service/main.py b/backend/bu_service/main.py index dd89e47f..e96ad504 100644 --- a/backend/bu_service/main.py +++ b/backend/bu_service/main.py @@ -3,12 +3,13 @@ from app.config.environment import * from app.controllers import tree_controller, bu_controller, metadata_controller -dotenv.load_dotenv() -logging.basicConfig(level=logging.INFO) +logging.basicConfig(level=logging.DEBUG) app = FastAPI() app.include_router(tree_controller.router, prefix="/tree", tags=["Tree management routes"]) app.include_router(bu_controller.router, prefix="/bu", tags=["BU management routes"]) +app.include_router(metadata_controller.router, prefix="/bu/metadata", + tags=["Route to insert a metadata related to a BU"]) @app.get("/") From 20402a5a6b9e53c8445bc5f03d1b01cc739a9156 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Wed, 31 Jul 2024 22:14:07 -0300 Subject: [PATCH 08/11] refactor: generalizar endpoint de outros dados eleitorais --- backend/bu_service/.env | 3 +- backend/bu_service/app/config/environment.py | 15 ++++-- .../controllers/election_data_controller.py | 32 +++++++++++++ .../app/controllers/metadata_controller.py | 32 ------------- .../app/database/models/bu_model.py | 14 +----- .../database/models/election_data_model.py | 15 ++++++ .../database/models/merkle_tree_info_model.py | 7 +++ .../repositories/election_data_repository.py | 8 ++++ .../app/services/election_data_service.py | 48 +++++++++++++++++++ .../app/services/metadata_service.py | 41 ---------------- backend/bu_service/main.py | 6 +-- 11 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 backend/bu_service/app/controllers/election_data_controller.py delete mode 100644 backend/bu_service/app/controllers/metadata_controller.py create mode 100644 backend/bu_service/app/database/models/election_data_model.py create mode 100644 backend/bu_service/app/database/models/merkle_tree_info_model.py create mode 100644 backend/bu_service/app/database/repositories/election_data_repository.py create mode 100644 backend/bu_service/app/services/election_data_service.py delete mode 100644 backend/bu_service/app/services/metadata_service.py diff --git a/backend/bu_service/.env b/backend/bu_service/.env index 801a32d1..0721bfb2 100644 --- a/backend/bu_service/.env +++ b/backend/bu_service/.env @@ -1,5 +1,6 @@ BU_TREE_NAME_TEMPLATE=bus_eleicao_${ELECTION_ID} -METADATA_TREE_NAME_TEMPLATE=${METADATA_NAME}_eleicao_${ELECTION_ID} +ELECTION_DATA_TREE_NAME_TEMPLATE=${ELECTION_DATA_NAME} +ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION=${ELECTION_DATA_NAME}_eleicao_${ELECTION_ID} TREE_DEFAULT_COMMITMENT_SIZE=8 TL_MANAGER_URL=http://127.0.0.1:8000 MONGO_URL=mongodb://localhost:27017 \ No newline at end of file diff --git a/backend/bu_service/app/config/environment.py b/backend/bu_service/app/config/environment.py index ecd2941c..e304de0b 100644 --- a/backend/bu_service/app/config/environment.py +++ b/backend/bu_service/app/config/environment.py @@ -7,21 +7,26 @@ BU_TREE_NAME_TEMPLATE = os.getenv('BU_TREE_NAME_TEMPLATE') TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') -METADATA_TREE_NAME_TEMPLATE = os.getenv('METADATA_TREE_NAME_TEMPLATE') +ELECTION_DATA_TREE_NAME_TEMPLATE = os.getenv('ELECTION_DATA_TREE_NAME_TEMPLATE') +ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION = os.getenv('ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION') TL_MANAGER_URL = os.getenv('TL_MANAGER_URL') MONGO_URL = os.getenv('MONGO_URL') if '${ELECTION_ID}' not in BU_TREE_NAME_TEMPLATE: raise Exception("BU_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}") -if '${METADATA_NAME}' not in METADATA_TREE_NAME_TEMPLATE: +if '${ELECTION_DATA_NAME}' not in ELECTION_DATA_TREE_NAME_TEMPLATE: raise Exception("METADATA_TREE_NAME_TEMPLATE must contain ${METADATA_NAME}") -if '${ELECTION_ID}' not in METADATA_TREE_NAME_TEMPLATE: - raise Exception("METADATA_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}") +if '${ELECTION_DATA_NAME}' not in ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: + raise Exception("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION must contain ${METADATA_NAME}") + +if '${ELECTION_ID}' not in ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: + raise Exception("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION must contain ${ELECTION_ID}") logging.debug("BU_TREE_NAME_TEMPLATE: %s", BU_TREE_NAME_TEMPLATE) -logging.debug("METADATA_TREE_NAME_TEMPLATE: %s", METADATA_TREE_NAME_TEMPLATE) +logging.debug("ELECTION_DATA_TREE_NAME_TEMPLATE: %s", ELECTION_DATA_TREE_NAME_TEMPLATE) +logging.debug("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: %s", ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION) logging.debug("TL_MANAGER_URL: %s", TL_MANAGER_URL) logging.debug("MONGO_URL: %s", MONGO_URL) logging.debug("TREE_DEFAULT_COMMITMENT_SIZE: %s", TREE_DEFAULT_COMMITMENT_SIZE) diff --git a/backend/bu_service/app/controllers/election_data_controller.py b/backend/bu_service/app/controllers/election_data_controller.py new file mode 100644 index 00000000..dd0d004c --- /dev/null +++ b/backend/bu_service/app/controllers/election_data_controller.py @@ -0,0 +1,32 @@ +import logging +from typing import List + +from fastapi import APIRouter, UploadFile, File, Form + +from app.services import election_data_service + +router = APIRouter() + + +@router.post("/create") +async def create( + metadata_name: str = Form(...), + id_eleicao: List[int] = Form(None), + zona: int = Form(None), + secao: int = Form(None), + file: UploadFile = File(...) +): + file_name = file.filename + file_content = await file.read() + + logging.info(f"Received file {file_name}") + if id_eleicao is not None or zona is not None or secao is not None: + logging.info(f"Received optional parameters: id_eleicao={id_eleicao}, zona={zona}, secao={secao}") + + try: + election_data = election_data_service.insert(file_content, file_name, metadata_name, id_eleicao, zona, secao) + except Exception as e: + logging.error(e) + return {"status": "error", "message": str(e)} + + return {"status": "ok", "bu": election_data.dict(exclude={"data"})} diff --git a/backend/bu_service/app/controllers/metadata_controller.py b/backend/bu_service/app/controllers/metadata_controller.py deleted file mode 100644 index 78b6f80c..00000000 --- a/backend/bu_service/app/controllers/metadata_controller.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging - -from fastapi import APIRouter, UploadFile, File, Form - -from app.services import metadata_service - -router = APIRouter() - - -@router.post("/create") -async def create( - metadata_name: str = Form(...), - id_eleicao: int = Form(...), - zona: int = Form(...), - secao: int = Form(...), - file: UploadFile = File(...) -): - file_name = file.filename - file_content = await file.read() - - logging.info(f"Received file {file_name}") - - try: - bu = metadata_service.insert(file_content, file_name, metadata_name, id_eleicao, zona, secao) - except Exception as e: - logging.error(e) - return {"status": "error", "message": str(e)} - - for metadata in bu.metadata: - del metadata.data - - return {"status": "ok", "bu": bu.dict(exclude={"bu"})} diff --git a/backend/bu_service/app/database/models/bu_model.py b/backend/bu_service/app/database/models/bu_model.py index 9518de39..d2f635f4 100644 --- a/backend/bu_service/app/database/models/bu_model.py +++ b/backend/bu_service/app/database/models/bu_model.py @@ -2,18 +2,7 @@ from pydantic import BaseModel - -class MerkleTreeInfo(BaseModel): - tree_name: str - index: int - hash: str - - -class Metadata(BaseModel): - data_name: str - file_name: str - data: bytes - merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} +from app.database.models.merkle_tree_info_model import MerkleTreeInfo class BuModel(BaseModel): @@ -26,4 +15,3 @@ class BuModel(BaseModel): bu_json: str bu: bytes merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} - metadata: list[Metadata] = [] diff --git a/backend/bu_service/app/database/models/election_data_model.py b/backend/bu_service/app/database/models/election_data_model.py new file mode 100644 index 00000000..40743668 --- /dev/null +++ b/backend/bu_service/app/database/models/election_data_model.py @@ -0,0 +1,15 @@ +from typing import Optional + +from pydantic import BaseModel + +from app.database.models.bu_model import MerkleTreeInfo + + +class ElectionData(BaseModel): + data_name: str + file_name: str + data: bytes + merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} + zona: Optional[int] = None + secao: Optional[int] = None + eleicoes: Optional[list[int]] = None diff --git a/backend/bu_service/app/database/models/merkle_tree_info_model.py b/backend/bu_service/app/database/models/merkle_tree_info_model.py new file mode 100644 index 00000000..ba5c239d --- /dev/null +++ b/backend/bu_service/app/database/models/merkle_tree_info_model.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class MerkleTreeInfo(BaseModel): + tree_name: str + index: int + hash: str diff --git a/backend/bu_service/app/database/repositories/election_data_repository.py b/backend/bu_service/app/database/repositories/election_data_repository.py new file mode 100644 index 00000000..5a70597b --- /dev/null +++ b/backend/bu_service/app/database/repositories/election_data_repository.py @@ -0,0 +1,8 @@ +from app.database.connection import get_db +from app.database.models.election_data_model import ElectionData + +db = get_db() + + +def save(data: ElectionData): + db[data.data_name].insert_one(data.model_dump()) diff --git a/backend/bu_service/app/services/election_data_service.py b/backend/bu_service/app/services/election_data_service.py new file mode 100644 index 00000000..52240fd6 --- /dev/null +++ b/backend/bu_service/app/services/election_data_service.py @@ -0,0 +1,48 @@ +import base64 +import logging +from typing import Optional, List + +from app.config.environment import ELECTION_DATA_TREE_NAME_TEMPLATE, ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION +from app.database.models.bu_model import MerkleTreeInfo +from app.database.models.election_data_model import ElectionData +from app.database.repositories.election_data_repository import save +from app.services.tlmanager_service import insert_creating_tree_if_not_exists + + +def insert(file_content: bytes, filename: str, metadata_name: str, id_eleicao: List[int], zona: Optional[int] = None, + secao: Optional[int] = None): + responses = {} + if id_eleicao is not None: + for eleicao in id_eleicao: + tree_name = ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION.replace('${ELECTION_DATA_NAME}', + metadata_name).replace('${ELECTION_ID}', + str(eleicao)) + response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) + responses[str(id_eleicao)] = MerkleTreeInfo( + tree_name=tree_name, + index=response.json()['index'], + hash=response.json()['value'] + ) + else: + tree_name = ELECTION_DATA_TREE_NAME_TEMPLATE.replace('${ELECTION_DATA_NAME}', metadata_name) + response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) + responses['metadata'] = MerkleTreeInfo( + tree_name=tree_name, + index=response.json()['index'], + hash=response.json()['value'] + ) + + election_data = ElectionData( + data_name=metadata_name, + file_name=filename, + data=file_content, + merkletree_info=responses, + eleicoes=id_eleicao, + zona=zona, + secao=secao + ) + + logging.debug("Saving election data to database") + save(election_data) + + return election_data diff --git a/backend/bu_service/app/services/metadata_service.py b/backend/bu_service/app/services/metadata_service.py deleted file mode 100644 index cf7cef4b..00000000 --- a/backend/bu_service/app/services/metadata_service.py +++ /dev/null @@ -1,41 +0,0 @@ -import base64 -import logging - -from app.adapter.tlmanager_adapter import * -from app.database.models.bu_model import MerkleTreeInfo, Metadata -from app.database.repositories.bu_repository import save, find_one -from app.services.tlmanager_service import insert_creating_tree_if_not_exists - -METADATA_TREE_NAME_TEMPLATE = os.getenv('METADATA_TREE_NAME_TEMPLATE') -TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') - - -def insert(file_content: bytes, filename: str, metadata_name: str, id_eleicao: int, zona: int, secao: int): - logging.debug("Retrieving BU with election id %d zone %d and section %d", id_eleicao, zona, secao) - bu = find_one(id_eleicao, zona, secao) - - if bu is None: - raise Exception("BU not found") - - responses = {} - for id_eleicao in bu.eleicoes: - tree_name = METADATA_TREE_NAME_TEMPLATE.replace('${METADATA_NAME}', metadata_name).replace('${ELECTION_ID}', - str(id_eleicao)) - logging.info(f"Inserting {filename} leaf for election {id_eleicao}") - response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) - responses[str(id_eleicao)] = MerkleTreeInfo( - tree_name=tree_name, - index=response.json()['index'], - hash=response.json()['value'] - ) - - bu.metadata.append(Metadata( - data_name=metadata_name, - file_name=filename, - data=file_content, - merkletree_info=responses)) - - logging.debug("Saving metadata to database") - save(bu) - - return bu diff --git a/backend/bu_service/main.py b/backend/bu_service/main.py index e96ad504..4b17bee3 100644 --- a/backend/bu_service/main.py +++ b/backend/bu_service/main.py @@ -1,15 +1,15 @@ from fastapi import FastAPI from app.config.environment import * -from app.controllers import tree_controller, bu_controller, metadata_controller +from app.controllers import tree_controller, bu_controller, election_data_controller logging.basicConfig(level=logging.DEBUG) app = FastAPI() app.include_router(tree_controller.router, prefix="/tree", tags=["Tree management routes"]) app.include_router(bu_controller.router, prefix="/bu", tags=["BU management routes"]) -app.include_router(metadata_controller.router, prefix="/bu/metadata", - tags=["Route to insert a metadata related to a BU"]) +app.include_router(election_data_controller.router, prefix="/election_data", + tags=["Route to insert election data in the TLManager"]) @app.get("/") From 63f1e553063df67edccc53ae702de96755f74cbb Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 1 Aug 2024 13:17:43 -0300 Subject: [PATCH 09/11] =?UTF-8?q?feat:=20adicionar=20m=C3=A9todos=20de=20l?= =?UTF-8?q?eitura=20de=20outros=20dados=20eleitorais?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database/models/election_data_model.py | 4 +- .../app/services/election_data_service.py | 2 +- backend/public/server.js | 3 ++ .../controllers/election-data.controller.js | 46 +++++++++++++++++++ .../public/src/database/models/bu.model.js | 9 +--- .../database/models/election-data.model.js | 16 +++++++ .../database/models/merkletree-info.model.js | 9 ++++ .../repository/election-data.repository.js | 26 +++++++++++ 8 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 backend/public/src/controllers/election-data.controller.js create mode 100644 backend/public/src/database/models/election-data.model.js create mode 100644 backend/public/src/database/models/merkletree-info.model.js create mode 100644 backend/public/src/database/repository/election-data.repository.js diff --git a/backend/bu_service/app/database/models/election_data_model.py b/backend/bu_service/app/database/models/election_data_model.py index 40743668..9fe811e8 100644 --- a/backend/bu_service/app/database/models/election_data_model.py +++ b/backend/bu_service/app/database/models/election_data_model.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Dict, Union from pydantic import BaseModel @@ -9,7 +9,7 @@ class ElectionData(BaseModel): data_name: str file_name: str data: bytes - merkletree_info: Optional[dict[str, MerkleTreeInfo]] = {} + merkletree_info: Union[Dict[str, MerkleTreeInfo], MerkleTreeInfo] = None zona: Optional[int] = None secao: Optional[int] = None eleicoes: Optional[list[int]] = None diff --git a/backend/bu_service/app/services/election_data_service.py b/backend/bu_service/app/services/election_data_service.py index 52240fd6..27283092 100644 --- a/backend/bu_service/app/services/election_data_service.py +++ b/backend/bu_service/app/services/election_data_service.py @@ -26,7 +26,7 @@ def insert(file_content: bytes, filename: str, metadata_name: str, id_eleicao: L else: tree_name = ELECTION_DATA_TREE_NAME_TEMPLATE.replace('${ELECTION_DATA_NAME}', metadata_name) response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) - responses['metadata'] = MerkleTreeInfo( + responses = MerkleTreeInfo( tree_name=tree_name, index=response.json()['index'], hash=response.json()['value'] diff --git a/backend/public/server.js b/backend/public/server.js index 76402c97..6c57ff2c 100644 --- a/backend/public/server.js +++ b/backend/public/server.js @@ -12,6 +12,9 @@ app.use("/bu", bu_controller); const tree_controller = require("./src/controllers/tree.controller"); app.use("/tree", tree_controller); +const election_data_controller = require("./src/controllers/election-data.controller"); +app.use("/election-data", election_data_controller); + // set port, listen for requests app.listen(application_config.port, () => { console.log(`Server is running on port ${application_config.port}.`); diff --git a/backend/public/src/controllers/election-data.controller.js b/backend/public/src/controllers/election-data.controller.js new file mode 100644 index 00000000..84a19669 --- /dev/null +++ b/backend/public/src/controllers/election-data.controller.js @@ -0,0 +1,46 @@ +const router = require('express').Router(); +const election_data_repository = require('../database/repository/election-data.repository'); + +router.get("/find_by_merkletree_index_range", async (req, res) => { + const data_name = req.query.data_name + const initial_index = req.query.initial_index + const final_index = req.query.final_index + + console.info(`[election-data.controller] GET /find_by_merkletree_index_range ${data_name} ${initial_index} ${final_index}`) + + const data = await election_data_repository.findByMerkletreeIndexRange(data_name, initial_index, final_index) + + res.json(data) +}) + +router.get("/find_by_id", async (req, res) => { + const id = req.query.id + const data_name = req.query.data_name + + console.info(`[election-data.controller] GET /find_by_id ${data_name} ${id}`) + + const data = await election_data_repository.findById(data_name, id) + + res.json(data) +}) + +router.get("/download", async (req, res) => { + const id = req.query.id + const data_name = req.query.data_name + + console.info(`[election-data.controller] GET /download ${data_name} ${id}`) + + const data = await election_data_repository.findById(data_name, id) + + if (!data) { + res.status(404).send('Not Found') + return + } + + const buffer = Buffer.from(data.data, 'base64') + res.setHeader('Content-Disposition', `attachment; filename=${data.filename}`) + + res.send(buffer) +}) + +module.exports = router; diff --git a/backend/public/src/database/models/bu.model.js b/backend/public/src/database/models/bu.model.js index c3246d01..32713eb4 100644 --- a/backend/public/src/database/models/bu.model.js +++ b/backend/public/src/database/models/bu.model.js @@ -1,10 +1,5 @@ const mongoose = require("mongoose"); - -const merkletree_info = new mongoose.Schema({ - tree_name: String, - index: Number, - hash: String, -}); +const {merkletree_info} = require("./merkletree-info.model"); const schema_bu = new mongoose.Schema({ eleicoes: [Number], @@ -19,11 +14,9 @@ const schema_bu = new mongoose.Schema({ }); schema_bu.index({ UF:1, zona:1, secao:1 }); - schema_bu.index({ eleicoes:1 }); schema_bu.index({ eleicoes:1, UF:1 }); schema_bu.index({ eleicoes:1, UF:1, municipio:1 }); - schema_bu.index({ "merkletree_info.$**": 1 }); module.exports = { schema_bu } \ No newline at end of file diff --git a/backend/public/src/database/models/election-data.model.js b/backend/public/src/database/models/election-data.model.js new file mode 100644 index 00000000..b554de19 --- /dev/null +++ b/backend/public/src/database/models/election-data.model.js @@ -0,0 +1,16 @@ +const mongoose = require("mongoose"); +const {merkletree_info} = require("./merkletree-info.model"); + +const schema_election_data = new mongoose.Schema({ + data_name: String, + filename: String, + data: Buffer, + merkletree_info: { type: Object, of: merkletree_info }, + zona: Number, + secao: Number, + eleicoes: [Number], +}); + +schema_election_data.index({ "merkletree_info.$**": 1 }); + +module.exports = { schema_election_data } \ No newline at end of file diff --git a/backend/public/src/database/models/merkletree-info.model.js b/backend/public/src/database/models/merkletree-info.model.js new file mode 100644 index 00000000..c6575752 --- /dev/null +++ b/backend/public/src/database/models/merkletree-info.model.js @@ -0,0 +1,9 @@ +const mongoose = require("mongoose"); + +const merkletree_info = new mongoose.Schema({ + tree_name: String, + index: Number, + hash: String, +}); + +module.exports = { merkletree_info } \ No newline at end of file diff --git a/backend/public/src/database/repository/election-data.repository.js b/backend/public/src/database/repository/election-data.repository.js new file mode 100644 index 00000000..3279a7f4 --- /dev/null +++ b/backend/public/src/database/repository/election-data.repository.js @@ -0,0 +1,26 @@ +const mongoose = require("mongoose"); +const { schema_election_data } = require("../models/election-data.model") + + +exports.findById = async (data_name, id) => { + const repository = mongoose.model(data_name, schema_election_data, data_name); + return await repository.findOne({ _id: id }) + .then((data) => { + data._doc.data = Buffer.from(data.data, 'base64').toString('base64') + return data + }) + .catch((err) => { + console.error(`[ERROR][election-data.repository] ${err}`) + }) +} + +exports.findByMerkletreeIndexRange = async (data_name, initial_index, final_index) => { + const repository = mongoose.model(data_name, schema_election_data, data_name); + return await repository.find({ "merkletree_info.index": { $gte: parseInt(initial_index), $lte: parseInt(final_index) } }, { data: 0 }) + .then((data) => { + return data; + }) + .catch((err) => { + console.error(`[ERROR][election-data.repository] ${err}`); + }); +}; \ No newline at end of file From b0ff59ade17e95b13a29ef9ad3cd9fdf3ae48cd4 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 1 Aug 2024 20:58:17 -0300 Subject: [PATCH 10/11] =?UTF-8?q?refactor:=20simplificar=20endpoint=20de?= =?UTF-8?q?=20escrita=20de=20outros=20dados=20eleitorais=20para=20receber?= =?UTF-8?q?=20nome=20da=20=C3=A1rvore=20e=20arquivo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/bu_service/.env | 2 - backend/bu_service/app/config/environment.py | 12 ------ .../controllers/election_data_controller.py | 12 ++---- .../database/models/election_data_model.py | 8 +--- .../repositories/election_data_repository.py | 2 +- .../app/services/election_data_service.py | 37 ++++--------------- 6 files changed, 12 insertions(+), 61 deletions(-) diff --git a/backend/bu_service/.env b/backend/bu_service/.env index 0721bfb2..df1c701e 100644 --- a/backend/bu_service/.env +++ b/backend/bu_service/.env @@ -1,6 +1,4 @@ BU_TREE_NAME_TEMPLATE=bus_eleicao_${ELECTION_ID} -ELECTION_DATA_TREE_NAME_TEMPLATE=${ELECTION_DATA_NAME} -ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION=${ELECTION_DATA_NAME}_eleicao_${ELECTION_ID} TREE_DEFAULT_COMMITMENT_SIZE=8 TL_MANAGER_URL=http://127.0.0.1:8000 MONGO_URL=mongodb://localhost:27017 \ No newline at end of file diff --git a/backend/bu_service/app/config/environment.py b/backend/bu_service/app/config/environment.py index e304de0b..650e64ba 100644 --- a/backend/bu_service/app/config/environment.py +++ b/backend/bu_service/app/config/environment.py @@ -7,26 +7,14 @@ BU_TREE_NAME_TEMPLATE = os.getenv('BU_TREE_NAME_TEMPLATE') TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE') -ELECTION_DATA_TREE_NAME_TEMPLATE = os.getenv('ELECTION_DATA_TREE_NAME_TEMPLATE') -ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION = os.getenv('ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION') TL_MANAGER_URL = os.getenv('TL_MANAGER_URL') MONGO_URL = os.getenv('MONGO_URL') if '${ELECTION_ID}' not in BU_TREE_NAME_TEMPLATE: raise Exception("BU_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}") -if '${ELECTION_DATA_NAME}' not in ELECTION_DATA_TREE_NAME_TEMPLATE: - raise Exception("METADATA_TREE_NAME_TEMPLATE must contain ${METADATA_NAME}") - -if '${ELECTION_DATA_NAME}' not in ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: - raise Exception("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION must contain ${METADATA_NAME}") - -if '${ELECTION_ID}' not in ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: - raise Exception("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION must contain ${ELECTION_ID}") logging.debug("BU_TREE_NAME_TEMPLATE: %s", BU_TREE_NAME_TEMPLATE) -logging.debug("ELECTION_DATA_TREE_NAME_TEMPLATE: %s", ELECTION_DATA_TREE_NAME_TEMPLATE) -logging.debug("ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION: %s", ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION) logging.debug("TL_MANAGER_URL: %s", TL_MANAGER_URL) logging.debug("MONGO_URL: %s", MONGO_URL) logging.debug("TREE_DEFAULT_COMMITMENT_SIZE: %s", TREE_DEFAULT_COMMITMENT_SIZE) diff --git a/backend/bu_service/app/controllers/election_data_controller.py b/backend/bu_service/app/controllers/election_data_controller.py index dd0d004c..50b75730 100644 --- a/backend/bu_service/app/controllers/election_data_controller.py +++ b/backend/bu_service/app/controllers/election_data_controller.py @@ -1,5 +1,4 @@ import logging -from typing import List from fastapi import APIRouter, UploadFile, File, Form @@ -10,21 +9,16 @@ @router.post("/create") async def create( - metadata_name: str = Form(...), - id_eleicao: List[int] = Form(None), - zona: int = Form(None), - secao: int = Form(None), + tree_name: str = Form(...), file: UploadFile = File(...) ): file_name = file.filename file_content = await file.read() - logging.info(f"Received file {file_name}") - if id_eleicao is not None or zona is not None or secao is not None: - logging.info(f"Received optional parameters: id_eleicao={id_eleicao}, zona={zona}, secao={secao}") + logging.info(f"Received file {file_name} to insert in tree {tree_name}") try: - election_data = election_data_service.insert(file_content, file_name, metadata_name, id_eleicao, zona, secao) + election_data = election_data_service.insert(file_content, file_name, tree_name) except Exception as e: logging.error(e) return {"status": "error", "message": str(e)} diff --git a/backend/bu_service/app/database/models/election_data_model.py b/backend/bu_service/app/database/models/election_data_model.py index 9fe811e8..45911878 100644 --- a/backend/bu_service/app/database/models/election_data_model.py +++ b/backend/bu_service/app/database/models/election_data_model.py @@ -1,15 +1,9 @@ -from typing import Optional, Dict, Union - from pydantic import BaseModel from app.database.models.bu_model import MerkleTreeInfo class ElectionData(BaseModel): - data_name: str file_name: str data: bytes - merkletree_info: Union[Dict[str, MerkleTreeInfo], MerkleTreeInfo] = None - zona: Optional[int] = None - secao: Optional[int] = None - eleicoes: Optional[list[int]] = None + merkletree_info: MerkleTreeInfo diff --git a/backend/bu_service/app/database/repositories/election_data_repository.py b/backend/bu_service/app/database/repositories/election_data_repository.py index 5a70597b..361f8a98 100644 --- a/backend/bu_service/app/database/repositories/election_data_repository.py +++ b/backend/bu_service/app/database/repositories/election_data_repository.py @@ -5,4 +5,4 @@ def save(data: ElectionData): - db[data.data_name].insert_one(data.model_dump()) + db[data.merkletree_info.tree_name].insert_one(data.model_dump()) diff --git a/backend/bu_service/app/services/election_data_service.py b/backend/bu_service/app/services/election_data_service.py index 27283092..26d9136a 100644 --- a/backend/bu_service/app/services/election_data_service.py +++ b/backend/bu_service/app/services/election_data_service.py @@ -1,45 +1,22 @@ import base64 import logging -from typing import Optional, List -from app.config.environment import ELECTION_DATA_TREE_NAME_TEMPLATE, ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION -from app.database.models.bu_model import MerkleTreeInfo from app.database.models.election_data_model import ElectionData +from app.database.models.merkle_tree_info_model import MerkleTreeInfo from app.database.repositories.election_data_repository import save from app.services.tlmanager_service import insert_creating_tree_if_not_exists -def insert(file_content: bytes, filename: str, metadata_name: str, id_eleicao: List[int], zona: Optional[int] = None, - secao: Optional[int] = None): - responses = {} - if id_eleicao is not None: - for eleicao in id_eleicao: - tree_name = ELECTION_DATA_TREE_NAME_TEMPLATE_WITH_ELECTION.replace('${ELECTION_DATA_NAME}', - metadata_name).replace('${ELECTION_ID}', - str(eleicao)) - response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) - responses[str(id_eleicao)] = MerkleTreeInfo( - tree_name=tree_name, - index=response.json()['index'], - hash=response.json()['value'] - ) - else: - tree_name = ELECTION_DATA_TREE_NAME_TEMPLATE.replace('${ELECTION_DATA_NAME}', metadata_name) - response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) - responses = MerkleTreeInfo( - tree_name=tree_name, - index=response.json()['index'], - hash=response.json()['value'] - ) +def insert(file_content: bytes, filename: str, tree_name: str): + response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8')) election_data = ElectionData( - data_name=metadata_name, file_name=filename, data=file_content, - merkletree_info=responses, - eleicoes=id_eleicao, - zona=zona, - secao=secao + merkletree_info=MerkleTreeInfo( + tree_name=tree_name, + index=response.json()['index'], + hash=response.json()['value']) ) logging.debug("Saving election data to database") From 183747ec167ee9e0d9d2a3e5cd6bc7dadd085481 Mon Sep 17 00:00:00 2001 From: Guilherme Fumagali Marques Date: Thu, 1 Aug 2024 21:13:24 -0300 Subject: [PATCH 11/11] =?UTF-8?q?refactor:=20adaptar=20m=C3=A9todos=20de?= =?UTF-8?q?=20leitura=20de=20outros=20dados=20eleitorais=20com=20a=20simpl?= =?UTF-8?q?ifica=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/election-data.controller.js | 29 +++++++++++++------ .../database/models/election-data.model.js | 7 ++--- .../repository/election-data.repository.js | 22 ++++++++++---- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/backend/public/src/controllers/election-data.controller.js b/backend/public/src/controllers/election-data.controller.js index 84a19669..27623f45 100644 --- a/backend/public/src/controllers/election-data.controller.js +++ b/backend/public/src/controllers/election-data.controller.js @@ -2,35 +2,46 @@ const router = require('express').Router(); const election_data_repository = require('../database/repository/election-data.repository'); router.get("/find_by_merkletree_index_range", async (req, res) => { - const data_name = req.query.data_name + const tree_name = req.query.tree_name const initial_index = req.query.initial_index const final_index = req.query.final_index - console.info(`[election-data.controller] GET /find_by_merkletree_index_range ${data_name} ${initial_index} ${final_index}`) + console.info(`[election-data.controller] GET /find_by_merkletree_index_range ${tree_name} ${initial_index} ${final_index}`) - const data = await election_data_repository.findByMerkletreeIndexRange(data_name, initial_index, final_index) + const data = await election_data_repository.findByMerkletreeIndexRange(tree_name, initial_index, final_index) res.json(data) }) router.get("/find_by_id", async (req, res) => { const id = req.query.id - const data_name = req.query.data_name + const tree_name = req.query.tree_name - console.info(`[election-data.controller] GET /find_by_id ${data_name} ${id}`) + console.info(`[election-data.controller] GET /find_by_id ${tree_name} ${id}`) - const data = await election_data_repository.findById(data_name, id) + const data = await election_data_repository.findById(tree_name, id) + + res.json(data) +}) + +router.get("/find_by_filename", async (req, res) => { + const filename = req.query.filename + const tree_name = req.query.tree_name + + console.info(`[election-data.controller] GET /find_by_filename ${tree_name} ${filename}`) + + const data = await election_data_repository.findByFilename(tree_name, filename) res.json(data) }) router.get("/download", async (req, res) => { const id = req.query.id - const data_name = req.query.data_name + const tree_name = req.query.tree_name - console.info(`[election-data.controller] GET /download ${data_name} ${id}`) + console.info(`[election-data.controller] GET /download ${tree_name} ${id}`) - const data = await election_data_repository.findById(data_name, id) + const data = await election_data_repository.findById(tree_name, id) if (!data) { res.status(404).send('Not Found') diff --git a/backend/public/src/database/models/election-data.model.js b/backend/public/src/database/models/election-data.model.js index b554de19..45b691e7 100644 --- a/backend/public/src/database/models/election-data.model.js +++ b/backend/public/src/database/models/election-data.model.js @@ -2,15 +2,12 @@ const mongoose = require("mongoose"); const {merkletree_info} = require("./merkletree-info.model"); const schema_election_data = new mongoose.Schema({ - data_name: String, - filename: String, + file_name: String, data: Buffer, merkletree_info: { type: Object, of: merkletree_info }, - zona: Number, - secao: Number, - eleicoes: [Number], }); +schema_election_data.index({ "file_name": 1 }); schema_election_data.index({ "merkletree_info.$**": 1 }); module.exports = { schema_election_data } \ No newline at end of file diff --git a/backend/public/src/database/repository/election-data.repository.js b/backend/public/src/database/repository/election-data.repository.js index 3279a7f4..dd27a65c 100644 --- a/backend/public/src/database/repository/election-data.repository.js +++ b/backend/public/src/database/repository/election-data.repository.js @@ -2,8 +2,8 @@ const mongoose = require("mongoose"); const { schema_election_data } = require("../models/election-data.model") -exports.findById = async (data_name, id) => { - const repository = mongoose.model(data_name, schema_election_data, data_name); +exports.findById = async (tree_name, id) => { + const repository = mongoose.model(tree_name, schema_election_data, tree_name); return await repository.findOne({ _id: id }) .then((data) => { data._doc.data = Buffer.from(data.data, 'base64').toString('base64') @@ -14,8 +14,8 @@ exports.findById = async (data_name, id) => { }) } -exports.findByMerkletreeIndexRange = async (data_name, initial_index, final_index) => { - const repository = mongoose.model(data_name, schema_election_data, data_name); +exports.findByMerkletreeIndexRange = async (tree_name, initial_index, final_index) => { + const repository = mongoose.model(tree_name, schema_election_data, tree_name); return await repository.find({ "merkletree_info.index": { $gte: parseInt(initial_index), $lte: parseInt(final_index) } }, { data: 0 }) .then((data) => { return data; @@ -23,4 +23,16 @@ exports.findByMerkletreeIndexRange = async (data_name, initial_index, final_inde .catch((err) => { console.error(`[ERROR][election-data.repository] ${err}`); }); -}; \ No newline at end of file +}; + +exports.findByFilename = async (tree_name, filename) => { + const repository = mongoose.model(tree_name, schema_election_data, tree_name); + return await repository.findOne({ file_name: filename }) + .then((data) => { + data._doc.data = Buffer.from(data.data, 'base64').toString('base64') + return data + }) + .catch((err) => { + console.error(`[ERROR][election-data.repository] ${err}`) + }) +} \ No newline at end of file