Feat/logs and signature

backend/bu_service/.env

@@ -1,4 +1,4 @@
-TREE_NAME_PREFIX=eleicao_
+BU_TREE_NAME_TEMPLATE=bus_eleicao_${ELECTION_ID}
 TREE_DEFAULT_COMMITMENT_SIZE=8
 TL_MANAGER_URL=http://127.0.0.1:8000
 MONGO_URL=mongodb://localhost:27017

backend/bu_service/app/adapter/tlmanager_adapter.py

@@ -1,11 +1,13 @@
-import requests
 import os
+
 import dotenv
+import requests
 
 dotenv.load_dotenv()
 
 TL_MANAGER_URL = os.getenv('TL_MANAGER_URL')
 
+
 def get_trees():
     return requests.get(f'{TL_MANAGER_URL}/').json()
 

backend/bu_service/app/config/environment.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+BU_TREE_NAME_TEMPLATE = os.getenv('BU_TREE_NAME_TEMPLATE')
+TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE')
+TL_MANAGER_URL = os.getenv('TL_MANAGER_URL')
+MONGO_URL = os.getenv('MONGO_URL')
+
+if '${ELECTION_ID}' not in BU_TREE_NAME_TEMPLATE:
+    raise Exception("BU_TREE_NAME_TEMPLATE must contain ${ELECTION_ID}")
+
+
+logging.debug("BU_TREE_NAME_TEMPLATE: %s", BU_TREE_NAME_TEMPLATE)
+logging.debug("TL_MANAGER_URL: %s", TL_MANAGER_URL)
+logging.debug("MONGO_URL: %s", MONGO_URL)
+logging.debug("TREE_DEFAULT_COMMITMENT_SIZE: %s", TREE_DEFAULT_COMMITMENT_SIZE)

backend/bu_service/app/controllers/bu_controller.py

@@ -1,4 +1,5 @@
 import logging
+
 from fastapi import APIRouter, UploadFile, File
 
 from app.services import bu_service
@@ -12,5 +13,5 @@ async def create(file: UploadFile = File(...)):
     file_content = await file.read()
 
     logging.info(f"Received file {file_name}")
-    bu_service.insert(file_content, file_name)
-    return {"status": "ok"}
+    bu_parsed = bu_service.insert(file_content, file_name)
+    return {"status": "ok", "bu": bu_parsed.dict(exclude={"bu"})}

backend/bu_service/app/controllers/election_data_controller.py

@@ -0,0 +1,26 @@
+import logging
+
+from fastapi import APIRouter, UploadFile, File, Form
+
+from app.services import election_data_service
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        tree_name: str = Form(...),
+        file: UploadFile = File(...)
+):
+    file_name = file.filename
+    file_content = await file.read()
+
+    logging.info(f"Received file {file_name} to insert in tree {tree_name}")
+
+    try:
+        election_data = election_data_service.insert(file_content, file_name, tree_name)
+    except Exception as e:
+        logging.error(e)
+        return {"status": "error", "message": str(e)}
+
+    return {"status": "ok", "bu": election_data.dict(exclude={"data"})}

backend/bu_service/app/database/models/bu_model.py

@@ -1,11 +1,8 @@
 from typing import Optional
-from pydantic import BaseModel
 
+from pydantic import BaseModel
 
-class MerkleTreeInfo(BaseModel):
-    tree_name: str
-    index: int
-    hash: str
+from app.database.models.merkle_tree_info_model import MerkleTreeInfo
 
 
 class BuModel(BaseModel):

backend/bu_service/app/database/models/election_data_model.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+
+from app.database.models.bu_model import MerkleTreeInfo
+
+
+class ElectionData(BaseModel):
+    file_name: str
+    data: bytes
+    merkletree_info: MerkleTreeInfo

backend/bu_service/app/database/models/merkle_tree_info_model.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+class MerkleTreeInfo(BaseModel):
+    tree_name: str
+    index: int
+    hash: str

backend/bu_service/app/database/repositories/bu_repository.py

@@ -4,5 +4,16 @@
 collection = get_db()["bus"]
 
 
+def find_one(election_id: int, zone: int, section: int):
+    data = collection.find_one({"eleicoes": election_id, "zona": zone, "secao": section})
+    if data is None:
+        return None
+    return BuModel(**data)
+
+
 def save(bu: BuModel):
-    collection.insert_one(bu.dict())
+    collection.update_one(
+        {"eleicoes": bu.eleicoes, "zona": bu.zona, "secao": bu.secao},
+        {"$set": bu.model_dump()},
+        upsert=True
+    )

backend/bu_service/app/database/repositories/election_data_repository.py

@@ -0,0 +1,8 @@
+from app.database.connection import get_db
+from app.database.models.election_data_model import ElectionData
+
+db = get_db()
+
+
+def save(data: ElectionData):
+    db[data.merkletree_info.tree_name].insert_one(data.model_dump())

backend/bu_service/app/services/bu_service.py

@@ -1,26 +1,22 @@
+import base64
+import json
 import logging
 
 import asn1tools
-import json
-import base64
-import os
 
+from app.config.environment import BU_TREE_NAME_TEMPLATE
 from app.database.models.bu_model import BuModel, MerkleTreeInfo
 from app.database.repositories.bu_repository import save
-from app.adapter.tlmanager_adapter import *
+from app.services.tlmanager_service import insert_creating_tree_if_not_exists
 
-TREE_NAME_PREFIX = os.getenv('TREE_NAME_PREFIX')
-TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE')
-
-conv = asn1tools.compile_files("app/services/bu.asn1")
+conv = asn1tools.compile_files("app/services/static/bu.asn1")
 county_codes = json.load(open("app/services/static/county_codes.json", encoding='utf-8'))
 
 
 class DictWithBytesToJsonEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, (bytes, bytearray)):
             return obj.hex()
-        # Let the base class default method raise the TypeError
         return json.JSONEncoder.default(self, obj)
 
 
@@ -73,27 +69,16 @@ def insert(file_content: bytes, filename: str):
     bu_parsed = parse_bu(bu_decoded, file_content, filename)
     logging.debug(bu_parsed)
 
-    for eleicao in bu_parsed.eleicoes:
-        logging.info(f"Inserting leaf for election {eleicao}")
-        response = insert_leaf(f'{TREE_NAME_PREFIX}{eleicao}', base64.b64encode(bu_parsed.bu).decode('utf-8'))
-        logging.info(response.json())
-        if response.status_code != 200 and response.json().get('message') == "Tree does not exist":
-            logging.info(f"Creating tree for election {eleicao}")
-            create_tree(f'{TREE_NAME_PREFIX}{eleicao}', TREE_DEFAULT_COMMITMENT_SIZE)
-            logging.info(response.json())
-
-            logging.info(f"Trying to insert leaf for election {eleicao} again")
-            response = insert_leaf(f'{TREE_NAME_PREFIX}{eleicao}', base64.b64encode(bu_parsed.bu).decode('utf-8'))
-            logging.info(response.json())
-            if response.status_code != 200:
-                raise Exception("Failed to insert leaf")
-        bu_parsed.merkletree_info[str(eleicao)] = MerkleTreeInfo(
-            tree_name=f'{TREE_NAME_PREFIX}{eleicao}',
+    for id_eleicao in bu_parsed.eleicoes:
+        tree_name = BU_TREE_NAME_TEMPLATE.replace('${ELECTION_ID}', str(id_eleicao))
+        logging.info(f"Inserting leaf for election {id_eleicao}")
+        response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(bu_parsed.bu).decode('utf-8'))
+        bu_parsed.merkletree_info[str(id_eleicao)] = MerkleTreeInfo(
+            tree_name=tree_name,
             index=response.json()['index'],
             hash=response.json()['value'])
 
     logging.debug("Saving BU to database")
     logging.debug(bu_parsed)
     save(bu_parsed)
-
-
+    return bu_parsed

backend/bu_service/app/services/election_data_service.py

@@ -0,0 +1,25 @@
+import base64
+import logging
+
+from app.database.models.election_data_model import ElectionData
+from app.database.models.merkle_tree_info_model import MerkleTreeInfo
+from app.database.repositories.election_data_repository import save
+from app.services.tlmanager_service import insert_creating_tree_if_not_exists
+
+
+def insert(file_content: bytes, filename: str, tree_name: str):
+    response = insert_creating_tree_if_not_exists(tree_name, base64.b64encode(file_content).decode('utf-8'))
+
+    election_data = ElectionData(
+        file_name=filename,
+        data=file_content,
+        merkletree_info=MerkleTreeInfo(
+            tree_name=tree_name,
+            index=response.json()['index'],
+            hash=response.json()['value'])
+    )
+
+    logging.debug("Saving election data to database")
+    save(election_data)
+
+    return election_data

backend/bu_service/app/services/tlmanager_service.py

@@ -0,0 +1,22 @@
+import logging
+
+from app.adapter.tlmanager_adapter import *
+
+TREE_DEFAULT_COMMITMENT_SIZE = os.getenv('TREE_DEFAULT_COMMITMENT_SIZE')
+
+
+def insert_creating_tree_if_not_exists(tree_name, data, commitment_size=TREE_DEFAULT_COMMITMENT_SIZE):
+    insert_leaf_response = insert_leaf(tree_name, data)
+    logging.info(insert_leaf_response.json())
+    if insert_leaf_response.status_code != 200 and insert_leaf_response.json().get('message') == "Tree does not exist":
+        logging.info(f"Creating tree with name {tree_name}")
+        create_tree(tree_name, commitment_size)
+
+        logging.info(f"Trying to insert leaf for tree {tree_name} again")
+        insert_leaf_response = insert_leaf(tree_name, data)
+        logging.info(insert_leaf_response.json())
+
+        if insert_leaf_response.status_code != 200:
+            raise Exception("Failed to insert leaf")
+
+    return insert_leaf_response

backend/bu_service/main.py

@@ -1,15 +1,15 @@
-import dotenv
-import logging
 from fastapi import FastAPI
 
-from app.controllers import tree_controller, bu_controller
+from app.config.environment import *
+from app.controllers import tree_controller, bu_controller, election_data_controller
 
-dotenv.load_dotenv()
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.DEBUG)
 
 app = FastAPI()
 app.include_router(tree_controller.router, prefix="/tree", tags=["Tree management routes"])
 app.include_router(bu_controller.router, prefix="/bu", tags=["BU management routes"])
+app.include_router(election_data_controller.router, prefix="/election_data",
+                   tags=["Route to insert election data in the TLManager"])
 
 
 @app.get("/")

backend/public/server.js

@@ -12,6 +12,9 @@ app.use("/bu", bu_controller);
 const tree_controller = require("./src/controllers/tree.controller");
 app.use("/tree", tree_controller);
 
+const election_data_controller = require("./src/controllers/election-data.controller");
+app.use("/election-data", election_data_controller);
+
 const public_key_controller = require("./src/controllers/public_key.controller");
 app.use("/public_key", public_key_controller);
 

backend/public/src/controllers/election-data.controller.js

@@ -0,0 +1,57 @@
+const router = require('express').Router();
+const election_data_repository = require('../database/repository/election-data.repository');
+
+router.get("/find_by_merkletree_index_range", async  (req, res) => {
+  const tree_name = req.query.tree_name
+  const initial_index = req.query.initial_index
+  const final_index = req.query.final_index
+
+  console.info(`[election-data.controller] GET /find_by_merkletree_index_range ${tree_name} ${initial_index} ${final_index}`)
+
+  const data = await election_data_repository.findByMerkletreeIndexRange(tree_name, initial_index, final_index)
+
+  res.json(data)
+})
+
+router.get("/find_by_id", async (req, res) => {
+    const id = req.query.id
+    const tree_name = req.query.tree_name
+
+    console.info(`[election-data.controller] GET /find_by_id ${tree_name} ${id}`)
+
+    const data = await election_data_repository.findById(tree_name, id)
+
+    res.json(data)
+})
+
+router.get("/find_by_filename", async (req, res) => {
+    const filename = req.query.filename
+    const tree_name = req.query.tree_name
+
+    console.info(`[election-data.controller] GET /find_by_filename ${tree_name} ${filename}`)
+
+    const data = await election_data_repository.findByFilename(tree_name, filename)
+
+    res.json(data)
+})
+
+router.get("/download", async (req, res) => {
+    const id = req.query.id
+    const tree_name = req.query.tree_name
+
+    console.info(`[election-data.controller] GET /download ${tree_name} ${id}`)
+
+    const data = await election_data_repository.findById(tree_name, id)
+
+    if (!data) {
+      res.status(404).send('Not Found')
+      return
+    }
+
+    const buffer = Buffer.from(data.data, 'base64')
+    res.setHeader('Content-Disposition', `attachment; filename=${data.filename}`)
+
+    res.send(buffer)
+})
+
+module.exports = router;

backend/public/src/database/models/bu.model.js

@@ -1,10 +1,5 @@
 const mongoose = require("mongoose");
-
-const merkletree_info = new mongoose.Schema({
-    tree_name: String,
-    index: Number,
-    hash: String,
-});
+const {merkletree_info} = require("./merkletree-info.model");
 
 const schema_bu = new mongoose.Schema({
     eleicoes: [Number],
@@ -19,11 +14,9 @@ const schema_bu = new mongoose.Schema({
 });
 
 schema_bu.index({ UF:1, zona:1, secao:1 });
-
 schema_bu.index({ eleicoes:1 });
 schema_bu.index({ eleicoes:1, UF:1 });
 schema_bu.index({ eleicoes:1, UF:1, municipio:1 });
-
 schema_bu.index({ "merkletree_info.$**": 1 });
 
 module.exports = { schema_bu }

backend/public/src/database/models/election-data.model.js

@@ -0,0 +1,13 @@
+const mongoose = require("mongoose");
+const {merkletree_info} = require("./merkletree-info.model");
+
+const schema_election_data = new mongoose.Schema({
+    file_name: String,
+    data: Buffer,
+    merkletree_info: { type: Object, of: merkletree_info },
+});
+
+schema_election_data.index({ "file_name": 1 });
+schema_election_data.index({ "merkletree_info.$**": 1 });
+
+module.exports = { schema_election_data }

backend/public/src/database/models/merkletree-info.model.js

@@ -0,0 +1,9 @@
+const mongoose = require("mongoose");
+
+const merkletree_info = new mongoose.Schema({
+    tree_name: String,
+    index: Number,
+    hash: String,
+});
+
+module.exports = { merkletree_info }