diff --git a/README.md b/README.md index bfe220d..8b2e494 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,11 @@ The first rectangle on the left is the vehicle data broadcast server, it's alrea ### Data Storage (To be build) After data is pushed to NATS it will be available for other services to listen. Now comes the part where you will have to start develop. Data that is broadcast-ed to NATS is not persisted, it means that we can not access historical data (such as data from past weeks) your task is to build a data storage server that will store all data in [MongoDB](https://www.mongodb.com/) and then serve it via an HTTP REST API and a WebSocket server for live data. So to summarize it here by the checklist of task you need to do: - - [ ] Create MongoDB database - - [ ] Push data from NATS to MongoDB - - [ ] Create REST API - - [ ] Create WebSocket API - - [ ] Test all APIs + - [x] Create MongoDB database + - [x] Push data from NATS to MongoDB + - [x] Create REST API + - [x] Create WebSocket API + - [x] Test ~~all~~ some APIs - [ ] Create Docker container for app (Optional) ### Incident Reporting (Optional) diff --git a/app/dbModels/incidentModel.js b/app/dbModels/incidentModel.js new file mode 100644 index 0000000..0cebe18 --- /dev/null +++ b/app/dbModels/incidentModel.js @@ -0,0 +1,8 @@ +const mongoose = require("mongoose"); +const incidentEntry = new mongoose.Schema({ + data : {} + } +,{timestamps : true}); +const IncidentEntry = mongoose.model("Incident", incidentEntry); + +module.exports = IncidentEntry; \ No newline at end of file diff --git a/app/dbModels/routeInformationModel.js b/app/dbModels/routeInformationModel.js new file mode 100644 index 0000000..57e7dcb --- /dev/null +++ b/app/dbModels/routeInformationModel.js @@ -0,0 +1,15 @@ +const mongoose = require("mongoose"); +const routeInformationEntry = new mongoose.Schema({ + time: {type : Date, required: true}, + energy: {type : mongoose.Decimal128, required : true}, + gps: {type : [String], required : true}, + odo: {type : mongoose.Decimal128, required : true}, + speed: {type : Number, required : true}, + soc: {type : mongoose.Decimal128, required : true}, + vehicleName : {type : String, required : true}, + error : {} + } +,{timestamps : true}); +const RouteInformationEntry = mongoose.model("RouteInformation", routeInformationEntry); + +module.exports = RouteInformationEntry; \ No newline at end of file diff --git a/app/server.js b/app/server.js new file mode 100644 index 0000000..00d059d --- /dev/null +++ b/app/server.js @@ -0,0 +1,133 @@ +// Database logic +const mongoose = require('mongoose') +const RouteInformationEntry = require('./dbModels/routeInformationModel.js') +const IncidentEntry = require('./dbModels/incidentModel.js') +const dbLocation = "mongodb://localhost/viricitiTech" +/* Note: +In production app we would hide locations like these and portinformation in enviroment variables stored in seperate files. +The elements of the server: Database logic, socket logic, routing logic and NATS logic would also be seperated. +*/ +mongoose.connect(dbLocation, async (err,res)=>{ + if(err){ + throw(err) + }else{ + console.log('Succesfully connected to database') + // Clean up collection if it gets too big + const routeInformationCount = await RouteInformationEntry.count() + if(routeInformationCount > 3000){ + RouteInformationEntry.collection.drop() + IncidentEntry.collection.drop() + console.log('Emptied the RouteInformation and Incident Collection') + } + } +}) + + +// REST Logic +const express = require('express') +const app = express() +app.use(express.json()); +app.use(express.urlencoded({extended:true})); +app.set("view engine", "ejs"); +const http = require('http').createServer(app) +const port = 2021 +http.listen(port, ()=>{ + console.log(`Listening on *:${port}`); +}); +app.get("/", async(req,res)=>{ + res.render('../app/views/routeInformation.ejs') +}) +app.get("/incidents", async(req,res)=>{ + const incidents = await IncidentEntry.find() + res.render('../app/views/incidents.ejs', {incidents : incidents}) +}) +app.get("/incidentLookup", async (req,res)=>{ + res.render('../app/views/incidentLookup.ejs', {incidents : null}) +}) +app.post("/findIncidents", async (req,res)=>{ + const dbFilter = getRouteFilter(req.body, true) + const foundIncidents = await IncidentEntry.find(dbFilter).catch((err)=>{console.log(err)}) + if(req.body.type === 'api'){ + res.send({incidents : foundIncidents}) + }else{ + res.render('../app/views/incidents.ejs', {incidents : foundIncidents}) + } +}) +app.get("/routeInformationLookup", async (req,res)=>{ + res.render('../app/views/routeInformationLookup.ejs', {routeEntries : null}) +}) +app.post("/findRouteEntries", async (req,res)=>{ + const dbFilter = getRouteFilter(req.body) + const foundRouteInformation = await RouteInformationEntry.find(dbFilter).catch((err)=>{console.log(err)}) + if(req.body.type === 'api'){ + res.send({routeEntries : foundRouteInformation}) + }else{ + res.render('../app/views/viewRouteInformation.ejs', {routeEntries : foundRouteInformation}) + } +}) +// Websocket Logic +const io = require('socket.io')(http); +io.on('connection', (socket) => { + let date = new Date (socket.handshake.time) + console.log(`Connection to socket made at ${date} with ${socket.handshake.headers["x-real-ip"]}. ${io.engine.clientsCount} open connection(s).`) + socket.emit('connectedToServer', true) +}) + +// Recieving and dealing with incoming data from NATS +const NATS = require('nats') +const nats = NATS.connect({json : true}) + +const listenForVehicleData = (vehicleName) => { + nats.subscribe(`vehicle.${vehicleName}`, async (data)=>{ + data.vehicleName = vehicleName + try{ + let addNewEntry = await RouteInformationEntry.create(data) + io.emit('vehicleMessage', data) + }catch(err){ + incidentHandling(err, data) + } + }) +} + +const incidentHandling = (data, dataEntered) =>{ + data.error = true + io.emit('vehicleError', dataEntered) + IncidentEntry.create({data : dataEntered}) +} +const getRouteFilter = (routeCriteria, lookForIncident)=>{ + let queryKeys = Object.keys(routeCriteria) + let dbFilter = {} + if(routeCriteria.time[0] !== 'ignore'){ + let newDate = new Date(routeCriteria.time[1]) + routeCriteria.time[1] = newDate.getTime() + } + queryKeys.forEach(key=>{ + let queryItem = routeCriteria[key] + let dbKey = key + if(dbKey.type === 'api'){ + return + } + if(lookForIncident){ + dbKey = `data.${key}` + } + if(queryItem[2]){ + dbFilter[`data.${key}`] = {$eq : ''} + }else if(queryItem[0] === 'ignore'){ + }else if(queryItem[0]=== 'gt'){ + dbFilter[dbKey] = {$gt : parseFloat(queryItem[1])} + }else if(queryItem[0]=== 'lt'){ + dbFilter[dbKey] = {$lt : parseFloat(queryItem[1])} + }else if(queryItem[0]=== '=='){ + dbFilter[dbKey] = {$eq : parseFloat(queryItem[1])} + } + }) + return dbFilter +} + +// +// ================================= +// + +listenForVehicleData('test-bus-1') + +module.exports = app.listen(2022) \ No newline at end of file diff --git a/app/views/incidentLookup.ejs b/app/views/incidentLookup.ejs new file mode 100644 index 0000000..4a8b249 --- /dev/null +++ b/app/views/incidentLookup.ejs @@ -0,0 +1,176 @@ + + + + + + + + Incident Report - Viriciti Tech Assesment Robbert-Jan Sebregts + + + + + +
+
+
Incident Report
+
+ See Most Current Data + Find Specific Route Information + See All Incidents +
+
+
+
+ Find Incidents +
+
+
+ + + + +
+ +
+
+
+ + + + +
+ +
+
+
+ + + + +
+ +
+
+
+ + + + +
+ +
+
+
+ + + + +
+ +
+
+ +
+
+ + <%if(incidents){%> + + + + + + + + + + + + + + <%if(incidents){ + incidents.forEach((incident,index) =>{%> + + + + + + + + + + <%})}%> + +
🕑 Time🚌 Vehicle Name🔌 Energy Usage🌎 GPS📏 Distance🏁 Speed🔋 Soc
+ <% + const date = new Date(incident.data.time) + const dateString = `${date.getFullYear()}-${date.getMonth()}-${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}` + %> + <%=dateString%> + + <%=incident.data.vehicleName%> + + <%=incident.data.energy%> + + <%=incident.data.gps%> + + <%=incident.data.odo%> + + <%=incident.data.speed%> + + <%=incident.data.soc%> +
+ <%}%> + +
+ + + + + + + + \ No newline at end of file diff --git a/app/views/incidents.ejs b/app/views/incidents.ejs new file mode 100644 index 0000000..0c9c9ad --- /dev/null +++ b/app/views/incidents.ejs @@ -0,0 +1,110 @@ + + + + + + + + Incident Report - Viriciti Tech Assesment Robbert-Jan Sebregts + + + + + +
+
+
Incident Report
+
+ See Most Current Data +
+
*New Incident
+
+ + + + + + + + + + + + + + <%if(incidents && incidents.length){ + incidents.forEach((incident,index) =>{%> + + + + + + + + + + <%})}%> + +
🕑 Time🚌 Vehicle Name🔌 Energy Usage🌎 GPS📏 Distance🏁 Speed🔋 Soc
+ <% + const date = new Date(incident.data.time) + const dateString = `${date.getFullYear()}-${date.getMonth()+1}-${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}` + %> + <%=dateString%> + + <%=incident.data.vehicleName%> + + <%=incident.data.energy%> + + <%=incident.data.gps%> + + <%=incident.data.odo%> + + <%=incident.data.speed%> + + <%=incident.data.soc%> +
+
+ + + + + + + + + \ No newline at end of file diff --git a/app/views/routeInformation.ejs b/app/views/routeInformation.ejs new file mode 100644 index 0000000..c158b7d --- /dev/null +++ b/app/views/routeInformation.ejs @@ -0,0 +1,121 @@ + + + + + + + + Route Information - Viriciti Tech Assesment Robbert-Jan Sebregts + + + + + +
+
+
+
Latest Route Information
+
+ Find specific routes +
+
+ See All Incidents + Specific Incidents +
+ +
+
+
+
+ +
+ + + + + + + + + + + + + +
🕑 Time🔌 Energy Usage🌎 GPS📏 Distance🏁 Speed🔋 Soc
+
+
+ +
+ + + + + + + + \ No newline at end of file diff --git a/app/views/routeInformationLookup.ejs b/app/views/routeInformationLookup.ejs new file mode 100644 index 0000000..65031da --- /dev/null +++ b/app/views/routeInformationLookup.ejs @@ -0,0 +1,110 @@ + + + + + + + + Incident Report - Viriciti Tech Assesment Robbert-Jan Sebregts + + + + + +
+
+
Find Routes
+
+ See Most Current Data + See All Incidents + Find specific Incidents +
+
+
+
+ Find Route Information +
+
+
+ + + +
+
+ + + + +
+
+ + + + +
+
+ + + +
+
+ + + +
+ +
+
+ +
+ + + + + + + + \ No newline at end of file diff --git a/app/views/viewRouteInformation.ejs b/app/views/viewRouteInformation.ejs new file mode 100644 index 0000000..10206d1 --- /dev/null +++ b/app/views/viewRouteInformation.ejs @@ -0,0 +1,89 @@ + + + + + + + + Route Information - Viriciti Tech Assesment Robbert-Jan Sebregts + + + + + +
+
+
+
You asked for this Route Information
+
We found + <%=routeEntries.length%> entries
+
+ See Latest Route Information + See All Incidents + Specific Incidents +
+ +
+
+
+
+ +
+ + + + + + + + + + + + + + <%if(routeEntries && routeEntries.length){ + routeEntries.forEach((route,index) =>{%> + + + + + + + + + + <%})}%> + +
🕑 Time🚌 Vehicle Name🔌 Energy Usage🌎 GPS📏 Distance🏁 Speed🔋 Soc
+ <% + const date = new Date(route.time) + const dateString = `${date.getFullYear()}-${date.getMonth()+1}-${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}` + %> + <%=dateString%> + + <%=route.vehicleName%> + + <%=route.energy%> + + <%=route.gps%> + + <%=route.odo%> + + <%=route.speed%> + + <%=route.soc%> +
+
+
+ +
+ + + + \ No newline at end of file diff --git a/package.json b/package.json index a8ac472..a1025fc 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,25 @@ { - "name": "nodejs-assignment", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "start-nats": "sh ./start-nats", - "start-broadcast": "node vehicle-data-generator/index.js" - }, - "author": "", - "license": "ISC", - "dependencies": { - "csv-parse": "^3.2.0", - "nats": "^1.0.1" - } + "name": "nodejs-assignment", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "start-nats": "sh ./start-nats", + "start-broadcast": "node vehicle-data-generator/index.js", + "connect-db": "node app/server.js", + "test": "mocha spec/*.js --exit -t 15000" + }, + "author": "", + "license": "ISC", + "dependencies": { + "chai": "^4.3.4", + "chai-http": "^4.3.0", + "csv-parse": "^3.2.0", + "ejs": "^3.1.6", + "express": "^4.17.1", + "mocha": "^9.1.3", + "mongoose": "^6.0.13", + "nats": "^1.0.1", + "socket.io": "^4.3.2" + } } diff --git a/spec/server.js b/spec/server.js new file mode 100644 index 0000000..e6bced1 --- /dev/null +++ b/spec/server.js @@ -0,0 +1,215 @@ +const chai = require('chai'); +const chaiHttp = require('chai-http'); +const server = require('../app/server'); +// Assertion Style +chai.should() +chai.use(chaiHttp) + + +// Server Test +describe('http server is running', ()=>{ + it('should return status OK', (done)=>{ + chai + .request(server) + .get('/') + .end((err,res)=>{ + res.should.have.status(200) + done() + }) + }) + it('should return 404', (done)=>{ + chai + .request(server) + .get('/asdfasdfasf') + .end((err,res)=>{ + res.should.have.status(404) + }) + done() + }) +}) + + +// Post Route Entries Test +describe('Posting /findRouteEntries', ()=>{ + it('should not return any route documents for absurd values', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'gt', 100000000 ], + odo: [ 'gt', 1000000000 ], + speed: [ 'gt', 1000 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findRouteEntries/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('routeEntries').lengthOf(0) + done() + }) + }) + it('should return only route documents speed > 20', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ 'gt', 20 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findRouteEntries/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('routeEntries') + res.body.routeEntries.forEach(entry =>{ + entry.should.have.property('speed').above(20) + }) + done() + }) + }) + it('should return only route documents speed < 20', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ 'lt', 20 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findRouteEntries/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('routeEntries') + res.body.routeEntries.forEach(entry =>{ + entry.should.have.property('speed').below(20) + }) + done() + }) + }) + it('should return only route documents speed = 12', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ '==', 12 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findRouteEntries/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('routeEntries') + res.body.routeEntries.forEach(entry =>{ + entry.should.have.property('speed').equals(12) + }) + done() + }) + }) +}) +// Post Incidents Test +describe('Posting /findIncidents', ()=>{ + it('should not return any incident documents for absurd values', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'gt', 100000000 ], + odo: [ 'gt', 1000000000 ], + speed: [ 'gt', 1000 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findIncidents/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('incidents').lengthOf(0) + done() + }) + }) + it('should return only incident documents speed > 20', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ 'gt', 20 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findIncidents/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('incidents') + res.body.incidents.forEach(entry =>{ + entry.should.have.property('speed').above(20) + }) + done() + }) + }) + it('should return only incident documents speed < 20', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ 'lt', 20 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findIncidents/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('incidents') + res.body.incidents.forEach(entry =>{ + entry.should.have.property('speed').below(20) + }) + done() + }) + }) + it('should return only incident documents speed = 12', (done)=>{ + const postBody = { + time: [ 'ignore', '2017-11-23T12:20' ], + energy: [ 'ignore', '' ], + odo: [ 'ignore', '' ], + speed: [ '==', 12 ], + soc: [ 'ignore', '' ], + type : 'api' + } + chai + .request(server) + .post('/findIncidents/') + .send(postBody) + .end((err,res)=>{ + res.should.have.status(200) + res.body.should.be.a('object') + res.body.should.have.property('incidents') + res.body.incidents.forEach(entry =>{ + entry.should.have.property('speed').equals(12) + }) + done() + }) + }) +}) \ No newline at end of file diff --git a/vehicle-data-generator/index.js b/vehicle-data-generator/index.js index 97db849..eb63e25 100644 --- a/vehicle-data-generator/index.js +++ b/vehicle-data-generator/index.js @@ -32,7 +32,15 @@ const readOutLoud = (vehicleName) => { // What's the difference betweeen fs.createReadStream, fs.readFileSync, and fs.readFileAsync? // And when to use one or the others // ========================= - + // Answer: + // fs.createReadStream will process data in chunks of a set size. + // This keeps the memory load smaller. Since it deals with data in small chunks you can start processing the data early. + // Freeing up memory for a readStream is more complex (for the system, that is). + // fs.readFileAsync will asynchronously add the entire file to memory. + // fs.readFileSync will synchronously add the entire file to memory, and thus will block the eventloop. This is less ideal for larger files. + // fs.readFileSync and fs.readFileAsync will put the entire file in memory. This makes it easy for Node to remove them from memory, but it will also take a lot of memory. + // Because fs.createReadStream works with smaller chunks of data your data will most likely be available for use quicker. + // fs.createReadStream is best used with more frequent requests and larger files. fs.readFile // Now comes the interesting part, // Handling this filestream requires us to create pipeline that will transform the raw string // to object and sent out to nats @@ -53,8 +61,8 @@ const readOutLoud = (vehicleName) => { // setTimeout in this case is there to emulate real life situation // data that came out of the vehicle came in with irregular interval // Hence the Math.random() on the second parameter + const connectionDelay = Math.ceil(Math.random()*10) setTimeout(() => { - i++ if((i % 100) === 0) console.log(`vehicle ${vehicleName} sent have sent ${i} messages`) @@ -63,8 +71,9 @@ const readOutLoud = (vehicleName) => { // it also includes the vehicle name to seggregate data between different vehicle nats.publish(`vehicle.${vehicleName}`, obj, cb) - - }, Math.ceil(Math.random() * 150)) + obj.vehicleName = vehicleName; + routeArray.push(obj) + }, Math.ceil(Math.random() * 150) * connectionDelay) } }))) // ========================= @@ -72,12 +81,33 @@ const readOutLoud = (vehicleName) => { // What would happend if it failed to publish to nats or connection to nats is slow? // Maybe you can try to emulate those slow connection // ========================= + // } // This next few lines simulate Henk's (our favorite driver) shift console.log("Henk checks in on test-bus-1 starting his shift...") readOutLoud("test-bus-1") .once("finish", () => { - console.log("henk is on the last stop and he is taking a cigarrete while waiting for his next trip") + console.log("Henk is on the last stop and he is taking a cigarrete while waiting for his next trip") + console.log("Henk is ready to go again!") + if(routeArray.length){ + const reversedRoute = routeArray.reverse() + readReverse(reversedRoute) + } }) -// To make your presentation interesting maybe you can make henk drive again in reverse \ No newline at end of file +// To make your presentation interesting maybe you can make henk drive again in reverse +let routeArray = [] +const readReverse = (array) =>{ + const publishRouteInfo = (i)=>{ + const data = array[i] + setTimeout( + function(){ + nats.publish(`vehicle.${data.vehicleName}`, data) + if((i % 100) === 0) + console.log(`vehicle ${data.vehicleName} sent have sent ${i} reversed messages`) + }, (i*100) + Math.ceil(Math.random() * 150)) + } + for (let i = 1; i < array.length; i++) { + publishRouteInfo(i) + } +} \ No newline at end of file