Feat/import collection

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- [#119](https://github.com/mia-platform/crud-service/pull/119) created route to import collection files (json, ndjson and csv)
+
+## 6.8.0 - 2023-07-11
+
+### Added
+
 - [#129](https://github.com/mia-platform/crud-service/pull/129) introduce the option to enable strict validation on service responses
 
 ### Fixed

Dockerfile

@@ -42,7 +42,7 @@ LABEL maintainer="Mia Platform Core Team<core@mia-platform.eu>" \
     name="CRUD Service" \
     description="HTTP interface to perform CRUD operations on configured MongoDB collections" \
     eu.mia-platform.url="https://www.mia-platform.eu" \
-    eu.mia-platform.version="6.8.0-rc.1"
+    eu.mia-platform.version="6.8.0"
 
 ENV NODE_ENV=production
 ENV LOG_LEVEL=info

envSchema.js

@@ -58,6 +58,12 @@ const properties = {
     description: 'Enable CRUD responses to be compliant with the schema (Changing the schema without sanitizing the data could break GETs)',
     default: false,
   },
+  MAX_MULTIPART_FILE_BYTES: {
+    type: 'number',
+    description: 'The max size (Mb) that is possible to process in multipart requests',
+    default: 100,
+    minimum: 1,
+  },
   TRUSTED_PROXIES: { type: 'string', default: '10.0.0.0/8,172.16.0.0/12,192.168.0.0/16' },
   KMS_PROVIDER: { type: 'string', enum: ['gcp', 'local', 'none'], description: 'Master key manager', default: 'none' },
   ALLOW_DISK_USE_IN_QUERIES: {

index.js

@@ -19,6 +19,7 @@
 
 const fp = require('fastify-plugin')
 const fastifyEnv = require('@fastify/env')
+const fastifyMultipart = require('@fastify/multipart')
 
 const Ajv = require('ajv')
 const ajvFormats = require('ajv-formats')
@@ -436,8 +437,16 @@ async function setupCruds(fastify) {
 }
 
 module.exports = async function plugin(fastify, opts) {
-  fastify
+  await fastify
     .register(fastifyEnv, { schema: fastifyEnvSchema, data: opts })
+  fastify.register(fastifyMultipart, {
+    limits: {
+      fields: 5,
+      // Conversion Byte to Mb
+      fileSize: fastify.config.MAX_MULTIPART_FILE_BYTES * 1000000,
+      files: 1,
+    },
+  })
     .register(fp(setupCruds, { decorators: { fastify: ['config'] } }))
 }
 

lib/BatchWritableStream.js

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2023 Mia s.r.l.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict'
+const { Writable } = require('stream')
+
+class BatchWritableStream extends Writable {
+  constructor(options) {
+    super(options)
+
+    if (!options.processBatch || options.processBatch[Symbol.toStringTag] !== 'AsyncFunction') {
+      throw new Error('BatchWritableStream requires an async "processBatch" function')
+    }
+
+    this.processBatch = options.processBatch
+    this.batchSize = options.batchSize || 1000
+    this.flushBatch()
+  }
+
+  flushBatch() {
+    this.batch = []
+  }
+
+  _write(chunk, _encoding, callback) {
+    this.batch.push(chunk)
+    if (this.batch.length >= this.batchSize) {
+      this.processBatch(this.batch)
+        .then(() => {
+          this.flushBatch()
+          callback()
+        })
+        .catch((error) => callback(error))
+    } else {
+      return callback()
+    }
+  }
+
+  _final(callback) {
+    if (this.batch.length > 0) {
+      this.processBatch(this.batch)
+        .then(() => {
+          this.flushBatch()
+          callback()
+        })
+        .catch((error) => callback(error))
+    } else {
+      return callback()
+    }
+  }
+}
+
+module.exports = BatchWritableStream

lib/JSONSchemaGenerator.js

@@ -47,6 +47,10 @@ const {
   SCHEMA_CUSTOM_KEYWORDS,
   DATE_FORMATS,
   PULLCMD,
+  UPDATERID,
+  UPDATEDAT,
+  CREATORID,
+  CREATEDAT,
 } = require('./consts')
 const {
   stateCreateValidationSchema,
@@ -205,11 +209,85 @@ module.exports = class JSONSchemaGenerator {
     }
   }
 
+  generateImportJSONSchema() {
+    const schemaDetail = this.getSchemaDetail(SCHEMAS_ID.POST_FILE)
+    const requiredFields = this._requiredFields
+
+    return {
+      summary: `Insert new items in the ${this._collectionName} collection by input file`,
+      tags: this._collectionTags,
+      consumes: ['multipart/form-data'],
+      body: {
+        ...schemaDetail.body,
+        anyOf: [
+          {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              file: {
+                type: 'file',
+                description: 'the supported content-type are: application/x-ndjson, application/json and text/csv',
+              },
+            },
+          },
+          {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              file: {
+                type: 'file',
+                description: 'the supported content-type are: application/x-ndjson, application/json and text/csv',
+              },
+              encoding: {
+                type: 'string',
+                enum: ['utf8', 'ucs2', 'utf16le', 'latin1', 'ascii', 'base64', 'hex'],
+                description: 'CSV: The encoding to use to parse the file',
+              },
+              delimiter: {
+                type: 'string',
+                minLength: 1,
+                maxLength: 10,
+                description: 'CSV: The delimiter to use to parse the file',
+              },
+              escape: {
+                type: 'string',
+                minLength: 1,
+                maxLength: 10,
+                description: 'CSV: The escape to use to parse the file',
+              },
+            },
+          },
+        ],
+      },
+      response: {
+        200: {
+          type: 'object',
+          properties: {
+            message: {
+              type: 'string',
+            },
+          },
+        },
+      },
+      streamBody: {
+        type: 'object',
+        ...schemaDetail.body,
+        ...requiredFields.length > 0 ? { required: requiredFields } : {},
+        properties: {
+          ...this._propertiesPostValidation,
+          ...mandatoryFieldsWithoutId(true),
+          [MONGOID]: { ...mongoIdTypeValidator[this._idType]() },
+        },
+        additionalProperties: false,
+      },
+    }
+  }
+
   generateGetListLookupJSONSchema() {
     const patternProperties = getQueryStringFromRawSchema(this._pathFieldsRawSchema.patternProperties)
     const schemaDetail = this.getSchemaDetail(SCHEMAS_ID.GET_LIST_LOOKUP)
 
-    const unsupportedQueryParams = ['creatorId', 'createdAt', 'updaterId', 'updatedAt', '_rawp']
+    const unsupportedQueryParams = [UPDATERID, UPDATEDAT, CREATORID, CREATEDAT, RAW_PROJECTION]
 
     const supportMongoID = this._collectionDefinition?.schema?.properties[MONGOID]
     || this._collectionDefinition?.fields?.findIndex((field) => field.name === MONGOID) > -1