diff --git a/README.md b/README.md index fcfbce0..3534450 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ The result will be: Zendro is the product of a joint effort between the Forschungszentrum Jülich, Germany and the Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México, to generate a tool that allows efficiently building data warehouses capable of dealing with diverse data generated by different research groups in the context of the FAIR principles and multidisciplinary projects. The name Zendro comes from the words Zenzontle and Drossel, which are Mexican and German words denoting a mockingbird, a bird capable of “talking” different languages, similar to how Zendro can connect your data warehouse from any programming language or data analysis pipeline. ### Zendro contributors in alphabetical order -Francisca Acevedo1, Vicente Arriaga1, Katja Dohm3, Constantin Eiteneuer2, Sven Fahrner2, Frank Fischer4, Asis Hallab2, Alicia Mastretta-Yanes1, Roland Pieruschka2, Alejandro Ponce1, Yaxal Ponce2, Francisco Ramírez1, Irene Ramos1, Bernardo Terroba1, Tim Rehberg3, Verónica Suaste1, Björn Usadel2, David Velasco2, Thomas Voecking3 +Francisca Acevedo1, Vicente Arriaga1, Katja Dohm3, Constantin Eiteneuer2, Sven Fahrner2, Frank Fischer4, Asis Hallab2, Alicia Mastretta-Yanes1, Roland Pieruschka2, Alejandro Ponce1, Yaxal Ponce2, Francisco Ramírez1, Irene Ramos1, Bernardo Terroba1, Tim Rehberg3, Verónica Suaste1, Björn Usadel2, David Velasco2, Thomas Voecking3, Dan Wang2 #### Author affiliations 1. CONABIO - Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México diff --git a/config/data_models_storage_config.json b/config/data_models_storage_config.json index 7d8295a..15b5b83 100644 --- a/config/data_models_storage_config.json +++ b/config/data_models_storage_config.json @@ -49,5 +49,13 @@ "schema":"public", "presto_host": "127.0.0.1", "presto_port": "8081" + }, + "default-neo4j": { + "storageType": "neo4j", + "username": "neo4j", + "password": "sciencedb", + "database": "neo4j", + "host": "127.0.0.1", + "port": "7687" } } \ No newline at end of file diff --git a/connection.js b/connection.js index 5d92835..7b1521a 100644 --- a/connection.js +++ b/connection.js @@ -4,6 +4,7 @@ const { MongoClient } = require("mongodb"); const AWS = require("aws-sdk"); const presto = require("presto-client"); const cassandraDriver = require("cassandra-driver"); +const neo4j = require("neo4j-driver"); const { queryData } = require("./utils/presto_helper"); const os = require("os"); const Op = Sequelize.Op; @@ -180,6 +181,23 @@ const addConnectionInstances = async () => { engine: "presto", }), }); + } else if ( + storageConfig.hasOwnProperty(key) && + key !== "operatorsAliases" && + storageType === "neo4j" + ) { + // note: https://stackoverflow.com/a/62816512/8132085 + connectionInstances.set(key, { + storageType, + connection: neo4j.driver( + `bolt://${storageConfig[key].host}:${storageConfig[key].port}`, + neo4j.auth.basic( + storageConfig[key].username, + storageConfig[key].password + ), + { disableLosslessIntegers: true } + ), + }); } } return connectionInstances; @@ -228,6 +246,10 @@ exports.checkConnections = async () => { } catch (error) { throw error; } + } else if (instance.storageType === "neo4j") { + await instance.connection.verifyConnectivity({ + database: storageConfig["default-neo4j"].database, + }); } checks.push({ key, valid: true }); } catch (exception) { diff --git a/models/adapters/index.js b/models/adapters/index.js index 75bd0fc..250ddb0 100644 --- a/models/adapters/index.js +++ b/models/adapters/index.js @@ -9,6 +9,7 @@ let adapters = { amazonS3: {}, trino: {}, presto: {}, + neo4j: {}, }; module.exports = adapters; getModulesSync(__dirname).forEach((file) => { @@ -30,6 +31,7 @@ getModulesSync(__dirname).forEach((file) => { case "cassandra-adapter": case "trino-adapter": case "presto-adapter": + case "neo4j-adapter": adapters[adapter.adapterType.split("-")[0]][adapter.adapterName] = adapter.definition; adapters[adapter.adapterName] = adapter; diff --git a/models/index.js b/models/index.js index 9600441..4c50e00 100644 --- a/models/index.js +++ b/models/index.js @@ -9,6 +9,7 @@ let models = { amazonS3: {}, trino: {}, presto: {}, + neo4j: {}, }; const storageTypes = Object.keys(models); module.exports = models; diff --git a/package.json b/package.json index 3f28350..322a905 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "moment": "^2.25.3", "mongodb": "^3.6.3", "mysql2": "^2.1.0", + "neo4j-driver": "^4.2.3", "node-jq": "^1.11.1", "nodemailer": "^6.4.6", "nodemailer-smtp-transport": "^2.7.4", diff --git a/utils/file-tools.js b/utils/file-tools.js index 553964c..7981970 100644 --- a/utils/file-tools.js +++ b/utils/file-tools.js @@ -1,12 +1,11 @@ -const XLSX = require('xlsx'); -const Promise = require('bluebird'); -const promise_csv_parse = Promise.promisify(require('csv-parse')); -const csv_parse = require('csv-parse'); -const fs = require('fs'); -const awaitifyStream = require('awaitify-stream'); -const validatorUtil = require('./validatorUtil'); -const admZip = require('adm-zip'); - +const XLSX = require("xlsx"); +const Promise = require("bluebird"); +const promise_csv_parse = Promise.promisify(require("csv-parse")); +const csv_parse = require("csv-parse"); +const fs = require("fs"); +const awaitifyStream = require("awaitify-stream"); +const validatorUtil = require("./validatorUtil"); +const admZip = require("adm-zip"); /** * replaceNullStringsWithLiteralNulls - Replace null entries of columns with literal null types @@ -14,18 +13,17 @@ const admZip = require('adm-zip'); * @param {array} arrOfObjs Each item correponds to a column represented as object. * @return {array} Each item corresponds to a column and all items have either a valid entry or null type. */ -replaceNullStringsWithLiteralNulls = function(arrOfObjs) { +replaceNullStringsWithLiteralNulls = function (arrOfObjs) { console.log(typeof arrOfObjs, arrOfObjs); - return arrOfObjs.map(function(csvRow) { - Object.keys(csvRow).forEach(function(csvCol) { - csvCell = csvRow[csvCol] - csvRow[csvCol] = csvCell === 'null' || csvCell === 'NULL' ? - null : csvCell - }) + return arrOfObjs.map(function (csvRow) { + Object.keys(csvRow).forEach(function (csvCol) { + csvCell = csvRow[csvCol]; + csvRow[csvCol] = + csvCell === "null" || csvCell === "NULL" ? null : csvCell; + }); return csvRow; }); -} - +}; /** * parseCsv - parse csv file (string) @@ -35,17 +33,16 @@ replaceNullStringsWithLiteralNulls = function(arrOfObjs) { * @param {array|boolean|function} cols Columns as in csv-parser options.(true if auto-discovered in the first CSV line). * @return {array} Each item correponds to a column represented as object and filtered with replaceNullStringsWithLiteralNulls function. */ -exports.parseCsv = function(csvStr, delim, cols) { - if (!delim) delim = "," - if (typeof cols === 'undefined') cols = true +exports.parseCsv = function (csvStr, delim, cols) { + if (!delim) delim = ","; + if (typeof cols === "undefined") cols = true; return replaceNullStringsWithLiteralNulls( promise_csv_parse(csvStr, { delimiter: delim, - columns: cols + columns: cols, }) - ) -} - + ); +}; /** * parseXlsx - description @@ -53,14 +50,13 @@ exports.parseCsv = function(csvStr, delim, cols) { * @param {string} bstr Xlsx file converted to string * @return {array} Each item correponds to a column represented as object and filtered with replaceNullStringsWithLiteralNulls function. */ -exports.parseXlsx = function(bstr) { +exports.parseXlsx = function (bstr) { var workbook = XLSX.read(bstr, { - type: "binary" + type: "binary", }); var sheet_name_list = workbook.SheetNames; return replaceNullStringsWithLiteralNulls( - XLSX.utils.sheet_to_json( - workbook.Sheets[sheet_name_list[0]]) + XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name_list[0]]) ); }; @@ -70,9 +66,9 @@ exports.parseXlsx = function(bstr) { * * @param {String} path - A path to the file */ -exports.deleteIfExists = function(path) { +exports.deleteIfExists = function (path) { console.log(`Removing ${path}`); - fs.unlink(path, function(err) { + fs.unlink(path, function (err) { // file may be already deleted }); }; @@ -85,9 +81,9 @@ exports.deleteIfExists = function(path) { * @return {Object} A modified clone of the argument pojo in which all String * "NULL" or "null" values are deleted. */ -exports.replacePojoNullValueWithLiteralNull = function(pojo) { +exports.replacePojoNullValueWithLiteralNull = function (pojo) { if (pojo === null || pojo === undefined) { - return null + return null; } let res = Object.assign({}, pojo); Object.keys(res).forEach((k) => { @@ -95,10 +91,9 @@ exports.replacePojoNullValueWithLiteralNull = function(pojo) { delete res[k]; } }); - return res + return res; }; - /** * castCsv - Cast values from csv file when converting to object. * Method used in the cast opition for csv-pars, more info: https://csv.js.org/parse/options/cast/ @@ -108,51 +103,51 @@ exports.replacePojoNullValueWithLiteralNull = function(pojo) { * @param {Object} attributes_type Key is the name of the attribute/column as given in the json file of the model, value is the type of the attribute. * @return {any} The value casted according to the attribute type given in attributes_type. */ -castCsv = function( value, column, attributes_type, array_delimiter=";"){ - if(!(typeof value === "string" && value.match(/\s*null\s*/i) )){ - switch ( attributes_type[column] ) { - case 'String': +castCsv = function (value, column, attributes_type, array_delimiter = ";") { + if (!(typeof value === "string" && value.match(/\s*null\s*/i))) { + switch (attributes_type[column]) { + case "String": value = String(value); break; - case 'Int': + case "Int": value = Number(value); break; - case 'Date': + case "Date": value = String(value); break; - case 'Time': + case "Time": value = String(value); break; - case 'DateTime': + case "DateTime": value = String(value); break; - case 'Boolean': - if(value === 'true') value = true; - if(value === 'false') value = false; + case "Boolean": + if (value === "true") value = true; + if (value === "false") value = false; break; - case 'Float': + case "Float": value = Number(value); break; - case '[String]': - value = value.split(array_delimiter) + case "[String]": + value = value.split(array_delimiter); break; - case '[Int]': - value = value.split(array_delimiter).map(x=>parseInt(x)) + case "[Int]": + value = value.split(array_delimiter).map((x) => parseInt(x)); break; - case '[Date]': - value = value.split(array_delimiter) + case "[Date]": + value = value.split(array_delimiter); break; - case '[Time]': - value = value.split(array_delimiter) + case "[Time]": + value = value.split(array_delimiter); break; - case '[DateTime]': - value = value.split(array_delimiter) + case "[DateTime]": + value = value.split(array_delimiter); break; - case '[Boolean]': - value.split(array_delimiter).map(x=> x === 'true') + case "[Boolean]": + value.split(array_delimiter).map((x) => x === "true"); break; - case '[Float]': - value = value.split(array_delimiter).map(x=>parseFloat(x)) + case "[Float]": + value = value.split(array_delimiter).map((x) => parseFloat(x)); break; default: @@ -161,8 +156,7 @@ castCsv = function( value, column, attributes_type, array_delimiter=";"){ } } return value; -} - +}; /** * Parse by streaming a csv file and create the records in the correspondant table @@ -173,26 +167,32 @@ castCsv = function( value, column, attributes_type, array_delimiter=";"){ * @param {array|boolean|function} cols - Columns as in csv-parser options.(true if auto-discovered in the first CSV line). * @param {string} storageType - Set the storage type(default: "sql"). */ -exports.parseCsvStream = async function(csvFilePath, model, delim, cols, storageType = "sql", arrayDelim=";") { - +exports.parseCsvStream = async function ( + csvFilePath, + model, + delim, + cols, + storageType = "sql", + arrayDelim = ";" +) { if (!delim) delim = ","; - if (typeof cols === 'undefined') cols = true; + if (typeof cols === "undefined") cols = true; console.log("TYPEOF", typeof model); // Wrap all database actions within a transaction for sequelize: let transaction; // define mongoDb collection let collection; - if (storageType === "sql"){ + if (storageType === "sql") { transaction = await model.sequelize.transaction(); - } else if (storageType === "mongodb"){ - const db = await model.storageHandler - collection = await db.collection('animal') + } else if (storageType === "mongodb") { + const db = await model.storageHandler; + collection = await db.collection("animal"); } - let addedFilePath = csvFilePath.substr(0, csvFilePath.lastIndexOf(".")) + - ".json"; - let addedZipFilePath = csvFilePath.substr(0, csvFilePath.lastIndexOf(".")) + - ".zip"; + let addedFilePath = + csvFilePath.substr(0, csvFilePath.lastIndexOf(".")) + ".json"; + let addedZipFilePath = + csvFilePath.substr(0, csvFilePath.lastIndexOf(".")) + ".zip"; console.log(addedFilePath); console.log(addedZipFilePath); @@ -204,9 +204,14 @@ exports.parseCsvStream = async function(csvFilePath, model, delim, cols, storage csv_parse({ delimiter: delim, columns: cols, - cast: function( value, context){ - return castCsv(value, context.column, model.definition.attributes, arrayDelim); - } + cast: function (value, context) { + return castCsv( + value, + context.column, + model.definition.attributes, + arrayDelim + ); + }, }) ) ); @@ -222,45 +227,48 @@ exports.parseCsvStream = async function(csvFilePath, model, delim, cols, storage while (null !== (record = await csvStream.readAsync())) { record = exports.replacePojoNullValueWithLiteralNull(record); try { - await validatorUtil.validateData('validateForCreate', model, record); - if (storageType === "sql"){ + await validatorUtil.validateData("validateForCreate", model, record); + if (storageType === "sql") { record = model.preWriteCast(record); - await model.create(record, { - transaction: transaction - }).then(created => { - // this is async, here we just push new line into the parallel thread - // synchronization goes at endAsync; - addedRecords.writeAsync(`${JSON.stringify(created)}\n`); - - }).catch(error => { - console.log( - `Caught sequelize error during CSV batch upload: ${JSON.stringify(error)}` - ); - error.record = record; - errors.push(error); - }) - } else if (storageType === "mongodb"){ + await model + .create(record, { + transaction: transaction, + }) + .then((created) => { + // this is async, here we just push new line into the parallel thread + // synchronization goes at endAsync; + addedRecords.writeAsync(`${JSON.stringify(created)}\n`); + }) + .catch((error) => { + console.log( + `Caught sequelize error during CSV batch upload: ${JSON.stringify( + error + )}` + ); + error.record = record; + errors.push(error); + }); + } else if (storageType === "mongodb") { try { const response = await collection.insertOne(record); addedRecords.writeAsync(`${JSON.stringify(response.ops[0])}\n`); - } catch (error){ + } catch (error) { console.log( - `Caught MongoDb error during CSV batch upload: ${JSON.stringify(error)}` + `Caught MongoDb error during CSV batch upload: ${JSON.stringify( + error + )}` ); error.record = record; errors.push(error); } } - } catch (error) { console.log( `Validation error during CSV batch upload: ${JSON.stringify(error)}` ); - error['record'] = record; + error["record"] = record; errors.push(error); - } - } // close the addedRecords file so it can be sent afterwards @@ -271,19 +279,23 @@ exports.parseCsvStream = async function(csvFilePath, model, delim, cols, storage "Some records could not be submitted. No database changes has been applied.\n"; message += "Please see the next list for details:\n"; - errors.forEach(function(error) { + errors.forEach(function (error) { valErrMessages = error.errors.reduce((acc, val) => { - return acc.concat(val.dataPath).concat(" ").concat(val.message) + return acc + .concat(val.dataPath) .concat(" ") - }) - message += - `record ${JSON.stringify(error.record)} ${error.message}: ${valErrMessages}; \n`; + .concat(val.message) + .concat(" "); + }); + message += `record ${JSON.stringify(error.record)} ${ + error.message + }: ${valErrMessages}; \n`; }); throw new Error(message.slice(0, message.length - 1)); } - if (storageType === "sql"){ + if (storageType === "sql") { await transaction.commit(); } @@ -297,16 +309,13 @@ exports.parseCsvStream = async function(csvFilePath, model, delim, cols, storage // At this moment the parseCsvStream caller is responsible in deleting the // addedZipFilePath return addedZipFilePath; - } catch (error) { - await transaction.rollback(); exports.deleteIfExists(addedFilePath); exports.deleteIfExists(addedZipFilePath); throw error; - } finally { exports.deleteIfExists(addedFilePath); } diff --git a/utils/helper.js b/utils/helper.js index 30ed935..c5ecf60 100644 --- a/utils/helper.js +++ b/utils/helper.js @@ -21,6 +21,7 @@ const { ConnectionError, getAndConnectDataModelClass, } = require("../connection"); +const config = require("../config/data_models_storage_config.json"); /** * paginate - Creates pagination argument as needed in sequelize cotaining limit and offset accordingly to the current * page implicit in the request info. @@ -2544,6 +2545,41 @@ module.exports.buildEdgeObject = function (records) { return edges; }; +module.exports.createIndexes = async (storage, model, definition, database) => { + if ("neo4j" === storage) { + const driver = await model.storageHandler; + const session = driver.session({ + database: config[database || `default-${storage}`].database, + }); + try { + const modelName = definition.model; + const label = + modelName.length === 1 + ? modelName.toUpperCase() + : modelName.slice(0, 1).toUpperCase() + + modelName.slice(1, modelName.length); + const id = definition.internalId ?? definition.id.name; + + await session.run( + `CREATE INDEX index_${id} IF NOT EXISTS FOR (n:${label}) ON (n.${id})` + ); + } catch (error) { + throw error; + } finally { + await session.close(); + } + } else if ("mongodb" === storage) { + try { + const db = await model.storageHandler; + const collection = await db.collection(definition.model); + const id = definition.internalId ?? definition.id.name; + await collection.createIndex({ [id]: 1 }); + } catch (error) { + throw error; + } + } +}; + module.exports.initializeStorageHandlersForModels = async (models) => { console.log("initialize storage handlers for models"); const connectionInstances = await getConnectionInstances(); @@ -2552,8 +2588,9 @@ module.exports.initializeStorageHandlersForModels = async (models) => { for (let name of Object.keys(models.sql)) { const database = models.sql[name].database; - const connection = connectionInstances.get(database || "default-sql") - .connection; + const connection = connectionInstances.get( + database || "default-sql" + ).connection; if (!connection) throw new ConnectionError(models.sql[name]); // setup storageHandler @@ -2574,7 +2611,14 @@ module.exports.initializeStorageHandlersForModels = async (models) => { } }); - const storageTypes = ["mongodb", "cassandra", "amazonS3", "trino", "presto"]; + const storageTypes = [ + "mongodb", + "cassandra", + "amazonS3", + "trino", + "presto", + "neo4j", + ]; for (let storage of storageTypes) { console.log(`assign storage handler to ${storage} models`); @@ -2590,6 +2634,14 @@ module.exports.initializeStorageHandlersForModels = async (models) => { getAndConnectDataModelClass(model, connection); console.log("assign storage handler to model: " + name); + if (["neo4j", "mongodb"].includes(storage)) { + await module.exports.createIndexes( + storage, + model, + models[storage][name], + database + ); + } } } }; @@ -2601,8 +2653,9 @@ module.exports.initializeStorageHandlersForAdapters = async (adapters) => { for (let name of Object.keys(adapters.sql)) { const database = adapters.sql[name].database; - const connection = connectionInstances.get(database || "default-sql") - .connection; + const connection = connectionInstances.get( + database || "default-sql" + ).connection; if (!connection) throw new ConnectionError(adapters.sql[name]); // setup storageHandler @@ -2613,7 +2666,14 @@ module.exports.initializeStorageHandlersForAdapters = async (adapters) => { console.log("assign storage handler to adapter: " + name); } - const storageTypes = ["mongodb", "cassandra", "amazonS3", "trino", "presto"]; + const storageTypes = [ + "mongodb", + "cassandra", + "amazonS3", + "trino", + "presto", + "neo4j", + ]; for (let storage of storageTypes) { console.log(`assign storage handler to ${storage} adapters`); @@ -2629,6 +2689,14 @@ module.exports.initializeStorageHandlersForAdapters = async (adapters) => { getAndConnectDataModelClass(adapter, connection); console.log("assign storage handler to adapter: " + name); + if (["neo4j", "mongodb"].includes(storage)) { + await module.exports.createIndexes( + storage, + adapter, + adapters[storage][name], + database + ); + } } } }; diff --git a/utils/neo4j_helper.js b/utils/neo4j_helper.js new file mode 100644 index 0000000..a7f8d6e --- /dev/null +++ b/utils/neo4j_helper.js @@ -0,0 +1,238 @@ +const searchArg = require("./search-argument"); + +/** + * processDateTime - process DateTime string into ISO string + * @param {object} input input + * @param {object} attributes attributes as specified in the model definition + * @return {object} result + */ +module.exports.processDateTime = (input, attributes) => { + for (let key of Object.keys(input)) { + if (["Date", "Time", "DateTime"].includes(attributes[key])) { + input[key] = input[key].toISOString(); + } else if (["[Date]", "[Time]", "[DateTime]"].includes(attributes[key])) { + input[key] = input[key].map((str) => str.toISOString()); + } + } + return input; +}; + +/** + * searchConditionsToNeo4j - translates search conditions as given in the graphQl query to 'where' options + * @param {object} search search argument for filtering records + * @param {object} dataModelDefinition definition as specified in the model + * @return {object} 'where' options + */ +module.exports.searchConditionsToNeo4j = (search, dataModelDefinition) => { + let whereOptions = ""; + if (search !== undefined && search !== null) { + if (typeof search !== "object") + throw new Error('Illegal "search" argument type, it must be an object.'); + let arg = new searchArg(search); + whereOptions = " WHERE " + arg.toNeo4j(dataModelDefinition.attributes); + } + return whereOptions; +}; + +/** + * orderConditionsToNeo4j - build the sort object for default pagination. Default order is by idAttribute ASC + * @param {array} order order array given in the graphQl query + * @param {string} idAttribute idAttribute of the model + * @param {boolean} isForwardPagination forward pagination + * @returns {object} orderOptions + */ +module.exports.orderConditionsToNeo4j = ( + order, + idAttribute, + isForwardPagination +) => { + let orderOptions = "ORDER BY "; + if (order !== undefined) { + orderOptions += order + .map((orderItem) => "n." + orderItem.field + " " + orderItem.order) + .join(", "); + } + if ( + !order || + !order.map((orderItem) => orderItem.field).includes(idAttribute) + ) { + const idOption = isForwardPagination + ? "n." + idAttribute + " ASC" + : "n." + idAttribute + " DESC"; + orderOptions += orderOptions === "ORDER BY " ? idOption : ", " + idOption; + } + return orderOptions; +}; + +/** + * mergeFilters - merge two filters into a new filter. + * @param {object} filterA first where options + * @param {object} filterB second where options (without 'where') + * @param {object} operator operator to combine filterA and filterB. Valid operators are 'and' or 'or'. default is 'and'. + */ +module.exports.mergeFilters = function (filterA, filterB, operator) { + if (operator && (operator !== "and" || operator !== "or")) + throw new Error('Only "and" or "or" operators are valid.'); + let mergeOp = operator ? operator : "AND"; + //check: no arguments + if (!filterA && !filterB) { + return ""; + } + //check: only whereB + if (!filterA && filterB) { + return " WHERE " + filterB; + } + //check: only whereA + if (filterA && !filterB) { + return filterA; + } + //check: types + if (typeof filterA !== "string" || typeof filterB !== "string") { + throw new Error("Illegal arguments provided to mergeFilters function."); + } + return `${filterA} ${mergeOp} (${filterB})`; +}; + +/** + * parseOrderCursor - Parse the order options and return the where statement for cursor based pagination (forward) + * + * Returns a set of {AND / OR} conditions that cause a ‘WHERE’ clause to deliver only the records ‘greater than’ a given cursor. + * + * @param {string} order Order options. Must contains at least the entry for 'idAttribute'. + * @param {Object} cursor Cursor record taken as start point(exclusive) to create the filter object. + * @param {String} idAttribute idAttribute of the calling model. + * @param {[string]} orderFields Order fields. Must contains at least the entry for 'idAttribute'. + * @param {Boolean} includeCursor Boolean flag that indicates if a strict or relaxed operator must be used for produce idAttribute conditions. + * @param {object} dataModelDefinition definition as specified in the model + * @return {Object} filter object which is used for retrieving records after the given cursor holding the order conditions. + */ +module.exports.parseOrderCursor = ( + order, + cursor, + idAttribute, + orderFields, + includeCursor, + dataModelDefinition +) => { + /** + * Checks + */ + //idAttribute: + if (idAttribute === undefined || idAttribute === null || idAttribute === "") { + return ""; + } + //order: must have idAttribute + if (!order.includes(idAttribute)) { + return ""; + } + //cursor: must have idAttribute + if ( + cursor === undefined || + cursor === null || + typeof cursor !== "object" || + cursor[idAttribute] === undefined + ) { + return ""; + } + + if (!orderFields.includes(idAttribute)) { + orderFields.push(idAttribute); + } + //index of base step: default -> idAttribute + let last_index = orderFields.length - 1; + //index of the starting recursive step + let start_index = orderFields.length - 2; + const stringType = ["String", "Date", "DateTime", "Time"]; + + /* + * Base step. + */ + //set operator according to order type. + let type = dataModelDefinition[orderFields[last_index]]; + let tmp_index = + order.indexOf(orderFields[last_index] + " ") + + orderFields[last_index].length + + 1; + let operator = order[tmp_index] === "A" ? ">=" : "<="; + //set strictly '>' or '<' for idAttribute (condition (1)). + if (!includeCursor && orderFields[last_index] === idAttribute) { + operator = operator.substring(0, 1); + } + + /* + * Produce condition for base step. + */ + let filter = `n.${orderFields[last_index]} ${operator} `; + let value = cursor[orderFields[last_index]]; + + filter += stringType.includes(type) ? `'${value}'` : value; + /* + * Recursive steps. + */ + for (let i = start_index; i >= 0; i--) { + /** + * Set operators + */ + //set relaxed operator '>=' or '<=' for condition (2.a or 2.b) + type = dataModelDefinition[orderFields[i]]; + + tmp_index = order.indexOf(orderFields[i] + " ") + orderFields[i].length + 1; + operator = order[tmp_index] === "A" ? ">=" : "<="; + //set strict operator '>' or '<' for condition (2.a). + let strict_operator = order[tmp_index] === "A" ? ">" : "<"; + //set strictly '>' or '<' for idAttribute (condition (1)). + if (!includeCursor && orderFields[i] === idAttribute) { + operator = operator.substring(0, 1); + } + + /** + * Produce: AND/OR conditions + */ + value = cursor[orderFields[i]]; + value = stringType.includes(type) ? `'${value}'` : value; + filter = `(n.${orderFields[i]} ${operator} ${value}) AND ( + (n.${orderFields[i]} ${strict_operator} ${value}) OR ( + ${filter}))`; + } + return filter; +}; + +/** + * cursorPaginationArgumentsToNeo4j - translate cursor based pagination object to the filter object. + * merge the original searchArguement and those needed for cursor-based pagination + * @see parseOrderCursor + * + * @param {object} pagination cursor-based pagination object + * @param {string} sort order options + * @param {string} filter where options + * @param {[string]} orderFields order fields + * @param {string} idAttribute idAttribute of the model + * @param {object} dataModelDefinition definition as specified in the model + */ +module.exports.cursorPaginationArgumentsToNeo4j = function ( + pagination, + sort, + filter, + orderFields, + idAttribute, + dataModelDefinition +) { + if (pagination) { + if (pagination.after || pagination.before) { + let cursor = pagination.after ? pagination.after : pagination.before; + let decoded_cursor = JSON.parse( + Buffer.from(cursor, "base64").toString("utf-8") + ); + let filterB = module.exports.parseOrderCursor( + sort, + decoded_cursor, + idAttribute, + orderFields, + pagination.includeCursor, + dataModelDefinition + ); + filter = module.exports.mergeFilters(filter, filterB); + } + } + return filter; +}; diff --git a/utils/search-argument.js b/utils/search-argument.js index 7670a46..ad0e4b4 100644 --- a/utils/search-argument.js +++ b/utils/search-argument.js @@ -220,7 +220,40 @@ module.exports = class search { case "in": return ` ${operator.toUpperCase()} `; default: - throw new Error(`Operator ${operator} not supported`); + throw new Error(`Operator ${operator} is not supported`); + } + } + /** + * + * @param {*} operator + */ + transformNeo4jOperator(operator) { + if (operator === undefined) { + return; + } + switch (operator) { + case "eq": + return " = "; + case "ne": + return " <> "; + case "lt": + return " < "; + case "gt": + return " > "; + case "lte": + return " <= "; + case "gte": + return " >= "; + case "regexp": + return " =~ "; + case "contains": + case "and": + case "or": + case "not": + case "in": + return ` ${operator.toUpperCase()} `; + default: + throw new Error(`Operator ${operator} is not supported`); } } /** @@ -423,4 +456,73 @@ module.exports = class search { return searchsInAmazonS3; } + /** + * toNeo4j - Convert recursive search instance to search string for use in Cypher + * + * @param{string} idAttribute - The name of the ID attribute + * + * @returns{string} Translated search instance + */ + toNeo4j(dataModelDefinition) { + let searchsInNeo4j = ""; + let type = dataModelDefinition[this.field]; + const transformedOperator = this.transformNeo4jOperator(this.operator); + const stringType = ["String", "Date", "DateTime", "Time"]; + const logicOperaters = ["and", "or", "not"]; + if ( + this.operator === undefined || + (this.value === undefined && this.search === undefined) + ) { + return searchsInNeo4j; + } else if (this.search === undefined && this.field === undefined) { + searchsInNeo4j = transformedOperator + this.value; + } else if (this.search === undefined) { + let arrayType = type != undefined && type.replace(/\s+/g, "")[0] === "["; + let value = this.value; + if (Array.isArray(value)) { + if ( + stringType.includes(type) || + stringType.includes(type.replace(/\s+/g, "").slice(1, -1)) + ) { + value = `[${value.map((e) => `"${e}"`)}]`; + } else { + value = `[${value.map((e) => `${e}`)}]`; + } + } else { + if ( + stringType.includes(type) || + stringType.includes(type.replace(/\s+/g, "").slice(1, -1)) + ) { + value = `'${value}'`; + } + } + if (arrayType && this.operator === "in") { + searchsInNeo4j = Array.isArray(this.value) + ? "ALL(x IN " + value + " WHERE x IN n." + this.field + ")" + : value + " IN n." + this.field; + } else { + // eq: array data = array value + // in: primitive data in array value + searchsInNeo4j = "n." + this.field + transformedOperator + value; + } + } else if (logicOperaters.includes(this.operator)) { + if (this.operator === "not") { + let new_search = new search(this.search[0]); + searchsInNeo4j = + transformedOperator + "(" + new_search.toNeo4j(dataModelDefinition); + } else { + searchsInNeo4j = this.search + .map((singleSearch) => + new search(singleSearch).toNeo4j(dataModelDefinition) + ) + .join(transformedOperator); + } + } else { + throw new Error( + "Statement not supported by Neo4j:\n" + JSON.stringify(this, null, 2) + ); + } + + return searchsInNeo4j; + } };