Neo4j Integration (#69)

* feat: read access from presto/trino

* fix: hasNextPage issue in cursor based pagination

* fix: add username to presto client

* fix: fix parameter order in queryData for presto helper

* fix: fix patterns for [int] type in presto/trino

* fix: add WHERE to filterB in mergeFilter methods for presto helper

* update: update query method for presto/trino

* feat: neo4j integration

* feat: create indexes for neo4j/mongodb data models

* fix: remove double quotes for neo4j connection

* fix: remove conversion for MongoDB array

README.md

@@ -119,7 +119,7 @@ The result will be:
 Zendro is the product of a joint effort between the Forschungszentrum Jülich, Germany and the Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México, to generate a tool that allows efficiently building data warehouses capable of dealing with diverse data generated by different research groups in the context of the FAIR principles and multidisciplinary projects. The name Zendro comes from the words Zenzontle and Drossel, which are Mexican and German words denoting a mockingbird, a bird capable of “talking” different languages, similar to how Zendro can connect your data warehouse from any programming language or data analysis pipeline.
 
 ### Zendro contributors in alphabetical order
-Francisca Acevedo<sup>1</sup>, Vicente Arriaga<sup>1</sup>, Katja Dohm<sup>3</sup>, Constantin Eiteneuer<sup>2</sup>, Sven Fahrner<sup>2</sup>, Frank Fischer<sup>4</sup>, Asis Hallab<sup>2</sup>, Alicia Mastretta-Yanes<sup>1</sup>, Roland Pieruschka<sup>2</sup>, Alejandro Ponce<sup>1</sup>, Yaxal Ponce<sup>2</sup>, Francisco Ramírez<sup>1</sup>, Irene Ramos<sup>1</sup>, Bernardo Terroba<sup>1</sup>, Tim Rehberg<sup>3</sup>, Verónica Suaste<sup>1</sup>, Björn Usadel<sup>2</sup>, David Velasco<sup>2</sup>, Thomas Voecking<sup>3</sup>
+Francisca Acevedo<sup>1</sup>, Vicente Arriaga<sup>1</sup>, Katja Dohm<sup>3</sup>, Constantin Eiteneuer<sup>2</sup>, Sven Fahrner<sup>2</sup>, Frank Fischer<sup>4</sup>, Asis Hallab<sup>2</sup>, Alicia Mastretta-Yanes<sup>1</sup>, Roland Pieruschka<sup>2</sup>, Alejandro Ponce<sup>1</sup>, Yaxal Ponce<sup>2</sup>, Francisco Ramírez<sup>1</sup>, Irene Ramos<sup>1</sup>, Bernardo Terroba<sup>1</sup>, Tim Rehberg<sup>3</sup>, Verónica Suaste<sup>1</sup>, Björn Usadel<sup>2</sup>, David Velasco<sup>2</sup>, Thomas Voecking<sup>3</sup>, Dan Wang<sup>2</sup>
 
 #### Author affiliations
 1. CONABIO - Comisión Nacional para el Conocimiento y Uso de la Biodiversidad, México

config/data_models_storage_config.json

@@ -49,5 +49,13 @@
     "schema":"public",
     "presto_host": "127.0.0.1",
     "presto_port": "8081"
+  },
+  "default-neo4j": {
+    "storageType": "neo4j",
+    "username": "neo4j",
+    "password": "sciencedb",
+    "database": "neo4j",
+    "host": "127.0.0.1",
+    "port": "7687"
   }
 }

connection.js

@@ -4,6 +4,7 @@ const { MongoClient } = require("mongodb");
 const AWS = require("aws-sdk");
 const presto = require("presto-client");
 const cassandraDriver = require("cassandra-driver");
+const neo4j = require("neo4j-driver");
 const { queryData } = require("./utils/presto_helper");
 const os = require("os");
 const Op = Sequelize.Op;
@@ -180,6 +181,23 @@ const addConnectionInstances = async () => {
           engine: "presto",
         }),
       });
+    } else if (
+      storageConfig.hasOwnProperty(key) &&
+      key !== "operatorsAliases" &&
+      storageType === "neo4j"
+    ) {
+      // note: https://stackoverflow.com/a/62816512/8132085
+      connectionInstances.set(key, {
+        storageType,
+        connection: neo4j.driver(
+          `bolt://${storageConfig[key].host}:${storageConfig[key].port}`,
+          neo4j.auth.basic(
+            storageConfig[key].username,
+            storageConfig[key].password
+          ),
+          { disableLosslessIntegers: true }
+        ),
+      });
     }
   }
   return connectionInstances;
@@ -228,6 +246,10 @@ exports.checkConnections = async () => {
         } catch (error) {
           throw error;
         }
+      } else if (instance.storageType === "neo4j") {
+        await instance.connection.verifyConnectivity({
+          database: storageConfig["default-neo4j"].database,
+        });
       }
       checks.push({ key, valid: true });
     } catch (exception) {

models/adapters/index.js

@@ -9,6 +9,7 @@ let adapters = {
   amazonS3: {},
   trino: {},
   presto: {},
+  neo4j: {},
 };
 module.exports = adapters;
 getModulesSync(__dirname).forEach((file) => {
@@ -30,6 +31,7 @@ getModulesSync(__dirname).forEach((file) => {
     case "cassandra-adapter":
     case "trino-adapter":
     case "presto-adapter":
+    case "neo4j-adapter":
       adapters[adapter.adapterType.split("-")[0]][adapter.adapterName] =
         adapter.definition;
       adapters[adapter.adapterName] = adapter;

models/index.js

@@ -9,6 +9,7 @@ let models = {
   amazonS3: {},
   trino: {},
   presto: {},
+  neo4j: {},
 };
 const storageTypes = Object.keys(models);
 module.exports = models;

package.json

@@ -40,6 +40,7 @@
     "moment": "^2.25.3",
     "mongodb": "^3.6.3",
     "mysql2": "^2.1.0",
+    "neo4j-driver": "^4.2.3",
     "node-jq": "^1.11.1",
     "nodemailer": "^6.4.6",
     "nodemailer-smtp-transport": "^2.7.4",