Merge pull request #289 from pelias/joxit/multi-sqlite

Supports for multiple databases
pelias · Mar 10, 2020 · e07f770 · e07f770
2 parents e158106 + 36337fd
commit e07f770
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 9 deletions.
diff --git a/src/pip/index.js b/src/pip/index.js
@@ -46,9 +46,9 @@ module.exports.create = function createPIPService(datapath, layers, localizedAdm
   layers = _.intersection(defaultLayers, _.isEmpty(layers) ? defaultLayers : layers);
 
   if (isSqlite === true) {
-    const filename = path.join(datapath, 'sqlite', 'whosonfirst-data-latest.db');
-    if (!fs.existsSync(filename)) {
-      return callback(`unable to locate sqlite file ${filename}`);
+    const folder = path.join(datapath, 'sqlite');
+    if (!fs.existsSync(folder)) {
+      return callback(`unable to locate sqlite folder`);
     }
   } else {
     // keep track of any missing metafiles for later reporting and error conditions

diff --git a/src/pip/readStream.js b/src/pip/readStream.js
@@ -7,8 +7,12 @@ const simplifyGeometry = require('./components/simplifyGeometry');
 const filterOutCitylessNeighbourhoods = require('./components/filterOutCitylessNeighbourhoods');
 const filterOutHierarchylessNeighbourhoods = require('./components/filterOutHierarchylessNeighbourhoods');
 const filterOutPointRecords = require('./components/filterOutPointRecords');
+const combinedStream = require('combined-stream');
+const fs = require('fs');
 const SQLiteStream = whosonfirst.SQLiteStream;
 
+const SQLITE_REGEX = /whosonfirst-data-[a-z0-9-]+\.db$/;
+
 function readBundleRecords(datapath, layer) {
   return whosonfirst.metadataStream(datapath).create(layer)
     .pipe(whosonfirst.parseMetaFiles())
@@ -17,13 +21,25 @@ function readBundleRecords(datapath, layer) {
     .pipe(whosonfirst.loadJSON(datapath, false));
 }
 
+function getSqliteFilePaths(root) {
+  return fs.readdirSync(root)
+    .filter(d => SQLITE_REGEX.test(d))
+    .map(db => path.join(root, db));
+}
+
 function readSqliteRecords(datapath, layer) {
-  return new SQLiteStream(
-    path.join(datapath, 'sqlite', 'whosonfirst-data-latest.db'),
-    config.importPlace ?
-    SQLiteStream.findGeoJSONByPlacetypeAndWOFId(layer, config.importPlace) :
-    SQLiteStream.findGeoJSONByPlacetype(layer)
-  ).pipe(whosonfirst.toJSONStream());
+  const sqliteStream = combinedStream.create();
+  getSqliteFilePaths(path.join(datapath, 'sqlite')).forEach(dbPath => {
+    sqliteStream.append(next => {
+      next(new SQLiteStream(
+        dbPath,
+        config.importPlace ?
+        SQLiteStream.findGeoJSONByPlacetypeAndWOFId(layer, config.importPlace) :
+        SQLiteStream.findGeoJSONByPlacetype(layer)
+      ));
+    });
+  });
+  return sqliteStream.pipe(whosonfirst.toJSONStream());
 }
 
 /**