Merge pull request #296 from pelias/postal_cities_custom_data_dir

allow customization of the search paths for postal cities TSV data

README.md

@@ -80,3 +80,47 @@ There are two: admin lookup slows down the process of loading data into Pelias,
 and it takes quite a bit of memory. Based on the current amount of data in Who's
 on First, count on using at least 4 or 5 GB of memory _just_ for admin lookup
 while importing.
+
+### Postal Cities
+
+This module comes bundled with data files which define a mapping between postal codes and their corresponding city name(s).
+
+This is particularly helpful in places where the locality name returned by the point-in-polygon system differs from the locality name commonly used by residents of that postcode.
+
+#### Contributing
+
+The mapping files are open-data, you can find more infomation about [how the data files are generated here](https://github.com/pelias/lastline).
+
+In the `src/data` directory of this repository you'll find the TSV (tab separated) files named after the corresponding 3-character country code (eg. `AUS.tsv`).
+
+Instead of editing these files directly (and risking the work being lost on the next regeneration of the files), you should add your changes to an 'override file', for example `USA.override.tsv`.
+
+These override files are intended for contribution from humans, so your data is safe!
+
+The TSV columns are (in order left-to-right):
+|name|type|comment|
+|:-:|:-:|:--|
+|postalcode|string|postal code eg. `90210`|
+|wofid|number|corresponding [WhosOnFirst](https://whosonfirst.org) ID|
+|name|string|name of the city/town/burough/hamlet etc.|
+|abbr|string|an abbreviation of the name (where available)|
+|placetype|string|the [WhosOnFirst](https://github.com/whosonfirst/whosonfirst-placetypes) placetype|
+|weight|number|an integer representing how many occurrences of this postalcode+wofid we found|
+
+Note that many editors will try to convert tabs to spaces, please ensure that this is not the case before saving your work!
+
+The `abbr` column is optional, if you don't specify an abbreviation please be sure that your line always contains 5 tabs (ie. 6 columns).
+
+The default value for `weight` is `Number.MAX_SAFE_INTEGER` (a very large number), you may wish to specify it lower if you're unsure how correct the entry is.
+
+In the case where there are multiple entries for the same postcode, all of the names are included in the Pelias index and can be used interchangeably to retrieve the document.
+
+The `weight` field is used to determine which entry is the most important, this entry is used to generate the label for display.
+
+#### Configuration
+
+To enable the postal cities functionality, set `imports.adminLookup.usePostalCities` to `true` in your `pelias.json` file.
+
+#### Advanced Configuration
+
+It's possible to use your own mapping files by setting `imports.adminLookup.postalCitiesDataPath` to point to a directory of your choice, if the corresponding TSV file is found in your path it will be used in place of the bundled data files. [more information](https://github.com/pelias/wof-admin-lookup/pull/296).

schema.js

@@ -8,7 +8,8 @@ module.exports = Joi.object().keys({
       maxConcurrentReqs: Joi.number().integer().default(cpus().length*10),
       enabled: Joi.boolean().default(true),
       missingMetafilesAreFatal: Joi.boolean().default(false),
-      usePostalCities: Joi.boolean().default(false)
+      usePostalCities: Joi.boolean().default(false),
+      postalCitiesDataPath: Joi.string()
     }).unknown(true),
     whosonfirst: Joi.object().keys({
       datapath: Joi.string().required(),

src/postalCityMap.js

@@ -4,6 +4,8 @@ const fs = require('fs');
 const path = require('path');
 const stable = require('stable');
 const csv = require('csv-parse/lib/sync');
+const config = require('pelias-config').generate();
+const logger = require('pelias-logger').get('wof-admin-lookup');
 const tsvOptions = {
   trim: true,
   skip_empty_lines: true,
@@ -13,6 +15,20 @@ const tsvOptions = {
   delimiter: '\t'
 };
 
+// $searchPaths behaves like the unix $PATH and $LD_LIBRARY_PATH variables,
+// you may specify multiple search paths to check for data files and the first matching
+// path will be used, subsequent paths matching the same file name will be ignored.
+// note: by default we use the data bundled within this module.
+const searchPaths = [path.resolve(__dirname, 'data')];
+
+// you may optionally change the location of where the TSV files are loaded
+// from by specifying an absolute path in your pelias.json file.
+// eg: imports.adminLookup.postalCitiesDataPath = "/data/postal_cities"
+const configDataPath = _.get(config, 'imports.adminLookup.postalCitiesDataPath');
+if (_.isString(configDataPath) && !_.isEmpty(configDataPath) && fs.existsSync(configDataPath)) {
+  searchPaths.unshift(configDataPath);
+}
+
 // ISO country codes using the USA 'zip' system
 const USA_ISO_CODES = ['USA','ASM','GUM','MNP','PRI','VIR'];
 
@@ -48,26 +64,28 @@ const tables = {
 function loadTable(cc){
   var m = {};
 
-  // each table can have multiple files, the base file is generated by
-  // scripts and should not be manually edited, the '*.override.tsv'
-  // files are human edited files which take priority.
-  var paths = {
-    base:       path.resolve(__dirname, `data/${cc}.tsv`),
-    override:   path.resolve(__dirname, `data/${cc}.override.tsv`)
-  };
-
   // load files from disk and parse contents
   var rows = [];
 
-  // load base file
-  if( fs.existsSync(paths.base) ){
-    rows = rows.concat( parse( paths.base ) );
-  }
+  // load base TSV file
+  searchPaths.some(datapath => {
+    const baseTSVFilePath = path.resolve(datapath, `${cc}.tsv`);
+    if (fs.existsSync(baseTSVFilePath)) {
+      logger.debug(`[postalCityMap] loaded ${baseTSVFilePath}`);
+      rows = rows.concat(parse(baseTSVFilePath));
+      return true;
+    }
+  });
 
   // load override file
-  if( fs.existsSync(paths.override) ){
-    rows = rows.concat( parse( paths.override, Number.MAX_SAFE_INTEGER ) );
-  }
+  searchPaths.some(datapath => {
+    const overrideTSVFilePath = path.resolve(datapath, `${cc}.override.tsv`);
+    if (fs.existsSync(overrideTSVFilePath)) {
+      logger.debug(`[postalCityMap] loaded ${overrideTSVFilePath}`);
+      rows = rows.concat(parse(overrideTSVFilePath, Number.MAX_SAFE_INTEGER));
+      return true;
+    }
+  });
 
   // generate map
   rows.forEach(row => {