feat: Merge pull request #87 from pelias/add-multiple-hierarchy-support

Add multiple hierarchy support
pelias · Oct 17, 2016 · d8c1a44 · d8c1a44
2 parents 4b51296 + d8a1b45
commit d8c1a44
Show file tree

Hide file tree

Showing 7 changed files with 262 additions and 203 deletions.
diff --git a/src/components/extractFields.js b/src/components/extractFields.js
@@ -62,7 +62,7 @@ function getName(properties) {
 */
 module.exports.create = function map_fields_stream() {
   return through2.obj(function(json_object, enc, callback) {
-    var base_record = {
+    var record = {
       id: json_object.id,
       name: getName(json_object.properties),
       abbreviation: json_object.properties['wof:abbreviation'],
@@ -73,29 +73,16 @@ module.exports.create = function map_fields_stream() {
       bounding_box: getBoundingBox(json_object.properties),
       iso2: json_object.properties['iso:country'],
       population: getPopulation(json_object.properties),
-      popularity: json_object.properties['misc:photo_sum']
+      popularity: json_object.properties['misc:photo_sum'],
+      hierarchies: _.get(json_object, 'properties.wof:hierarchy', [])
     };
 
     // use the QS altname if US county and available
-    if (isUsCounty(base_record, json_object.properties['qs:a2_alt'])) {
-      base_record.name = json_object.properties['qs:a2_alt'];
+    if (isUsCounty(record, json_object.properties['qs:a2_alt'])) {
+      record.name = json_object.properties['qs:a2_alt'];
     }
 
-    // if there's no hierarchy then just add the base record
-    if (_.isUndefined(json_object.properties['wof:hierarchy'])) {
-      this.push(base_record);
-
-    } else {
-      // otherwise, clone the base record for each hierarchy in the list and push
-      json_object.properties['wof:hierarchy'].forEach(function(hierarchy) {
-        var clone = _.clone(base_record, true);
-        clone.hierarchy = hierarchy;
-        this.push(clone);
-      }, this);
-
-    }
-
-    return callback();
+    return callback(null, record);
 
   });
 

diff --git a/src/hierarchyFinder.js b/src/hierarchyFinder.js
@@ -27,7 +27,7 @@ module.exports.parent_id_walker = function(wofRecords) {
       parent_id = parent.parent_id;
     }
 
-    return parents.filter(hasName);
+    return [parents.filter(hasName)];
 
   };
 
@@ -53,10 +53,22 @@ module.exports.parent_id_walker = function(wofRecords) {
   lastly, filter out any hierarchy elements that are undefined or w/o a name
 
 */
+function resolveHierarchy(wofRecords, hierarchy) {
+  return Object.keys(hierarchy).map(function(key) {
+    return wofRecords[hierarchy[key]];
+  }).filter(isDefined).filter(hasName);
+}
+
+/*
+ This function returns all the resolved hierarchies for a wofRecord.  Each
+ wofRecord can have multiple hierarchies, so resolve them by looking up the
+ referenced wofRecord in the big collection of wofRecords.
+*/
 module.exports.hierarchies_walker = function(wofRecords) {
   return function(wofRecord) {
-    return Object.keys(wofRecord.hierarchy).map(function(key) {
-      return wofRecords[wofRecord.hierarchy[key]];
-    }).filter(isDefined).filter(hasName);
+    return wofRecord.hierarchies.reduce(function(resolvedHierarchies, hierarchy) {
+      resolvedHierarchies.push(resolveHierarchy(wofRecords, hierarchy));
+      return resolvedHierarchies;
+    }, []);
   };
 };
diff --git a/src/peliasDocGenerators.js b/src/peliasDocGenerators.js
@@ -1,88 +1,116 @@
-var map_stream = require('through2-map');
+var through2 = require('through2');
 var _ = require('lodash');
 var iso3166 = require('iso3166-1');
 
 var Document = require('pelias-model').Document;
 
 module.exports = {};
 
-module.exports.create = function(hierarchy_finder) {
-  return map_stream.obj(function(record) {
-    var wofDoc = new Document( 'whosonfirst', record.place_type, record.id );
-
-    if (record.name) {
-      wofDoc.setName('default', record.name);
-    }
-    wofDoc.setCentroid({ lat: record.lat, lon: record.lon });
+function assignField(hierarchyElement, wofDoc) {
+  switch (hierarchyElement.place_type) {
+    case 'neighbourhood':
+    case 'locality':
+    case 'borough':
+    case 'localadmin':
+    case 'county':
+    case 'macrocounty':
+    case 'macroregion':
+      // the above place_types don't have abbrevations (yet)
+      wofDoc.addParent(hierarchyElement.place_type, hierarchyElement.name, hierarchyElement.id.toString());
+      break;
+    case 'region':
+    case 'dependency':
+      if (hierarchyElement.hasOwnProperty('abbreviation')) {
+        wofDoc.addParent(hierarchyElement.place_type, hierarchyElement.name, hierarchyElement.id.toString(), hierarchyElement.abbreviation);
+      } else {
+        wofDoc.addParent(hierarchyElement.place_type, hierarchyElement.name, hierarchyElement.id.toString());
+      }
+      break;
+    case 'country':
+      // this is placetype=country, so lookup and set the iso3 from iso2
+      if (iso3166.is2(hierarchyElement.iso2)) {
+        var iso3 = iso3166.to3(hierarchyElement.iso2);
 
-    // only set population if available
-    if (record.population) {
-      wofDoc.setPopulation(record.population);
-    }
+        wofDoc.setAlpha3(iso3);
+        wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString(), iso3);
 
-    // only set popularity if available
-    if (record.popularity) {
-      wofDoc.setPopularity(record.popularity);
-    }
+      } else {
+        wofDoc.addParent('country', hierarchyElement.name, hierarchyElement.id.toString());
 
-    // WOF bbox is defined as:
-    // lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat
-    // so convert to what ES understands
-    if (!_.isUndefined(record.bounding_box)) {
-      var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat);
-      var marshaledBoundingBoxBox = {
-        upperLeft: {
-          lat: parsedBoundingBox[3],
-          lon: parsedBoundingBox[0]
-        },
-        lowerRight: {
-          lat: parsedBoundingBox[1],
-          lon: parsedBoundingBox[2]
-        }
-
-      };
-      wofDoc.setBoundingBox(marshaledBoundingBoxBox);
-    }
+      }
 
-    // iterate the hierarchy, assigning fields
-    hierarchy_finder(record).forEach(function(hierarchy_element) {
-      switch (hierarchy_element.place_type) {
-        case 'neighbourhood':
-        case 'locality':
-        case 'borough':
-        case 'localadmin':
-        case 'county':
-        case 'macrocounty':
-        case 'macroregion':
-          // the above place_types don't have abbrevations (yet)
-          wofDoc.addParent(hierarchy_element.place_type, hierarchy_element.name, hierarchy_element.id.toString());
-          break;
-        case 'region':
-        case 'dependency':
-          if (hierarchy_element.hasOwnProperty('abbreviation')) {
-            wofDoc.addParent(hierarchy_element.place_type, hierarchy_element.name, hierarchy_element.id.toString(), hierarchy_element.abbreviation);
-          } else {
-            wofDoc.addParent(hierarchy_element.place_type, hierarchy_element.name, hierarchy_element.id.toString());
-          }
-          break;
-        case 'country':
-          // this is placetype=country, so lookup and set the iso3 from iso2
-          if (iso3166.is2(hierarchy_element.iso2)) {
-            var iso3 = iso3166.to3(hierarchy_element.iso2);
-
-            wofDoc.setAlpha3(iso3);
-            wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString(), iso3);
-
-          } else {
-            wofDoc.addParent('country', hierarchy_element.name, hierarchy_element.id.toString());
-
-          }
-
-          break;
+      break;
+  }
+
+}
+
+// method that extracts the logic for Document creation.  `hierarchy` is optional
+function setupDocument(record, hierarchy) {
+  var wofDoc = new Document( 'whosonfirst', record.place_type, record.id );
+
+  if (record.name) {
+    wofDoc.setName('default', record.name);
+  }
+  wofDoc.setCentroid({ lat: record.lat, lon: record.lon });
+
+  // only set population if available
+  if (record.population) {
+    wofDoc.setPopulation(record.population);
+  }
+
+  // only set popularity if available
+  if (record.popularity) {
+    wofDoc.setPopularity(record.popularity);
+  }
+
+  // WOF bbox is defined as:
+  // lowerLeft.lon, lowerLeft.lat, upperRight.lon, upperRight.lat
+  // so convert to what ES understands
+  if (!_.isUndefined(record.bounding_box)) {
+    var parsedBoundingBox = record.bounding_box.split(',').map(parseFloat);
+    var marshaledBoundingBoxBox = {
+      upperLeft: {
+        lat: parsedBoundingBox[3],
+        lon: parsedBoundingBox[0]
+      },
+      lowerRight: {
+        lat: parsedBoundingBox[1],
+        lon: parsedBoundingBox[2]
       }
+
+    };
+    wofDoc.setBoundingBox(marshaledBoundingBoxBox);
+  }
+
+  // a `hierarchy` is composed of potentially multiple WOF records, so iterate
+  // and assign fields
+  if (!_.isUndefined(hierarchy)) {
+    hierarchy.forEach(function(hierarchyElement) {
+      assignField(hierarchyElement, wofDoc);
     });
 
-    return wofDoc;
+  }
+
+  return wofDoc;
+
+}
+
+module.exports.create = function(hierarchy_finder) {
+  return through2.obj(function(record, enc, next) {
+    // if there are no hierarchies, then just return the doc as-is
+    var hierarchies = hierarchy_finder(record);
+
+    if (hierarchies && hierarchies.length > 0) {
+      hierarchies.forEach(function(hierarchy) {
+        this.push(setupDocument(record, hierarchy));
+      }, this);
+
+    } else {
+      this.push(setupDocument(record));
+
+    }
+
+    next();
 
   });