Merge pull request #471 from pelias/joxit/fix/sqlite-hierarchy

Ensure the placetype order when an import is done

src/readStream.js

@@ -72,14 +72,16 @@ function createMetaRecordStream(metaFilePaths, types) {
  */
 function createSQLiteRecordStream(dbPaths, importPlace) {
   const sqliteStream = combinedStream.create();
-  const sqliteStatement = importPlace ?
-    SQLiteStream.findGeoJSONByPlacetypeAndWOFId(getPlacetypes(), importPlace) :
-    SQLiteStream.findGeoJSONByPlacetype(getPlacetypes());
 
   dbPaths.forEach((dbPath) => {
-    sqliteStream.append( (next) => {
-      logger.info( `Loading ${path.basename(dbPath)} database from ${path.dirname(dbPath)}` );
-      next(new SQLiteStream(dbPath, sqliteStatement));
+    getPlacetypes().forEach(placetype => {
+      sqliteStream.append( (next) => {
+        logger.debug( `Loading '${placetype}' of ${path.basename(dbPath)} database from ${path.dirname(dbPath)}` );
+        const sqliteStatement = importPlace ?
+          SQLiteStream.findGeoJSONByPlacetypeAndWOFId(placetype, importPlace) :
+          SQLiteStream.findGeoJSONByPlacetype(placetype);
+        next(new SQLiteStream(dbPath, sqliteStatement));
+      });
     });
   });
 

test/functional.js

@@ -0,0 +1,60 @@
+const tape = require('tape');
+const importStream = require('../src/importStream');
+const readStreamModule = require('../src/readStream');
+const hierarchyFinder = require('../src/hierarchyFinder');
+const peliasDocGenerators = require('../src/peliasDocGenerators');
+const sink = require('through2-sink');
+const map_stream = require('through2-map');
+const event_stream = require('event-stream');
+const temp = require('temp').track();
+const generateWOFDB = require('./generateWOFDB');
+const path = require('path');
+
+tape('functional', function(test) {
+  test.test('SQLite database read with correct hierarchy for Tavel', t => {
+    temp.mkdir('tmp_sqlite', (err, temp_dir) => {
+      generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), require('./resources/france.json').tavel);
+      const wofAdminRecords = {};
+      const recordOrder = [];
+      const dbRecords = {};
+      const readStream = readStreamModule.create({ datapath: temp_dir, sqlite: true }, ['whosonfirst-data-latest.db'], wofAdminRecords);
+      const documentGenerator = peliasDocGenerators.create(hierarchyFinder(wofAdminRecords));
+      const dbClientStream = sink.obj(record => {
+        recordOrder.push(record.data.source_id);
+        dbRecords[record.data.source_id] = record.data;
+      });
+
+      importStream(readStream, documentGenerator, dbClientStream, () => {
+        t.ok(dbRecords[101772677]);
+        t.deepEqual(recordOrder, ['85633147', '1108826387', '85683431', '404227861', '102068587', '404414453', '101772677']);
+        t.deepEqual(dbRecords[101772677].parent, {
+          country: ['France'],
+          country_id: ['85633147'],
+          country_a: ['FRA'],
+          county: ['Roquemaure'],
+          county_id: ['102068587'],
+          county_a: [null],
+          localadmin: ['Tavel'],
+          localadmin_id: ['404414453'],
+          localadmin_a: [null],
+          locality: ['Tavel'],
+          locality_id: ['101772677'],
+          locality_a: [null],
+          macrocounty: ['Arrondissement of Nimes'],
+          macrocounty_id: ['404227861'],
+          macrocounty_a: [null],
+          macroregion: ['Occitanie'],
+          macroregion_id: ['1108826387'],
+          macroregion_a: [null],
+          region: ['département des Gard'],
+          region_id: ['85683431'],
+          region_a: ['GA']
+        }, 'correct parent hierarchy for Tavel');
+        temp.cleanupSync();
+        t.end();
+      });
+    });
+  });
+  test.end();
+
+});

test/resources/france.json

@@ -0,0 +1,10 @@
+{ "tavel": [
+  {"id":85633147,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":71.727401,"geom:area_square_m":637043109269.3989,"geom:bbox":"-61.7978409495,-21.3707821584,55.8545028003,51.0893825","geom:latitude":42.191695,"geom:longitude":-2.736467,"gn:population":64768389,"iso:country":"FR","itu:country_code":["33"],"label:eng_x_preferred_shortcode":["FR"],"lbl:bbox":"-5.132188,41.365694,9.560119,51.087167","lbl:latitude":46.698481,"lbl:longitude":2.549047,"mps:latitude":46.698481,"mps:longitude":2.549047,"name:deu_x_colloquial":["Franzoesische Republik","Franzosische Republik"],"name:deu_x_preferred":["Frankreich"],"name:deu_x_variant":["Französische Republik"],"name:eng_x_preferred":["France"],"name:eng_x_unknown":["FR","FRA"],"name:eng_x_variant":["French","French Republic","Furansu"],"name:fra_x_colloquial":["Gallia"],"name:fra_x_preferred":["France"],"name:fra_x_variant":["Gallía","République Française"],"name:ita_x_preferred":["Francia"],"name:nld_x_preferred":["Frankrijk"],"name:spa_x_preferred":["Francia"],"wd:area":675000,"wd:latitude":47,"wd:longitude":2,"wd:population":65939866,"wd:wordcount":26790,"wof:belongsto":[102191581,136253037],"wof:breaches":[],"wof:capital":[101751119],"wof:concordances":{"dbp:id":"France","fb:id":"en.france","fifa:id":"FRA","fips:code":"FR","gaul:id":"85","gn:id":3017382,"gp:id":23424819,"hasc:id":"FR","ioc:id":"FRA","itu:id":"F","loc:id":"n79006404","m49:code":"250","marc:id":"fr","mzb:id":"france","ne:adm0_a3":"FRA","nyt:id":"67127587134490296321","qs:id":1030839,"uncrt:id":"F","wd:id":"Q142","wmo:id":"FR"},"wof:country":"FR","wof:country_alpha3":"FRA","wof:geomhash":"d53b6b7694e5edf01c8fc72be547eebc","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"empire_id":136253037}],"wof:id":85633147,"wof:lang_x_official":["fra"],"wof:lang_x_spoken":["fra"],"wof:lastmodified":1565710065,"wof:name":"France","wof:parent_id":102191581,"wof:placetype":"country","wof:population":65939866,"wof:population_rank":16,"wof:repo":"whosonfirst-data-admin-fr","wof:shortcode":"FR","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} ,
+  {"id":1108826387,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":8.192862,"geom:area_square_m":73233965966.66705,"geom:bbox":"-0.3269715,42.332741,4.8453465,45.0465165","geom:latitude":43.702418,"geom:longitude":2.144119,"iso:country":"FR","label:eng_x_preferred_longname":["Occitania Region"],"label:eng_x_preferred_placetype":["region"],"label:eng_x_preferred_shortcode":["LP"],"label:fra_x_preferred_longname":["Occitanie"],"label:fra_x_preferred_placetype":["région"],"lbl:latitude":43.620178,"lbl:longitude":2.144662,"name:deu_x_preferred":["Okzitanien"],"name:eng_x_preferred":["Occitania"],"name:fra_x_preferred":["Occitanie"],"name:ita_x_preferred":["Occitania"],"name:nld_x_preferred":["Occitanie"],"name:spa_x_preferred":["Occitania"],"wof:belongsto":[102191581,85633147,136253037],"wof:breaches":[],"wof:concordances":{"hasc:id":"FR.LP","wd:id":"Q18678265"},"wof:country":"FR","wof:created":1487273090,"wof:geomhash":"9b10e097612280b5b8f0f225d5d4bfe6","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"empire_id":136253037,"macroregion_id":1108826387}],"wof:id":1108826387,"wof:lang_x_official":["fra"],"wof:lang_x_spoken":["fra"],"wof:lastmodified":1565710115,"wof:name":"Occitania","wof:parent_id":85633147,"wof:placetype":"macroregion","wof:placetype_alt":[],"wof:placetype_local":"region","wof:population":5667583,"wof:population_rank":13,"wof:repo":"whosonfirst-data-admin-fr","wof:shortcode":"LP","wof:superseded_by":[],"wof:supersedes":[404227443,404227437],"wof:tags":[]}} ,
+  {"id":85683431,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.659191,"geom:area_square_m":5863950397.840382,"geom:bbox":"3.262074,43.460186,4.8453465,44.459797","geom:latitude":43.993645,"geom:longitude":4.179915,"iso:country":"FR","label:eng_x_preferred_longname":["Gard Department"],"label:eng_x_preferred_placetype":["department"],"label:eng_x_preferred_shortcode":["GA"],"label:fra_x_preferred_longname":["département des Gard"],"label:fra_x_preferred_placetype":["département"],"lbl:latitude":44.013111,"lbl:longitude":4.191147,"mps:latitude":44.013111,"mps:longitude":4.191147,"name:deu_x_preferred":["Département Gard"],"name:deu_x_variant":["Gard"],"name:eng_x_preferred":["Gard"],"name:fra_x_preferred":["Gard"],"name:fra_x_variant":["Département du Gard"],"name:ita_x_preferred":["Gard"],"name:nld_x_preferred":["Gard"],"name:spa_x_preferred":["Gard"],"unlc:subdivision":"FR-30","wd:wordcount":530,"wof:belongsto":[102191581,85633147,136253037,1108826387],"wof:breaches":[],"wof:concordances":{"fips:code":"FRA9","gn:id":3016670,"gp:id":12597193,"hasc:id":"FR.LP.GA","unlc:id":"FR-30","wd:id":"Q12515"},"wof:country":"FR","wof:geomhash":"2ce0dcba603b77b35f4f7cee617a9157","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"empire_id":136253037,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":85683431,"wof:lang_x_official":["fra"],"wof:lang_x_spoken":["fra"],"wof:lastmodified":1565710074,"wof:name":"Gard","wof:parent_id":1108826387,"wof:placetype":"region","wof:placetype_local":"department","wof:population":726285,"wof:population_rank":11,"wof:repo":"whosonfirst-data-admin-fr","wof:shortcode":"GA","wof:subdivision":"FR-30","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} ,
+  {"id":404227861,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.35206,"geom:area_square_m":3136211027.53961,"geom:bbox":"4.00134078086,43.4601854327,4.84534415757,44.341674883","geom:latitude":43.910218,"geom:longitude":4.423829,"iso:country":"FR","label:eng_x_preferred_longname":["Arrondissement of Nimes"],"label:eng_x_preferred_placetype":["arrondissement"],"label:fra_x_preferred_longname":["arrondissement de Nîmes"],"label:fra_x_preferred_placetype":["arrondissement"],"name:deu_x_preferred":["Arrondissement Nîmes"],"name:eng_x_preferred":["Nimes"],"name:eng_x_variant":["arrondissement of Nîmes"],"name:fra_x_preferred":["arrondissement de Nîmes"],"name:ita_x_preferred":["arrondissement di Nîmes"],"name:nld_x_preferred":["Arrondissement Nîmes"],"name:spa_x_preferred":["Distrito de Nîmes"],"wd:wordcount":2377,"wof:belongsto":[85683431,102191581,85633147,1108826387,136253037],"wof:breaches":[],"wof:concordances":{"wd:id":"Q702009"},"wof:country":"FR","wof:geomhash":"22cb8cfcff37fcc059691d958c618668","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"empire_id":136253037,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":404227861,"wof:lastmodified":1563296151,"wof:name":"Nimes","wof:parent_id":85683431,"wof:placetype":"macrocounty","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} ,
+  {"id":102068587,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.017864,"geom:area_square_m":158756574.026902,"geom:bbox":"4.60483917319,43.9916304709,4.84311842532,44.1321854902","geom:latitude":44.053675,"geom:longitude":4.700463,"iso:country":"FR","lbl:latitude":44.050429,"lbl:longitude":4.680607,"mps:latitude":44.050429,"mps:longitude":4.680607,"name:deu_x_preferred":["Roquemaure"],"name:eng_x_preferred":["Roquemaure"],"name:fra_x_preferred":["Roquemaure"],"name:ita_x_preferred":["Roquemaure"],"name:nld_x_preferred":["Roquemaure"],"name:spa_x_preferred":["Roquemaure"],"wd:wordcount":24,"wof:belongsto":[85683431,102191581,85633147,404227861,1108826387,136253037],"wof:breaches":[],"wof:concordances":{"gp:id":12597193,"wd:id":"Q293958"},"wof:country":"FR","wof:geomhash":"218f43551cd3ca214069fe9cda361ea2","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":102068587,"wof:lastmodified":1563275379,"wof:name":"Roquemaure","wof:parent_id":404227861,"wof:placetype":"county","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} ,
+  {"id":404414453,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.002249,"geom:area_square_m":20005440.423096,"geom:bbox":"4.63273757889,43.9905659369,4.74172258101,44.0263921976","geom:latitude":44.009261,"geom:longitude":4.692728,"iso:country":"FR","lbl:latitude":44.00898,"lbl:longitude":4.707933,"name:deu_x_preferred":["Tavel"],"name:fra_x_preferred":["Tavel"],"name:nld_x_preferred":["Tavel"],"wof:belongsto":[85683431,102191581,85633147,404227861,1108826387,136253037,102068587],"wof:breaches":[],"wof:concordances":{"gn:id":6431462,"gp:id":12671550,"qs_pg:id":945760},"wof:coterminous":[101772677],"wof:country":"FR","wof:geomhash":"533be5788201eceb1e49923f366f0421","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"localadmin_id":404414453,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":404414453,"wof:lastmodified":1564130533,"wof:name":"Tavel","wof:parent_id":102068587,"wof:placetype":"localadmin","wof:placetype_local":"Commune simple","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} ,
+  {"id":101772677,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.002249,"geom:area_square_m":20005440.423096,"geom:bbox":"4.63273757889,43.9905659369,4.74172258101,44.0263921976","geom:latitude":44.009261,"geom:longitude":4.692728,"gn:population":1668,"iso:country":"FR","lbl:latitude":44.01144,"lbl:longitude":4.698562,"mps:latitude":44.01144,"mps:longitude":4.698562,"name:deu_x_preferred":["Tavel"],"name:eng_x_preferred":["Tavel"],"name:fra_x_preferred":["Tavel"],"name:ita_x_preferred":["Tavel"],"name:nld_x_preferred":["Tavel"],"name:spa_x_preferred":["Tavel"],"wd:wordcount":48,"wof:belongsto":[85683431,102191581,404227861,404414453,85633147,1108826387,136253037,102068587],"wof:breaches":[],"wof:concordances":{"fct:id":"0a5b4d46-8f76-11e1-848f-cfd5bf3ef515","gn:id":2973264,"gp:id":628145,"qs:id":1040667,"qs_pg:id":1040667,"wd:id":"Q334324","wk:page":"Priego"},"wof:coterminous":[404414453],"wof:country":"FR","wof:geomhash":"533be5788201eceb1e49923f366f0421","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"localadmin_id":404414453,"locality_id":101772677,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":101772677,"wof:lang":["fre"],"wof:lastmodified":1563293671,"wof:name":"Tavel","wof:parent_id":404414453,"wof:placetype":"locality","wof:population":1668,"wof:population_rank":3,"wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}}
+]
+}

test/test.js

@@ -15,3 +15,4 @@ require ('./peliasDocGeneratorsTest.js');
 require ('./readStreamTest.js');
 require ('./schema.js');
 require ('./bundleList.js');
+require('./functional.js');