update readme, some pmtile export fixes, map styling

README.md

@@ -1,10 +1,19 @@
 # overture-maps
 
-Playing around with data from Overture Maps. A notebook to play around in and some scripts to extract and convert data to geoparquet, geojson and pmtiles.
+Playing around with data from OvertureMaps.
+
+-   Download OvertureMaps data
+-   Extract data from OvertureMaps for a country and convert to geoparquet
+-   Create PMTiles from extracted data
+-   Notebook with some queries and a map to display some data
+
+Demo viewer for generated PMTiles: [https://tebben.github.io/overture-maps/](https://tebben.github.io/overture-maps/)
+
+![PMTiles](./data/images/pmtiles.jpg)
 
 ## Download overture-maps data
 
-Run `scripts/download_overture_data.sh` to download all overture parquet files from AWS without needing to login. This will take some time to run since this is around 200GB. This script will install AWS cli if not found on the system. Using duckdb we could also directly request the data we need because the data is hive partitioned but I prefer to have everything locally.
+For the notebook and scripts to create extracts, geoparquet and PMTiles we need all the data locally. Run `scripts/download_overture_data.sh` to download all overture parquet files from AWS without needing to login. This will take some time to run since this is around 200GB. This script will install AWS cli if not found on the system.
 
 ```sh
 ./scripts/download_overture_data.sh
@@ -27,7 +36,7 @@ Install miniconda if not installed yet.
 Create a conda environment and install some dependencies.
 
 ```sh
-conda create -n overture python=3.10 
+conda create -n overture python=3.10
 conda activate overture
 conda install pip
 pip install -r requirements.txt
@@ -66,21 +75,21 @@ The following script will create geoparquet files for all overture-maps themes w
 
 ### Create GeoJSON
 
-For the PMTiles creation we need GeoJSON files `parquet_to_geojson.sh` creates geojson files for each theme from our geoparquet files which can be feeded into tippecanoe. Not every field is added to the GeoJSON features, if you miss something you can update the queries.
+For the PMTiles creation we need GeoJSON files, the script `parquet_to_geojson.sh` creates geojson files for each theme from our geoparquet files which than can feeded into tippecanoe. Not every field is added to the GeoJSON feature properties at the moment.
 
 ```sh
 ./scripts/convert/parquet_to_geojson.sh
 ```
 
 ### Create PMTiles
 
-When we have the GeoJSON files we can create our PMTiles using tippecanoe.
+When we have the GeoJSON files we can create our PMTiles using tippecanoe and pmtiles.
 
 ```sh
 ./scripts/convert/geojson_to_pmtiles.sh
 ```
 
-This will create mbtiles for each theme, merge them and convert to PMTiles. Directly creating pmtiles with tippecanoe resulted in a PMTiles V2 files which could not be converted to v3, therefore mbtiles are created and later converted into PMTiles using `pmtiles convert`
+This will create mbtiles for each theme, merge them and convert to PMTiles. Directly creating pmtiles with tippecanoe resulted in a PMTiles V2 file which could not be converted to v3, therefore mbtiles are created and later converted into PMTiles using `pmtiles convert`
 
 ### Run viewer to view PMTiles
 
@@ -90,4 +99,4 @@ In the folder `viewer` a maplibre-gl viewer can be found which loads the PMTiles
 cd viewer
 npm install
 npm run dev
-```
+```

notebooks/hello_overture.ipynb

@@ -454,13 +454,6 @@
     "with open(\"./map_configs/hello_overture_map_1.json\", \"w\") as f:\n",
     "    f.write(json.dumps(map_1.config))"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

scripts/convert/geojson_to_pmtiles.sh

@@ -50,15 +50,14 @@ tile-join -pk -o $OUTPUT_PATH/overture.mbtiles $OUTPUT_PATH/admins.mbtiles $OUTP
 # tippecanoe outputs pmtiles V2, we want v3
 echo "convert mbtiles to pmtiles"
 pmtiles convert $OUTPUT_PATH/overture.mbtiles $OUTPUT_PATH/overture.pmtiles
-#pmtiles convert $OUTPUT_PATH/overture.mbtiles $OUTPUT_PATH/overture.pmtiles
 
 echo "Cleaning up intermediate files"
 # remove mbtiles file
-#rm $OUTPUT_PATH/admins.mbtiles
-#rm $OUTPUT_PATH/buildings.mbtiles
-#rm $OUTPUT_PATH/roads.mbtiles
+rm $OUTPUT_PATH/admins.mbtiles
+rm $OUTPUT_PATH/buildings.mbtiles
+rm $OUTPUT_PATH/roads.mbtiles
 rm $OUTPUT_PATH/overture.mbtiles
-#rm $OUTPUT_PATH/places.mbtiles
+rm $OUTPUT_PATH/places.mbtiles
 
 # copy pmtiles to the viewer
 cp $OUTPUT_PATH/overture.pmtiles ./viewer/dist/overture.pmtiles

scripts/convert/parquet_to_geojson.sh

@@ -63,7 +63,7 @@ duckdb -c """
             COALESCE(height, -1) as height,
             COALESCE(numFloors, -1) as numFloors,
             COALESCE(class, '') as class,
-            COALESCE(replace(json_extract(names::json,'\$.common[0].value'),'\"','')::varchar, '') as name,
+            COALESCE(replace(json_extract(names::json,'\$.key_value[0].value.list[0].key_value[0].value'),'\"','')::varchar, '') as name,
             ST_GeomFromWkb(geometry) AS geometry
         FROM read_parquet('$INPUT_PATH/buildings_geo.parquet')
     ) 
@@ -79,16 +79,12 @@ duckdb -c """
     LOAD spatial; 
     COPY (
         SELECT
-            COALESCE(replace(json_extract(addresses::json,'\$[0].locality'),'\"','')::varchar, '') as locality,
-            COALESCE(replace(json_extract(addresses::json,'\$[0].region'),'\"','')::varchar, '') as region,
-            COALESCE(replace(json_extract(addresses::json,'\$[0].postcode'),'\"','')::varchar, '') as postcode,
-            COALESCE(replace(json_extract(addresses::json,'\$[0].freeform'),'\"','')::varchar, '') as freeform,
             COALESCE(replace(json_extract(categories::json,'\$.main'),'\"','')::varchar, '') as category,
-            COALESCE(replace(json_extract(names::json,'\$.common[0].value'),'\"','')::varchar, '') as name,
-            COALESCE(confidence, 0),
+            COALESCE(replace(json_extract(names::json,'\$.key_value[0].value.list[0].key_value[0].value'), '\"', '')::varchar, '') as name,
+            COALESCE(confidence, 0) as confidence,
             ST_GeomFromWkb(geometry) AS geometry
         FROM read_parquet('$INPUT_PATH/places_geo.parquet')
     ) 
     TO '$OUTPUT_PATH/places.geojson'
     WITH (FORMAT GDAL, DRIVER 'GeoJSON');
-"""
+"""

viewer/dist/index.html

@@ -5,13 +5,17 @@
         <meta http-equiv="X-UA-Compatible" content="IE=edge" />
         <meta name="viewport" content="width=device-width, initial-scale=1.0" />
         <title>OvertureMaps PMTiles</title>
-      <script type="module" crossorigin src="./assets/index-a09a7a30.js"></script>
-      <link rel="stylesheet" href="./assets/index-3db1e16f.css">
+      <script type="module" crossorigin src="./assets/index-4314267e.js"></script>
+      <link rel="stylesheet" href="./assets/index-e2144531.css">
     </head>
     <body>
         <div id="info">
             <div class="header">OvertureMaps PMTiles</div>
-            <div class="description">Demo showing PMTiles generated from OvertureMaps</div>
+            <div class="description">
+                <a href="https://github.com/tebben/overture-maps"
+                    >Demo showing PMTiles generated from OvertureMaps</a
+                >
+            </div>
         </div>
         <div id="map"></div>
 

viewer/index.html

@@ -9,7 +9,11 @@
     <body>
         <div id="info">
             <div class="header">OvertureMaps PMTiles</div>
-            <div class="description">Demo showing PMTiles generated from OvertureMaps</div>
+            <div class="description">
+                <a href="https://github.com/tebben/overture-maps"
+                    >Demo showing PMTiles generated from OvertureMaps</a
+                >
+            </div>
         </div>
         <div id="map"></div>
         <script type="module" src="/src/main.ts"></script>

viewer/src/main.ts

@@ -7,6 +7,9 @@ import * as pmtiles from 'pmtiles'
 let protocol = new pmtiles.Protocol();
 maplibregl.addProtocol("pmtiles", protocol.tile);
 
+let hostname = location.hostname;
+console.log(hostname);
+
 const map = new maplibregl.Map({
     container: 'map',
     hash: true,
@@ -16,7 +19,8 @@ const map = new maplibregl.Map({
         sources: {
             "overture": {
                 "type": "vector",
-                "url": "pmtiles://dist/overture.pmtiles",
+                "url": location.hostname.includes("github.io") ? "pmtiles://https://storage.googleapis.com/ahp-research/overture/pmtiles/overture.pmtiles" : "pmtiles://dist/overture.pmtiles",
+                "attribution": "© OpenStreetMap, Overture Maps Foundation, Barry"
             }
         },
         layers: [
@@ -39,59 +43,55 @@ const map = new maplibregl.Map({
                     'line-width': 1
                 }
             },
-            {
+            /* {
                 "id": 'buldings',
                 "type": 'fill',
                 "source": 'overture',
                 "source-layer": 'buildings',
                 "paint": {
-                    'fill-color': '#353535'
+                    'fill-color': '#1C1C1C'
+                }
+            }, */
+            {
+                "id": 'buldings-extruded',
+                "type": 'fill-extrusion',
+                "source": 'overture',
+                "source-layer": 'buildings',
+                "paint": {
+                    'fill-extrusion-color': '#1C1C1C',
+                    "fill-extrusion-height": [
+                        "case",
+                        ["==", ["get", "height"], -1],
+                        0,
+                        ["get", "height"]  // Otherwise, use the actual height value
+                    ],
+                    'fill-extrusion-opacity': 0.9
                 }
             },
-            /*   {
-                  "id": 'buldings-extruded',
-                  "type": 'fill-extrusion',
-                  "source": 'overture',
-                  "source-layer": 'buildings',
-                  "paint": {
-                      'fill-extrusion-color': '#353535',
-                      'fill-extrusion-height': ['get', 'height'],
-                  }
-              }, */
             {
                 "id": 'roads',
                 "type": 'line',
                 "source": 'overture',
                 "source-layer": 'roads',
-                /*  'filter': [
-                     ">=", ["zoom"],
-                     ["match", ["get", "class"],
-                         ["motorway", "primary", "secondary"],
-                         10,
-                         ["tertiary", "residential"],
-                         13,
-                         14
-                     ]
-                 ], */
                 "paint": {
                     'line-color':
                         ['case',
                             ['==', ['get', 'class'], 'motorway'], '#717171',
                             ['==', ['get', 'class'], 'primary'], '#717171',
                             ['==', ['get', 'class'], 'secondary'], '#717171',
                             ['==', ['get', 'class'], 'tertiary'], '#717171',
-                            ['==', ['get', 'class'], 'residential'], '#717171',
-                            ['==', ['get', 'class'], 'livingStreet'], '#717171',
+                            ['==', ['get', 'class'], 'residential'], '#464646',
+                            ['==', ['get', 'class'], 'livingStreet'], '#464646',
                             ['==', ['get', 'class'], 'trunk'], '#5A5A5A',
                             ['==', ['get', 'class'], 'unclassified'], '#5A5A5A',
-                            ['==', ['get', 'class'], 'parkingAisle'], '#5A5A5A',
-                            ['==', ['get', 'class'], 'driveway'], '#5A5A5A',
-                            ['==', ['get', 'class'], 'pedestrian'], '#5A5A5A',
-                            ['==', ['get', 'class'], 'footway'], '#6a6a6a',
-                            ['==', ['get', 'class'], 'steps'], '#6a6a6a',
-                            ['==', ['get', 'class'], 'track'], '#6a6a6a',
-                            ['==', ['get', 'class'], 'cycleway'], '#6a6a6a',
-                            ['==', ['get', 'class'], 'unknown'], '#6a6a6a',
+                            ['==', ['get', 'class'], 'parkingAisle'], '#323232',
+                            ['==', ['get', 'class'], 'driveway'], '#1C1C1C',
+                            ['==', ['get', 'class'], 'pedestrian'], '#232323',
+                            ['==', ['get', 'class'], 'footway'], '#1C1C1C',
+                            ['==', ['get', 'class'], 'steps'], '#1C1C1C',
+                            ['==', ['get', 'class'], 'track'], '#1C1C1C',
+                            ['==', ['get', 'class'], 'cycleway'], '#1C1C1C',
+                            ['==', ['get', 'class'], 'unknown'], '#1C1C1C',
                             '#5A5A5A'
                         ],
                     'line-width':
@@ -106,22 +106,21 @@ const map = new maplibregl.Map({
                             ['==', ['get', 'class'], 'unclassified'], 2,
                             ['==', ['get', 'class'], 'parkingAisle'], 2,
                             ['==', ['get', 'class'], 'driveway'], 2,
-                            ['==', ['get', 'class'], 'pedestrian'], 2,
-                            ['==', ['get', 'class'], 'footway'], 2,
+                            ['==', ['get', 'class'], 'pedestrian'], 3,
+                            ['==', ['get', 'class'], 'footway'], 3,
                             ['==', ['get', 'class'], 'steps'], 2,
                             ['==', ['get', 'class'], 'track'], 2,
                             ['==', ['get', 'class'], 'cycleway'], 2,
                             ['==', ['get', 'class'], 'unknown'], 2,
                             2
                         ],
-
                 },
                 "layout": {
                     "line-cap": "round"
                 }
             },
             {
-                "id": 'road-names',
+                "id": 'roads-labels',
                 "type": 'symbol',
                 "source": 'overture',
                 "source-layer": 'roads',
@@ -144,14 +143,38 @@ const map = new maplibregl.Map({
                 "type": 'circle',
                 "source": 'overture',
                 "source-layer": 'places',
+
                 "paint": {
-                    "circle-color": "#fff",
-                    "circle-radius": 10
+                    "circle-color": "#4EDAD8",
+                    "circle-radius": 5,
+                    "circle-blur": 0.3,
+                    "circle-opacity": 0.8
+                }
+            },
+            {
+                "id": 'places-labels',
+                "type": 'symbol',
+                "source": 'overture',
+                "source-layer": 'places',
+                "layout": {
+                    'icon-allow-overlap': false,
+                    'text-allow-overlap': false,
+                    'text-anchor': "bottom",
+                    'text-radial-offset': 1,
+                    'symbol-placement': "point",
+                    'text-field': ['get', 'name'],
+                    'text-size': 10,
+                },
+                "paint": {
+                    'text-color': "#4EDAD8",
+                    'text-halo-color': "black",
+                    'text-halo-width': 3,
+                    'text-halo-blur': 2
                 }
             },
         ],
     },
-    center: [5.300171258675078, 51.69073045148227],
+    center: [4.91431, 52.34304],
     zoom: 14,
 });
 
@@ -182,8 +205,8 @@ map.on('click', function (e) {
     }
 });
 
-map.addControl(
+/* map.addControl(
     new maplibregl.NavigationControl({
         visualizePitch: true,
     })
-);
+); */

viewer/src/style.css

@@ -3,6 +3,7 @@ body {
     height: 100%;
     padding: 0;
     margin: 0;
+    background-color: black;
     font-family: Arial, sans-serif;
 }
 
@@ -13,26 +14,26 @@ body {
 
 #info {
     position: absolute;
-    top: 0.75rem;
-    left: 0.75rem;
-    max-width: 22rem;
-    border-radius: 0.25rem;
-    background-color: #353535;
-    color: #e2e2e2;
-    padding: 0.75rem;
+    bottom: 1rem;
+    left: 1rem;
+    color: #d3d3d3;
     z-index: 1;
 }
 
 #info .header {
-    font-size: 1.25rem;
+    font-size: 2.25rem;
     font-weight: 600;
-    line-height: 2rem;
+    line-height: 3rem;
 }
 
 #info .description {
     font-size: 0.85rem;
 }
 
+#info a {
+    color: inherit;
+}
+
 .maplibregl-popup-content h1 {
     font-size: 1rem;
     line-height: 0.1rem;