Merge branch 'py3_11' into master

.gitlab-ci.yml

@@ -0,0 +1,14 @@
+image: python:3.9
+
+variables:
+  PYTHONPATH: `dirname $(realpath $0)`
+
+before_script:
+  - pip3 install -r requirements.txt
+stages:
+  - test
+
+unit_tests:
+  stage: test
+  script:
+    - python3 ${PYTHONPATH}/test/test_create_db.py

.travis.yml

@@ -13,8 +13,8 @@ install:
   - "pip install --pre -r requirements.txt"
 
 env:
-  - PYTHONPATH="."
+  - PYTHONPATH= `dirname $(realpath $0)`
 
 # command to run test
 script:
-  - python test/test_create_db.py
+  - python3 ${PYTHONPATH}/test/test_create_db.py

build_osmloader_docker.sh

@@ -1,3 +1,7 @@
 #!/bin/bash
 
+if [[ $(uname -m) == 'arm64' ]]; then
+  export DOCKER_DEFAULT_PLATFORM=linux/amd64
+fi
+
 docker build -t osm_loader -f osm_loader.Dockerfile .

doc/01_hu_working.md

@@ -39,7 +39,7 @@ Az egyes POI típusokra jellemző keresési sugár az app.conf fájl alapján ke
 ```
 geo.default.poi.distance=70
 geo.amenity.atm.poi.distance=20
-geo.shop.conveience.poi.distance=50
+geo.shop.convenience.poi.distance=50
 geo.amenity.post.office.poi.distance=260
 ```
 
@@ -63,7 +63,7 @@ Az egyes POI típusokra jellemző keresési sugár az app.conf fájl alapján ke
 ```
 geo.default.poi.distance=70
 geo.amenity.atm.poi.distance=20
-geo.shop.conveience.poi.distance=50
+geo.shop.convenience.poi.distance=50
 geo.amenity.post.office.poi.distance=260
 ```
 

doc/02_debugging.md

@@ -209,3 +209,34 @@ WHERE (amenity='post_office') AND osm_id < 0  AND name ~* 'posta'
     AND ST_DWithin(ST_Buffer(way,10),ST_Transform(point.geom, 3857), 430)
 ORDER BY priority ASC, distance ASC;
 ```
+
+## Additional indexes
+
+```
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE INDEX IF NOT EXISTS i_street_type ON street_type(street_type);
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_way ON planet_osm_point(way);
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_addr ON planet_osm_point("addr:postcode","addr:city","addr:street","addr:housenumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_addrcon ON planet_osm_point("addr:city","addr:conscriptionnumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_amenity ON planet_osm_point(amenity);
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_shop ON planet_osm_point(shop);
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_name ON planet_osm_point(name);
+CREATE INDEX IF NOT EXISTS i_planet_osm_point_brand ON planet_osm_point(brand);
+
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_way ON planet_osm_line using gist(way);
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_addr ON planet_osm_line("addr:postcode","addr:city","addr:street","addr:housenumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_addrcon ON planet_osm_line("addr:city","addr:conscriptionnumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_amenity ON planet_osm_line(amenity);
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_shop ON planet_osm_line(shop);
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_name ON planet_osm_line(name);
+CREATE INDEX IF NOT EXISTS i_planet_osm_line_brand ON planet_osm_line(brand);
+
+
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_way ON planet_osm_polygon using gist (way);
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_addr ON planet_osm_polygon("addr:postcode","addr:city","addr:street","addr:housenumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_addrcon ON planet_osm_polygon("addr:city","addr:conscriptionnumber");
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_amenity ON planet_osm_polygon(amenity);
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_shop ON planet_osm_polygon(shop);
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_name ON planet_osm_polygon(name);
+CREATE INDEX IF NOT EXISTS i_planet_osm_polygon_brand ON planet_osm_polygon(brand);
+```

docker-compose.yml

@@ -6,7 +6,7 @@ volumes:
   opm_outputdata:
 services:
   db:
-    image: postgis/postgis:13-3.0
+    image: postgis/postgis:14-3.2
     container_name: opm_db
     environment:
       - POSTGRES_USER=poi
@@ -20,19 +20,21 @@ services:
       # - ../container/dbdata-9.6:/var/lib/postgresql/data/
       - ../container/dbdata-fresh:/var/lib/postgresql/data/
       # - ${PWD}/osm2pgsql/postgresql.conf:/var/lib/postgresql/data/postgresql.conf
-    restart: on-failure:10
+    restart: on-failure
     oom_score_adj: -100
   osm_load_app:
     image: osm_loader:latest
     container_name: opm_osm_load_app
     environment:
       - PGPASSWORD=poitest
-      - OSM2PGSQL_VERSION=1.0.0
+      - OSM2PGSQL_VERSION=1.6.0
     volumes:
       - ../container/osmdata:/opm/osm
     depends_on:
       - db
-    restart: on-failure:10
+    links:
+      - db
+    restart: on-failure
     oom_score_adj: -100
     mem_limit: 4g
   poi_load_app:
@@ -44,6 +46,6 @@ services:
       - ../container/downloaddata:/opm/osm_poi_matchmaker/cache_url
     depends_on:
       - db
-    restart: on-failure:10
+    restart: on-failure
     oom_score_adj: -100
     mem_limit: 4g

osm2pgsql/default.style

@@ -143,6 +143,8 @@ node,way   military     text         polygon
 node,way   motorcar     text         linear
 node,way   name         text         linear
 node,way   natural      text         polygon  # natural=coastline tags are discarded by a hard coded rule in osm2pgsql
+node,way   network      text         linear
+node,way   post_office  text         linear
 node,way   office       text         polygon
 node,way   oneway       text         linear
 node,way   operator     text         linear
@@ -155,7 +157,7 @@ node,way   contact:email  text         linear
 node,way   contact:phone  text         linear
 node,way   contact:fax    text         linear
 node,way   contact:youtube text        linear
-node,way   contact:instagram text      linears
+node,way   contact:instagram text      linear
 node,way   place        text         polygon
 node       poi          text         linear
 node,way   population   text         linear
@@ -176,6 +178,8 @@ node,way   tower:type   text         linear
 way        tracktype    text         linear
 node,way   tunnel       text         linear
 node,way   vending      text         linear
+node,way   parcel_mail_in text       linear
+node,way   parcel_pickup text        linear
 node,way   water        text         polygon
 node,way   waterway     text         polygon
 node,way   wetland      text         polygon
@@ -201,6 +205,11 @@ node,way   socket:type2:output text     linear
 node,way   manufacturer text     linear
 node,way   model        text     linear
 
+
+### Add these additional tags
+node,way   ref:bkk text         linear
+node,way   ref:mav text         linear
+node,way   ref:volanbusz text         linear
 # Area tags
 # We don't make columns for these tags, but objects with them are areas.
 # Mainly for use with hstore

osm2pgsql/postgresql.conf

@@ -61,7 +61,7 @@ listen_addresses = '*'
 					# defaults to 'localhost'; use '*' for all
 					# (change requires restart)
 #port = 5432				# (change requires restart)
-max_connections = 20			# (change requires restart)
+max_connections = 300			# (change requires restart)
 #superuser_reserved_connections = 3	# (change requires restart)
 #unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
 					# (change requires restart)
@@ -205,12 +205,12 @@ max_parallel_workers = 8		# maximum number of max_worker_processes that
 					#   fsync_writethrough
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
-#wal_compression = off			# enable compression of full-page writes
+wal_compression = on			# enable compression of full-page writes
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
 #wal_recycle = on			# recycle WAL files
-wal_buffers = 16MB			# min 32kB, -1 sets based on shared_buffers
+wal_buffers = 2MB			# min 32kB, -1 sets based on shared_buffers
 					# (change requires restart)
 #wal_writer_delay = 200ms		# 1-10000 milliseconds
 #wal_writer_flush_after = 1MB		# measured in pages, 0 disables
@@ -221,9 +221,9 @@ wal_buffers = 16MB			# min 32kB, -1 sets based on shared_buffers
 # - Checkpoints -
 
 #checkpoint_timeout = 5min		# range 30s-1d
-max_wal_size = 12GB
-min_wal_size = 80MB
-#checkpoint_completion_target = 0.5	# checkpoint target duration, 0.0 - 1.0
+max_wal_size = 8GB
+min_wal_size = 1GB
+checkpoint_completion_target = 0.5	# checkpoint target duration, 0.0 - 1.0
 #checkpoint_flush_after = 256kB		# measured in pages, 0 disables
 checkpoint_warning = 30s		# 0 disables
 
@@ -384,7 +384,7 @@ random_page_cost = 1.3			# same scale as above
 
 #min_parallel_table_scan_size = 8MB
 #min_parallel_index_scan_size = 512kB
-#effective_cache_size = 4GB
+effective_cache_size = 4GB
 
 # - Genetic Query Optimizer -
 

osm_poi_matchmaker.Dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:stretch
+FROM python:3.11-slim-bullseye
 
 LABEL maintainer Kálmán Szalai (KAMI) <kami911@gmail.com>
 
@@ -10,18 +10,17 @@ ENV PATH /usr/local/bin:$PATH
 ENV LANG C.UTF-8
 # extra dependencies (over what buildpack-deps already includes)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-  python3 \
-  python3-pip \
-  build-essential \
-  python3-dev \
-  python3-setuptools \
-  python3-wheel \
-  apt-utils \
-    && rm -rf /var/lib/apt/lists/*
+   build-essential \
+   apt-utils \
+   libgdal-dev \
+   libxml2-dev \
+   libxslt-dev \
+ && python -m pip install --upgrade pip \
+ && rm -rf /var/lib/apt/lists/*
 
 COPY ./requirements.txt /opm/requirements.txt
 
-RUN pip3 install -r /opm/requirements.txt
+RUN python3.11 --version && python3.11 -m pip --version && python3.11 -m pip install -r /opm/requirements.txt
 
 COPY ./osm_poi_matchmaker /opm/osm_poi_matchmaker
 ENV PYTHONPATH /opm/

osm_poi_matchmaker/app.conf-template

@@ -6,6 +6,9 @@ db.write.port=5432
 db.write.username=poi
 db.write.password=poitest
 db.poi.database=poi
+db.enable.analyze=False
+db.enable.query_log=False
+db.enable.huge_query=False
 
 # Directory settings
 dir.output=output
@@ -17,7 +20,7 @@ geo.default.poi.unsafe.distance=5
 geo.default.poi.perfect.distance=300
 geo.default.poi.road.distance=600
 geo.amenity.atm.poi.distance=20
-geo.shop.conveience.poi.distance=50
+geo.shop.convenience.poi.distance=50
 geo.amenity.post.office.poi.distance=200
 geo.prefer.osm.postcode=True
 geo.alternative.opening.hours=False
@@ -26,5 +29,9 @@ geo.alternative.opening.hours.tag=opening_hours:covid19
 download.verify.link=True
 download.use.cached.data=False
 
-dataproviders.modules.available = hu_aldi, hu_avia, hu_benu, hu_budapest_bank, hu_cba, hu_cib_bank, hu_dm, hu_emobi_ev, hu_foxpost, hu_jysk, hu_kh_bank, hu_kulcs_patika, hu_magnet_bank, hu_mobiliti_ev, hu_mobil_petrol,hu_mol_bubi, hu_mol_plugee_ev, hu_mol, hu_oil, hu_omv, hu_penny_market, hu_pepco, hu_posta_json, hu_posta, hu_rossmann, hu_sber_bank, hu_shell, hu_spar, hu_tesco, hu_tom_market, hu_yves_rocher
-dataproviders.modules.enable = hu_posta, hu_aldi, hu_avia, hu_benu, hu_budapest_bank, hu_cba, hu_cib_bank, hu_dm, hu_emobi_ev, hu_foxpost, hu_jysk, hu_kh_bank, hu_kulcs_patika, hu_magnet_bank, hu_mobiliti_ev, hu_mobil_petrol, hu_mol_bubi, hu_mol_plugee_ev, hu_mol, hu_oil, hu_omv, hu_penny_market, hu_pepco, hu_posta_json, hu_rossmann, hu_sber_bank, hu_shell, hu_spar, hu_tesco, hu_tom_market, hu_yves_rocher
+use.general.source.website.date=True
+use.general.source.website.date.tag=source:date
+
+dataproviders.limit.elements=None
+dataproviders.modules.available = hu_aldi, hu_avia, hu_benu, hu_cba, hu_ccc, hu_cib_bank, hu_deichmann, hu_dm, hu_easybox, hu_fire_station, hu_foxpost, hu_gls, hu_jysk, hu_kh_bank, hu_kulcs_patika, hu_magnet_bank, hu_mobiliti_ev, hu_mobil_petrol,hu_mol_bubi, hu_mol_plugee_ev, hu_mol, hu_obi, hu_ofotert, hu_oil, hu_omv, hu_penny_market, hu_pepco, hu_pingvin_patika, hu_posta_json, hu_posta, hu_rossmann, hu_sber_bank, hu_shell, hu_spar, hu_tesco, hu_tom_market, hu_yves_rocher, hu_volanbusz
+dataproviders.modules.enable = hu_posta, hu_aldi, hu_avia, hu_benu, hu_cba, hu_ccc, hu_cib_bank, hu_deichmann, hu_dm, hu_easybox, hu_fire_station, hu_foxpost, hu_gls, hu_jysk, hu_kh_bank, hu_kulcs_patika, hu_magnet_bank, hu_mobiliti_ev, hu_mobil_petrol, hu_mol_bubi, hu_mol_plugee_ev, hu_mol, hu_obi, hu_ofotert, hu_oil, hu_omv, hu_penny_market, hu_pepco, hu_pingvin_patika, hu_posta_json, hu_rossmann, hu_sber_bank, hu_shell, hu_spar, hu_tesco, hu_tom_market, hu_yves_rocher, hu_volanbusz