Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

reduce-number-of-spatial-queries #2045

Closed
wants to merge 2 commits into from
Closed

reduce-number-of-spatial-queries #2045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19e2f11
reduce number of spatial queries
michmuel Aug 29, 2024
aed8369
change any to all
michmuel Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pyramid_oereb/contrib/data_sources/interlis_2_3/models/theme.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from sqlalchemy import LargeBinary, String, Integer, Date, Text
from geoalchemy2.types import Geometry as GeoAlchemyGeometry
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.orm import query_expression


class Models(object):
Expand Down Expand Up @@ -350,6 +351,7 @@ class Geometry(Base):
PublicLawRestriction,
backref='geometries'
)
inside_real_estate = query_expression()

class MultilingualUri(Base):
"""
Expand Down
136 changes: 79 additions & 57 deletions pyramid_oereb/contrib/data_sources/interlis_2_3/sources/plr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from shapely.geometry import Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, \
GeometryCollection
from sqlalchemy import or_
from sqlalchemy.orm import with_expression
from sqlalchemy.orm import selectinload
from geoalchemy2.functions import ST_DWithin

Expand Down Expand Up @@ -412,44 +413,87 @@ def get_document_records(self, params, public_law_restriction_from_db):
document_records = self.from_db_to_document_records(documents_from_db)
return document_records

def collect_related_geometries_by_real_estate(self, session, real_estate):
def collect_related_geometries_by_real_estate_and_bbox(self, session, real_estate, bbox):
"""
Extracts all geometries in the topic which have spatial relation with the passed real estate

Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
real_estate (pyramid_oereb.lib.records.real_estate.RealEstateRecord): The real
estate in its record representation.
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.

Returns:
list: The result of the related geometries unique by the public law restriction id
"""
if self._tolerances is None:
query = session.query(self._model_).filter(
query = session.query(
self._model_
).options(
with_expression(
self._model_.inside_real_estate,
or_(
self._model_.point.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
),
self._model_.line.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
),
self._model_.surface.ST_Intersects(
from_shape(real_estate.limit, srid=Config.get('srid'))
)
)
)
).filter(
or_(
self._model_.point.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(real_estate.limit, srid=Config.get('srid')))
))
self._model_.point.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(bbox, srid=Config.get('srid')))
)
)
else:
query = session.query(self._model_).filter(
query = session.query(
self._model_
).options(
with_expression(
self._model_.inside_real_estate,
or_(
ST_DWithin(
self._model_.point,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
),
ST_DWithin(
self._model_.line,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
),
ST_DWithin(
self._model_.surface,
from_shape(real_estate.limit, srid=Config.get('srid')),
self._tolerances['ALL']
)
)
)
).filter(
or_(
ST_DWithin(
self._model_.point,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('Point', 0))
),
ST_DWithin(
self._model_.line,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('LineString', 0))
),
ST_DWithin(
self._model_.surface,
from_shape(real_estate.limit, srid=Config.get('srid')),
from_shape(bbox, srid=Config.get('srid')),
self._tolerances.get('ALL', self._tolerances.get('Polygon', 0))
)
))
)
)
return query.distinct(self._model_.public_law_restriction_id).options(
selectinload(self.models.Geometry.public_law_restriction)
.selectinload(self.models.PublicLawRestriction.geometries),
Expand All @@ -470,37 +514,6 @@ def collect_related_geometries_by_real_estate(self, session, real_estate):
.selectinload(self.models.MultilingualUri.localised_uri)
).all()

def collect_legend_entries_by_bbox(self, session, bbox, law_status):
"""
Extracts all legend entries in the topic which have spatial relation with the passed bounding box of
visible extent.

Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.
law_status (str): String of the law status for which the legend entries should be queried.

Returns:
list: The result of the related geometries unique by the public law restriction id and law status
"""
distinct_legend_entry_ids = []
geometries = session.query(self._model_).filter(
or_(
self._model_.point.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.line.ST_Intersects(from_shape(bbox, srid=Config.get('srid'))),
self._model_.surface.ST_Intersects(from_shape(bbox, srid=Config.get('srid')))
)).distinct(self._model_.public_law_restriction_id).options(
selectinload(self.models.Geometry.public_law_restriction)
).all()

for geometry in geometries:
if geometry.public_law_restriction.legend_entry_id not in distinct_legend_entry_ids\
and geometry.public_law_restriction.law_status == law_status:
distinct_legend_entry_ids.append(geometry.public_law_restriction.legend_entry_id)

return session.query(self.legend_entry_model).filter(
self.legend_entry_model.t_id.in_((distinct_legend_entry_ids))).all()

def read(self, params, real_estate, bbox):
"""
The read point which creates a extract, depending on a passed real estate.
Expand All @@ -525,10 +538,10 @@ def read(self, params, real_estate, bbox):
# We need to investigate more in detail

# Try to find geometries which have spatial relation with real estate
geometry_results = self.collect_related_geometries_by_real_estate(
session, real_estate
geometry_results = self.collect_related_geometries_by_real_estate_and_bbox(
session, real_estate, bbox
)
if len(geometry_results) == 0:
if any([x.inside_real_estate for x in geometry_results]):
# We checked if there are spatially related elements in database. But there is none.
# So we can stop here.
self.records = [EmptyPlrRecord(
Expand All @@ -537,20 +550,29 @@ def read(self, params, real_estate, bbox):
else:
# We found spatially related elements. This means we need to extract the actual plr
# information related to the found geometries.
law_status_of_geometry = []
# get distinct values of law_status for all geometries found
for geometry in geometry_results:
if (geometry.public_law_restriction.law_status not in law_status_of_geometry):
law_status_of_geometry.append(geometry.public_law_restriction.law_status)

legend_entries_from_db = []
# get legend_entries per law_status
for law_status in law_status_of_geometry:
legend_entry_with_law_status = [
self.collect_legend_entries_by_bbox(session, bbox, law_status),
law_status
]
legend_entries_from_db.append(legend_entry_with_law_status)
legend_entries_from_db = []
for law_status in list(set([x.public_law_restriction.law_status
for x in geometry_results])):
legend_entries_from_db.append(
[
session.query(
self.legend_entry_model
).filter(
self.legend_entry_model.t_id.in_(
list(
set([x.public_law_restriction.legend_entry_id
for x in geometry_results
if x.public_law_restriction.law_status == law_status]))
)
).all(),
law_status
]
)

# keep only the geometries that intersects with the real estate
geometry_results = [x for x in geometry_results if x.inside_real_estate is True]

self.records = []
for geometry_result in geometry_results:
Expand Down
5 changes: 3 additions & 2 deletions pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,19 +107,20 @@ def document_records_from_oereblex(self, params, geolink, law_status, oereblex_p
self._queried_geolinks[identifier] = self._oereblex_source.records
return self._queried_geolinks[identifier]

def collect_related_geometries_by_real_estate(self, session, real_estate):
def collect_related_geometries_by_real_estate_and_bbox(self, session, real_estate, bbox):
"""
Extracts all geometries in the topic which have spatial relation with the passed real estate

Args:
session (sqlalchemy.orm.Session): The requested clean session instance ready for use
real_estate (pyramid_oereb.lib.records.real_estate.RealEstateRecord): The real
estate in its record representation.
bbox (shapely.geometry.base.BaseGeometry): The bbox to search the records.

Returns:
list: The result of the related geometries unique by the public law restriction id
"""
return self.handle_collection(session, real_estate.limit).distinct(
return self.handle_collection(session, real_estate.limit, bbox).distinct(
self._model_.public_law_restriction_id
).options(
selectinload(self.models.Geometry.public_law_restriction)
Expand Down
2 changes: 2 additions & 0 deletions pyramid_oereb/contrib/data_sources/standard/models/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from geoalchemy2.types import Geometry as GeoAlchemyGeometry
from sqlalchemy.orm import relationship
from sqlalchemy_utils import JSONType
from sqlalchemy.orm import query_expression


NAMING_CONVENTION = {
Expand Down Expand Up @@ -371,6 +372,7 @@ class Geometry(base):
PublicLawRestriction,
backref='geometries'
)
inside_real_estate = query_expression()

return Geometry

Expand Down
Loading
Loading