Skip to content

Commit

Permalink
moved manualcovenant parcel matching back to post-save signal to avoi…
Browse files Browse the repository at this point in the history 
…d error
  • Loading branch information
@mikejcorey
mikejcorey committed Mar 6, 2024
1 parent 18a4b9f commit b692763
Show file tree
Hide file tree
Showing 3 changed files with 290 additions and 20 deletions.
12 changes: 12 additions & 0 deletions apps/parcel/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,18 @@ class Meta:
ordering = ('-id',)


class AllCovenantedDocsCSVExport(models.Model):
workflow = models.ForeignKey(
"zoon.ZooniverseWorkflow", null=True, on_delete=models.SET_NULL)
csv = models.FileField(
storage=PublicMediaStorage(), upload_to="main_exports/", null=True)
doc_count = models.IntegerField()
created_at = models.DateTimeField()

class Meta:
ordering = ('-id',)


class GeoJSONExport(models.Model):
workflow = models.ForeignKey(
"zoon.ZooniverseWorkflow", null=True, on_delete=models.SET_NULL)
Expand Down
180 changes: 179 additions & 1 deletion apps/parcel/utils/export_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from django.contrib.gis.db.models.functions import AsWKT

from apps.parcel.models import Parcel
from apps.zoon.models import ZooniverseSubject
from apps.zoon.models import ZooniverseSubject, ManualCovenant
from apps.zoon.models import MATCH_TYPE_OPTIONS, MANUAL_COV_OPTIONS


Expand Down Expand Up @@ -241,3 +241,181 @@ def build_validation_df(workflow):
validation_df = pd.DataFrame(validation_subjects)[VALIDATION_ATTRS]

return validation_df


def deduped_str_list(series):
uniques = set(s for s in series if s)
if len(uniques) > 0:
return '; '.join(uniques)
return ''


def build_all_covenanted_docs_df(workflow):
'''Export all copies of covenanted documents from both ZooniverseSubject and ManualCovenant'''

ALL_DOCS_ATTRIBUTES = [
'workflow',
'cov_type',
'db_id',
'doc_num',
'deed_year',
'deed_date',
'cov_text',
'seller',
'buyer',
'add_cov',
'block_cov',
'lot_cov',
'is_mapped',
'addresses',
'cities',
'state',
'cnty_pins',
'join_strgs',
'match_type',
'manual_cx',
'dt_updated',
'zn_subj_id',
'zn_dt_ret',
'image_ids',
'med_score',
]

zoon_covenanted_subjects = ZooniverseSubject.all_covenanted_docs_objects.filter(
workflow=workflow
).values(
'db_id',
'workflow',
'is_mapped',
'deed_date_final', # Need to rename with pd
'cov_text',
'image_ids',
'zn_subj_id',
'zn_dt_ret',
'med_score',
'manual_cx',
'add_cov',
'block_cov',
'lot_cov',
'join_strgs',
'mapped_address',
'mapped_city',
'mapped_state',
'mapped_parcel_pin',
'seller_final', # Need to rename with pd
'buyer_final', # Need to rename with pd
'dt_updated',
'doc_num',
'cov_type',
'match_type_final', # Need to rename with pd =Value('unmapped')
)

# TK
all_manual_covenants = ManualCovenant.all_covenanted_docs_objects.filter(
workflow=workflow
).values(
'db_id',
'workflow',
'is_mapped',
'deed_date',
'cov_text',
# 'image_ids', # Need to set to null with pd
'zn_subj_id',
'zn_dt_ret',
'med_score',
'manual_cx',
'add_cov',
'block_cov',
'lot_cov',
'join_strgs',
'mapped_address',
'mapped_city',
'mapped_state',
'mapped_parcel_pin',
'seller',
'buyer',
'dt_updated',
'doc_num',
'cov_type', # Need to rename with pd to match_type
# 'cov_type_manual', # Need to rename with pd to cov_type (janky!)
# 'match_type', # Need to rename with pd =Value('unmapped')
)


# TODO: fix image ids

# Convert to pandas DF
zoon_covenanted_docs_expanded_df = pd.DataFrame(zoon_covenanted_subjects)
zoon_covenanted_docs_expanded_df.rename(columns={
'deed_date_final': 'deed_date',
'seller_final': 'seller',
'buyer_final': 'buyer',
'city_final': 'city',
'match_type_final': 'match_type',
}, inplace=True)

manual_covenanted_docs_expanded_df = pd.DataFrame(all_manual_covenants)

manual_covenanted_docs_expanded_df.rename(
columns={'cov_type': 'match_type'}, inplace=True
)
manual_covenanted_docs_expanded_df['image_ids'] = ''
manual_covenanted_docs_expanded_df['cov_type'] = 'manual'

all_covenanted_docs_expanded_df = pd.concat([zoon_covenanted_docs_expanded_df, manual_covenanted_docs_expanded_df])

# Drop parcel-level fields to get de-duped list of subjects
all_covenanted_docs_df = all_covenanted_docs_expanded_df.drop(columns=[
'mapped_address',
'mapped_state',
'mapped_parcel_pin',
]).drop_duplicates(subset=["cov_type", "db_id"])

# created group-by dfs of address, city, state, cnty_pin to deal with multiple parcels
address_list = all_covenanted_docs_expanded_df[[
'cov_type',
'db_id',
'mapped_address'
]].groupby(["cov_type", "db_id"])['mapped_address'].apply(deduped_str_list).reset_index(name='addresses')

city_list = all_covenanted_docs_expanded_df[[
'cov_type',
'db_id',
'mapped_city'
]].groupby(["cov_type", "db_id"])['mapped_city'].apply(deduped_str_list).reset_index(name='cities')

state_list = all_covenanted_docs_expanded_df[[
'cov_type',
'db_id',
'mapped_state'
]].groupby(["cov_type", "db_id"])['mapped_state'].apply(deduped_str_list).reset_index(name='state')

pin_list = all_covenanted_docs_expanded_df[[
'cov_type',
'db_id',
'mapped_parcel_pin'
]].groupby(["cov_type", "db_id"])['mapped_parcel_pin'].apply(deduped_str_list).reset_index(name='cnty_pins')

# Merged groups of addresses back to de-duped list of docs
all_covenanted_docs_df = all_covenanted_docs_df.merge(
address_list,
how="left",
on=["cov_type", "db_id"]
).merge(
city_list,
how="left",
on=["cov_type", "db_id"]
).merge(
state_list,
how="left",
on=["cov_type", "db_id"]
).merge(
pin_list,
how="left",
on=["cov_type", "db_id"]
)

all_covenanted_docs_df['deed_date'] = pd.to_datetime(all_covenanted_docs_df['deed_date'])
all_covenanted_docs_df['deed_year'] = all_covenanted_docs_df['deed_date'].dt.year

return all_covenanted_docs_df[ALL_DOCS_ATTRIBUTES]
118 changes: 99 additions & 19 deletions apps/zoon/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,46 @@ def get_queryset(self):
)


class AllCovenantedDocsZooniverseManager(models.Manager):
'''This model manager is mainly used for exports of all covenanted documents.
It returns a list of all covenant/parcel combinations, so should not be used for covenant counts.
It is reduced to a per-docnumber list in the export stage in apps/parcel/utils/export_utils.py
The main model manager used for covenant exports is in apps/parcel/models.py.
Unlike the main exporter, de-duping is not done here to eliminate multiple occurences
of the same document.'''

def get_queryset(self):

return super().get_queryset().filter(
bool_covenant=True
).annotate(
db_id=F('pk'),
# deed_date=F('deed_date_final'), # Need to rename with pd
is_mapped=F('bool_parcel_match'),
cov_text=F('covenant_text_final'),
zn_subj_id=F('zoon_subject_id'),
zn_dt_ret=F('dt_retired'),
med_score=F('median_score'),
manual_cx=F('bool_manual_correction'),
add_cov=F('addition_final'),
block_cov=F('block_final'),
lot_cov=F('lot_final'),
join_strgs=F('join_candidates'),
# city_cov=F('city_final'),
# seller=F('seller_final'), # Need to rename with pd
# buyer=F('buyer_final'), # Need to rename with pd
dt_updated=F('date_updated'),
doc_num=F('deedpage_doc_num'),
cov_type=Value('zooniverse'),
# match_type=Value('unmapped') # Need to rename with pd
# addresses=F('parcel_addresses'),
mapped_address=F('parcel_matches__street_address'),
mapped_city=F('parcel_matches__city'),
mapped_state=F('parcel_matches__state'),
mapped_parcel_pin=F('parcel_matches__pin_primary'),
)


class ValidationZooniverseManager(models.Manager):
'''This model manager is used to run statistics on retired subjects and
for other analysis purposes'''
Expand Down Expand Up @@ -206,6 +246,7 @@ class ZooniverseSubject(models.Model):

objects = models.Manager()
unmapped_objects = UnmappedZooniverseManager()
all_covenanted_docs_objects = AllCovenantedDocsZooniverseManager()
validation_objects = ValidationZooniverseManager()

def __str__(self):
Expand Down Expand Up @@ -556,6 +597,41 @@ def model_delete(sender, instance, **kwargs):
instance.zooniverse_subject.save()


class AllCovenantedDocsManualCovenantManager(models.Manager):
'''This model manager is mainly used for exports of all covenanted documents.
It returns a list of all covenant/parcel combinations, so should not be used for covenant counts.
It is reduced to a per-docnumber list in the export stage in apps/parcel/utils/export_utils.py
The main model manager used for covenant exports is in apps/parcel/models.py.
Unlike the main exporter, de-duping is not done here to eliminate multiple occurences
of the same document.'''

def get_queryset(self):

return super().get_queryset().filter(
# bool_covenant=True # They all are covenants
bool_confirmed=True
).annotate(
db_id=F('pk'),
# deed_date=F('deed_date_final'), # Need to rename with pd
is_mapped=F('bool_parcel_match'),
cov_text=F('covenant_text'),
zn_subj_id=Value(''),
zn_dt_ret=Value(''),
med_score=Value(''),
manual_cx=Value(''),
add_cov=F('addition'),
block_cov=F('block'),
lot_cov=F('lot'),
join_strgs=F('join_candidates'),
dt_updated=F('date_updated'),
# cov_type_manual=Value('manual'),
mapped_address=F('parcel_matches__street_address'),
mapped_city=F('parcel_matches__city'),
mapped_state=F('parcel_matches__state'),
mapped_parcel_pin=F('parcel_matches__pin_primary'),
)


MANUAL_COV_OPTIONS = (
('PS', 'Public submission (single property)'),
('SE', 'Something else'),
Expand Down Expand Up @@ -588,6 +664,9 @@ class ManualCovenant(models.Model):
date_added = models.DateTimeField(auto_now_add=True)
date_updated = models.DateTimeField(auto_now=True)

objects = models.Manager()
all_covenanted_docs_objects = AllCovenantedDocsManualCovenantManager()

# TODO: manual geometries

def __str__(self):
Expand Down Expand Up @@ -646,31 +725,32 @@ def check_parcel_match(self, parcel_lookup=None):
self.parcel_matches.all().update(bool_covenant=True)

def save(self, *args, **kwargs):
# Can pass parcel lookup for bulk matches
self.check_parcel_match(kwargs.get('parcel_lookup', None))
if 'parcel_lookup' in kwargs:
del kwargs['parcel_lookup']


super(ManualCovenant, self).save(*args, **kwargs)


# @receiver(models.signals.post_save, sender=ManualCovenant)
# def manual_cov_post_save(sender, instance=None, created=False, **kwargs):
@receiver(models.signals.post_save, sender=ManualCovenant)
def manual_cov_post_save(sender, instance=None, created=False, **kwargs):

# if not instance:
# return
if not instance:
return

# if hasattr(instance, '_dirty'):
# return
if hasattr(instance, '_dirty'):
return

# # Can pass parcel lookup for bulk matches
# instance.check_parcel_match(kwargs.get('parcel_lookup', None))

# try:
# instance._dirty = True
# instance.save()
# finally:
# del instance._dirty
# # Can pass parcel lookup for bulk matches
# instance.check_parcel_match(kwargs.get('parcel_lookup', None))
# Can pass parcel lookup for bulk matches
print('Checking parcel matches')
instance.check_parcel_match(kwargs.get('parcel_lookup', None))
if 'parcel_lookup' in kwargs:
del kwargs['parcel_lookup']

try:
instance._dirty = True
instance.save()
finally:
del instance._dirty


SUPPORTING_DOC_TYPES = (
Expand Down

0 comments on commit b692763

Please sign in to comment.