diff --git a/operators/entities/__init__.py b/operators/entities/__init__.py index 22c63c1..ba4fb86 100644 --- a/operators/entities/__init__.py +++ b/operators/entities/__init__.py @@ -9,7 +9,7 @@ from openlocationcode import openlocationcode as olc -from srm_tools.stats import Stats +from srm_tools.stats import Report, Stats from srm_tools.update_table import airtable_updater from srm_tools.guidestar_api import GuidestarAPI from srm_tools.budgetkey import fetch_from_budgetkey @@ -44,6 +44,14 @@ def fetchEntityFromBudgetKey(regNum): def updateOrgFromSourceData(ga: GuidestarAPI, stats: Stats): + + unknown_entity_ids = Report( + 'Entities: Unknown ID Report', + 'entities-unknown-id-report', + ['id', 'kind'], + ['id'] + ) + def func(rows): for row in rows: regNums = [row['id']] @@ -83,9 +91,11 @@ def func(rows): row.update(data['data']) else: stats.increase('Entities: Unknown ID') + unknown_entity_ids.add(row) if 'name' in row: row['name'] = row['name'].replace(' (חל"צ)', '').replace(' (ע"ר)', '') yield row + unknown_entity_ids.save() return func def recent_org(row): diff --git a/operators/manual_data_entry/__init__.py b/operators/manual_data_entry/__init__.py index 559259e..cbfce45 100644 --- a/operators/manual_data_entry/__init__.py +++ b/operators/manual_data_entry/__init__.py @@ -1,21 +1,29 @@ import dataflows as DF -from dataflows_airtable import load_from_airtable +from dataflows_airtable import load_from_airtable, AIRTABLE_ID_FIELD from conf import settings from srm_tools.logger import logger -from srm_tools.stats import Stats +from srm_tools.stats import Report, Stats from .mde_utils import load_manual_data from .external import main as load_external_data stats = Stats() def mde_prepare(): + + unknown_entity_ids = Report( + 'Manual Data Entry: No Org ID or Org Name Report', + 'manual-data-entry-no-org-id-or-org-name', + ['Org Id', 'Org Name', 'Service Name'], + [AIRTABLE_ID_FIELD] + ) + source_flow = DF.Flow( load_from_airtable(settings.AIRTABLE_DATAENTRY_BASE, settings.AIRTABLE_SERVICE_TABLE, settings.AIRTABLE_VIEW, settings.AIRTABLE_API_KEY), DF.filter_rows(lambda r: r.get('Org Id') != 'dummy'), - stats.filter_with_stat('Manual Data Entry: No Org ID or Org Name', lambda r: (r.get('Org Id') or r.get('Org Name'))), stats.filter_with_stat('Manual Data Entry: Entry not ready to publish', lambda r: r.get('Status') == 'בייצור'), + stats.filter_with_stat('Manual Data Entry: No Org ID or Org Name', lambda r: (r.get('Org Id') or r.get('Org Name')), report=unknown_entity_ids), ) data_sources = DF.Flow(