diff --git a/crowdsourcer/management/commands/import_national_data.py b/crowdsourcer/management/commands/import_national_data.py index af8ee7bf..0c0fee57 100644 --- a/crowdsourcer/management/commands/import_national_data.py +++ b/crowdsourcer/management/commands/import_national_data.py @@ -27,8 +27,9 @@ class Command(BaseCommand): question_file = settings.BASE_DIR / "data" / "national_data.xlsx" - sheets = { - "Copy of Planning Q10B (pivot ta": { + sheets = [ + { + "sheet": "Copy of Planning Q10B (pivot ta", "section": "Planning & Land Use", "number": 10, "number_part": "b", @@ -47,13 +48,15 @@ class Command(BaseCommand): ], }, # XXX - negative points - "Planning Q11 - Fossil Fuel infr": { + { + "sheet": "Planning Q11 - Fossil Fuel infr", "section": "Planning & Land Use", "number": 11, "default_if_missing": 0, "missing_filter": {"type__in": ["CTY", "UTA"]}, }, - "Recycling": { + { + "sheet": "Recycling", "section": "Waste Reduction & Food", "number": 8, "header_row": 6, @@ -69,7 +72,8 @@ class Command(BaseCommand): ] # "options": [{"desc": "Criteria not met", "score": 0},] }, - "Residual Waste": { + { + "sheet": "Residual Waste", "section": "Waste Reduction & Food", "number": 9, "header_row": 6, @@ -83,7 +87,8 @@ class Command(BaseCommand): {"desc": "Less than 300kg residual waste per household", "score": 2}, ], }, - "Transport Q4 - 20mph": { + { + "sheet": "Transport Q4 - 20mph", "section": "Transport", "number": 4, "council_col": "Council name", @@ -92,7 +97,8 @@ class Command(BaseCommand): "missing_filter": {"type__in": ["CTY", "UTA"], "country": "england"}, "type": "yes_no", }, - "Transport Q6 - Active Travel England scores": { + { + "sheet": "Transport Q6 - Active Travel England scores", "section": "Transport", "number": 6, "header_row": 1, @@ -107,7 +113,8 @@ class Command(BaseCommand): {"desc": "Active travel capability rating 4/4", "score": 4}, ], }, - "Transport 8B - Bus Ridership": { + { + "sheet": "Transport 8B - Bus Ridership", "section": "Transport", "number": 8, "number_part": "b", @@ -121,7 +128,8 @@ class Command(BaseCommand): {"desc": "150 journeys per head of population", "score": 2}, ], }, - "Transport 10 - EV chargers": { + { + "sheet": "Transport 10 - EV chargers", "section": "Transport", "number": 10, "header_row": 1, @@ -141,18 +149,21 @@ class Command(BaseCommand): ], }, # XXX - negative points - "Transport 12a - Air Quality NO2": { + { + "sheet": "Transport 12a - Air Quality NO2", "section": "Transport", "number": 12, "number_part": "a", }, # XXX - negative points - "Transport 12b - Air Quality PM2.5": { + { + "sheet": "Transport 12b - Air Quality PM2.5", "section": "Transport", "number": 12, "number_part": "b", }, - "Biodiversity Q2 - Pesticides": { + { + "sheet": "Biodiversity Q2 - Pesticides", "section": "Biodiversity", "number": 2, "header_row": 1, @@ -161,7 +172,8 @@ class Command(BaseCommand): "default_if_missing": 0, "type": "yes_no", }, - "Biodiversity Q4 - Wildlife Sites ": { + { + "sheet": "Biodiversity Q4 - Wildlife Sites ", "section": "Biodiversity", "number": 4, "header_row": 2, @@ -169,7 +181,8 @@ class Command(BaseCommand): "score_col": "Point awarded", "type": "yes_no", }, - "Sheet20": { + { + "sheet": "Sheet20", "section": "Biodiversity", "number": 7, "council_col": "local authority council code", @@ -182,7 +195,8 @@ class Command(BaseCommand): {"desc": "4 or more Green Flag parks", "score": 2}, ], }, - "Gov&Fin Q11a": { + { + "sheet": "Gov&Fin Q11a", "section": "Governance & Finance", "number": 11, "number_part": "a", @@ -195,7 +209,8 @@ class Command(BaseCommand): {"desc": "Divestment of pensions's investments", "score": 2}, ], }, - "Gov&Fin Q11b": { + { + "sheet": "Gov&Fin Q11b", "section": "Governance & Finance", "number": 11, "number_part": "b", @@ -209,7 +224,8 @@ class Command(BaseCommand): {"desc": "All Fossil fuels divestment", "score": 2}, ], }, - "Gov&Fin Q4": { + { + "sheet": "Gov&Fin Q4", "section": "Governance & Finance", "number": 4, "header_row": 6, @@ -224,7 +240,8 @@ class Command(BaseCommand): {"desc": "10% or more emission reduction", "score": 3}, ], }, - "Gov&Fin Q5 CA": { + { + "sheet": "Gov&Fin Q5 CA", "section": "Governance & Finance (CA)", "number": 5, "header_row": 6, @@ -238,7 +255,8 @@ class Command(BaseCommand): {"desc": "10% or more emission reduction", "score": 3}, ], }, - "Gov&Fin Q8": { + { + "sheet": "Gov&Fin Q8", "section": "Governance & Finance", "number": 8, "gss_col": "local-authority-code", @@ -251,7 +269,8 @@ class Command(BaseCommand): {"desc": "more than 2% of staff working on climate", "score": 3}, ], }, - "Gov&Fin Q8 CA": { + { + "sheet": "Gov&Fin Q8 CA", "section": "Governance & Finance (CA)", "number": 9, "header_row": 1, @@ -265,7 +284,8 @@ class Command(BaseCommand): {"desc": "more than 2% of staff working on climate", "score": 3}, ], }, - "EPC - England & Wales": { + { + "sheet": "EPC - England & Wales", "section": "Buildings & Heating", "number": 7, "header_row": 5, @@ -279,13 +299,15 @@ class Command(BaseCommand): {"desc": "90% rated EPC C or above", "score": 3}, ], }, - "EPC - Scotland": { + { + "sheet": "EPC - Scotland", "section": "Buildings & Heating", "number": 7, "header_row": 5, "council_col": "Local authority", "score_col": "Tiered mark", "type": "select_one", + "skip_clear_existing": True, "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "50% rated EPC C or above", "score": 1}, @@ -293,13 +315,15 @@ class Command(BaseCommand): {"desc": "90% rated EPC C or above", "score": 3}, ], }, - "EPC - NI": { + { + "sheet": "EPC - NI", "section": "Buildings & Heating", "number": 7, "header_row": 2, "gss_col": "Northern Irish Council GSS code", "score_col": "Tiered mark", "type": "select_one", + "skip_clear_existing": True, "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "50% rated EPC C or above", "score": 1}, @@ -307,7 +331,8 @@ class Command(BaseCommand): {"desc": "90% rated EPC C or above", "score": 3}, ], }, - "Collab & Engagement Q11": { + { + "sheet": "Collab & Engagement Q11", "section": "Collaboration & Engagement", "number": 11, "header_row": 1, @@ -323,10 +348,11 @@ class Command(BaseCommand): }, ], }, - } + ] - ca_sheets = { - "Gov&Fin Q11a": { + ca_sheets = [ + { + "sheet": "Gov&Fin Q11a", "section": "Governance & Finance (CA)", "number": 12, "number_part": "a", @@ -339,7 +365,8 @@ class Command(BaseCommand): {"desc": "Divestment of pensions's investments", "score": 2}, ], }, - "Gov&Fin Q11b": { + { + "sheet": "Gov&Fin Q11b", "section": "Governance & Finance (CA)", "number": 12, "number_part": "b", @@ -353,7 +380,8 @@ class Command(BaseCommand): {"desc": "All Fossil fuels divestment", "score": 2}, ], }, - "Transport Q6 - Active Travel England scores": { + { + "sheet": "Transport Q6 - Active Travel England scores", "section": "Transport (CA)", "number": 7, "header_row": 1, @@ -368,7 +396,8 @@ class Command(BaseCommand): {"desc": "Active travel capability rating 4/4", "score": 4}, ], }, - "Transport 8B - Bus Ridership": { + { + "sheet": "Transport 8B - Bus Ridership", "section": "Transport (CA)", "number": 4, "number_part": "b", @@ -382,7 +411,8 @@ class Command(BaseCommand): {"desc": "150 journeys per head of population", "score": 2}, ], }, - "Transport 10 - EV chargers": { + { + "sheet": "Transport 10 - EV chargers", "section": "Transport (CA)", "number": 8, "header_row": 1, @@ -391,14 +421,17 @@ class Command(BaseCommand): "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, - {"desc": "60+ chargers per 100,000 people (but less than 434 per 100k)", "score": 1}, + { + "desc": "60+ chargers per 100,000 people (but less than 434 per 100k)", + "score": 1, + }, { "desc": "434+ chargers per 100,000 people", "score": 2, }, ], }, - } + ] def add_arguments(self, parser): parser.add_argument( @@ -439,8 +472,9 @@ def get_df(self, sheet, details): return df - def clear_existing_answers(self, q): - Response.objects.filter(question=q, response_type=self.rt).delete() + def clear_existing_answers(self, q, details): + if details.get("skip_clear_existing", None) is None: + Response.objects.filter(question=q, response_type=self.rt).delete() def popuplate_council_lookup(self): df = self.get_df("council names", {}) @@ -630,7 +664,7 @@ def handle_sheet(self, sheet, details, user): self.print_info("") df = self.get_df(sheet, details) q = self.get_question(details) - self.clear_existing_answers(q) + self.clear_existing_answers(q, details) self.add_options(q, details) self.import_answers(user, self.rt, df, q, details) @@ -646,13 +680,15 @@ def handle(self, quiet: bool = False, only_sheet: str = None, *args, **kwargs): self.popuplate_council_lookup() user, _ = User.objects.get_or_create(username="National_Importer") self.rt = ResponseType.objects.get(type="Audit") - for sheet, details in self.sheets.items(): + for details in self.sheets: + sheet = details["sheet"] if only_sheet is not None and sheet != only_sheet: continue self.handle_sheet(sheet, details, user) - for sheet, details in self.ca_sheets.items(): + for details in self.ca_sheets: + sheet = details["sheet"] if only_sheet is not None and sheet != only_sheet: continue