Skip to content

Commit

Permalink
make sheets into array rather than dict
Browse files Browse the repository at this point in the history 
allows guaranteed processing order

also add a skip_clear_existing option for data that is split across
multiple sheets
  • Loading branch information
@struan
struan committed Sep 4, 2023
1 parent 59f89af commit edbaeea
Showing 1 changed file with 74 additions and 38 deletions.
112 changes: 74 additions & 38 deletions crowdsourcer/management/commands/import_national_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ class Command(BaseCommand):

question_file = settings.BASE_DIR / "data" / "national_data.xlsx"

sheets = {
"Copy of Planning Q10B (pivot ta": {
sheets = [
{
"sheet": "Copy of Planning Q10B (pivot ta",
"section": "Planning & Land Use",
"number": 10,
"number_part": "b",
Expand All @@ -47,13 +48,15 @@ class Command(BaseCommand):
],
},
# XXX - negative points
"Planning Q11 - Fossil Fuel infr": {
{
"sheet": "Planning Q11 - Fossil Fuel infr",
"section": "Planning & Land Use",
"number": 11,
"default_if_missing": 0,
"missing_filter": {"type__in": ["CTY", "UTA"]},
},
"Recycling": {
{
"sheet": "Recycling",
"section": "Waste Reduction & Food",
"number": 8,
"header_row": 6,
Expand All @@ -69,7 +72,8 @@ class Command(BaseCommand):
]
# "options": [{"desc": "Criteria not met", "score": 0},]
},
"Residual Waste": {
{
"sheet": "Residual Waste",
"section": "Waste Reduction & Food",
"number": 9,
"header_row": 6,
Expand All @@ -83,7 +87,8 @@ class Command(BaseCommand):
{"desc": "Less than 300kg residual waste per household", "score": 2},
],
},
"Transport Q4 - 20mph": {
{
"sheet": "Transport Q4 - 20mph",
"section": "Transport",
"number": 4,
"council_col": "Council name",
Expand All @@ -92,7 +97,8 @@ class Command(BaseCommand):
"missing_filter": {"type__in": ["CTY", "UTA"], "country": "england"},
"type": "yes_no",
},
"Transport Q6 - Active Travel England scores": {
{
"sheet": "Transport Q6 - Active Travel England scores",
"section": "Transport",
"number": 6,
"header_row": 1,
Expand All @@ -107,7 +113,8 @@ class Command(BaseCommand):
{"desc": "Active travel capability rating 4/4", "score": 4},
],
},
"Transport 8B - Bus Ridership": {
{
"sheet": "Transport 8B - Bus Ridership",
"section": "Transport",
"number": 8,
"number_part": "b",
Expand All @@ -121,7 +128,8 @@ class Command(BaseCommand):
{"desc": "150 journeys per head of population", "score": 2},
],
},
"Transport 10 - EV chargers": {
{
"sheet": "Transport 10 - EV chargers",
"section": "Transport",
"number": 10,
"header_row": 1,
Expand All @@ -141,18 +149,21 @@ class Command(BaseCommand):
],
},
# XXX - negative points
"Transport 12a - Air Quality NO2": {
{
"sheet": "Transport 12a - Air Quality NO2",
"section": "Transport",
"number": 12,
"number_part": "a",
},
# XXX - negative points
"Transport 12b - Air Quality PM2.5": {
{
"sheet": "Transport 12b - Air Quality PM2.5",
"section": "Transport",
"number": 12,
"number_part": "b",
},
"Biodiversity Q2 - Pesticides": {
{
"sheet": "Biodiversity Q2 - Pesticides",
"section": "Biodiversity",
"number": 2,
"header_row": 1,
Expand All @@ -161,15 +172,17 @@ class Command(BaseCommand):
"default_if_missing": 0,
"type": "yes_no",
},
"Biodiversity Q4 - Wildlife Sites ": {
{
"sheet": "Biodiversity Q4 - Wildlife Sites ",
"section": "Biodiversity",
"number": 4,
"header_row": 2,
"council_col": "Council",
"score_col": "Point awarded",
"type": "yes_no",
},
"Sheet20": {
{
"sheet": "Sheet20",
"section": "Biodiversity",
"number": 7,
"council_col": "local authority council code",
Expand All @@ -182,7 +195,8 @@ class Command(BaseCommand):
{"desc": "4 or more Green Flag parks", "score": 2},
],
},
"Gov&Fin Q11a": {
{
"sheet": "Gov&Fin Q11a",
"section": "Governance & Finance",
"number": 11,
"number_part": "a",
Expand All @@ -195,7 +209,8 @@ class Command(BaseCommand):
{"desc": "Divestment of pensions's investments", "score": 2},
],
},
"Gov&Fin Q11b": {
{
"sheet": "Gov&Fin Q11b",
"section": "Governance & Finance",
"number": 11,
"number_part": "b",
Expand All @@ -209,7 +224,8 @@ class Command(BaseCommand):
{"desc": "All Fossil fuels divestment", "score": 2},
],
},
"Gov&Fin Q4": {
{
"sheet": "Gov&Fin Q4",
"section": "Governance & Finance",
"number": 4,
"header_row": 6,
Expand All @@ -224,7 +240,8 @@ class Command(BaseCommand):
{"desc": "10% or more emission reduction", "score": 3},
],
},
"Gov&Fin Q5 CA": {
{
"sheet": "Gov&Fin Q5 CA",
"section": "Governance & Finance (CA)",
"number": 5,
"header_row": 6,
Expand All @@ -238,7 +255,8 @@ class Command(BaseCommand):
{"desc": "10% or more emission reduction", "score": 3},
],
},
"Gov&Fin Q8": {
{
"sheet": "Gov&Fin Q8",
"section": "Governance & Finance",
"number": 8,
"gss_col": "local-authority-code",
Expand All @@ -251,7 +269,8 @@ class Command(BaseCommand):
{"desc": "more than 2% of staff working on climate", "score": 3},
],
},
"Gov&Fin Q8 CA": {
{
"sheet": "Gov&Fin Q8 CA",
"section": "Governance & Finance (CA)",
"number": 9,
"header_row": 1,
Expand All @@ -265,7 +284,8 @@ class Command(BaseCommand):
{"desc": "more than 2% of staff working on climate", "score": 3},
],
},
"EPC - England & Wales": {
{
"sheet": "EPC - England & Wales",
"section": "Buildings & Heating",
"number": 7,
"header_row": 5,
Expand All @@ -279,35 +299,40 @@ class Command(BaseCommand):
{"desc": "90% rated EPC C or above", "score": 3},
],
},
"EPC - Scotland": {
{
"sheet": "EPC - Scotland",
"section": "Buildings & Heating",
"number": 7,
"header_row": 5,
"council_col": "Local authority",
"score_col": "Tiered mark",
"type": "select_one",
"skip_clear_existing": True,
"options": [
{"desc": "Criteria not met", "score": 0},
{"desc": "50% rated EPC C or above", "score": 1},
{"desc": "60% rated EPC C or above", "score": 2},
{"desc": "90% rated EPC C or above", "score": 3},
],
},
"EPC - NI": {
{
"sheet": "EPC - NI",
"section": "Buildings & Heating",
"number": 7,
"header_row": 2,
"gss_col": "Northern Irish Council GSS code",
"score_col": "Tiered mark",
"type": "select_one",
"skip_clear_existing": True,
"options": [
{"desc": "Criteria not met", "score": 0},
{"desc": "50% rated EPC C or above", "score": 1},
{"desc": "60% rated EPC C or above", "score": 2},
{"desc": "90% rated EPC C or above", "score": 3},
],
},
"Collab & Engagement Q11": {
{
"sheet": "Collab & Engagement Q11",
"section": "Collaboration & Engagement",
"number": 11,
"header_row": 1,
Expand All @@ -323,10 +348,11 @@ class Command(BaseCommand):
},
],
},
}
]

ca_sheets = {
"Gov&Fin Q11a": {
ca_sheets = [
{
"sheet": "Gov&Fin Q11a",
"section": "Governance & Finance (CA)",
"number": 12,
"number_part": "a",
Expand All @@ -339,7 +365,8 @@ class Command(BaseCommand):
{"desc": "Divestment of pensions's investments", "score": 2},
],
},
"Gov&Fin Q11b": {
{
"sheet": "Gov&Fin Q11b",
"section": "Governance & Finance (CA)",
"number": 12,
"number_part": "b",
Expand All @@ -353,7 +380,8 @@ class Command(BaseCommand):
{"desc": "All Fossil fuels divestment", "score": 2},
],
},
"Transport Q6 - Active Travel England scores": {
{
"sheet": "Transport Q6 - Active Travel England scores",
"section": "Transport (CA)",
"number": 7,
"header_row": 1,
Expand All @@ -368,7 +396,8 @@ class Command(BaseCommand):
{"desc": "Active travel capability rating 4/4", "score": 4},
],
},
"Transport 8B - Bus Ridership": {
{
"sheet": "Transport 8B - Bus Ridership",
"section": "Transport (CA)",
"number": 4,
"number_part": "b",
Expand All @@ -382,7 +411,8 @@ class Command(BaseCommand):
{"desc": "150 journeys per head of population", "score": 2},
],
},
"Transport 10 - EV chargers": {
{
"sheet": "Transport 10 - EV chargers",
"section": "Transport (CA)",
"number": 8,
"header_row": 1,
Expand All @@ -391,14 +421,17 @@ class Command(BaseCommand):
"type": "select_one",
"options": [
{"desc": "Criteria not met", "score": 0},
{"desc": "60+ chargers per 100,000 people (but less than 434 per 100k)", "score": 1},
{
"desc": "60+ chargers per 100,000 people (but less than 434 per 100k)",
"score": 1,
},
{
"desc": "434+ chargers per 100,000 people",
"score": 2,
},
],
},
}
]

def add_arguments(self, parser):
parser.add_argument(
Expand Down Expand Up @@ -439,8 +472,9 @@ def get_df(self, sheet, details):

return df

def clear_existing_answers(self, q):
Response.objects.filter(question=q, response_type=self.rt).delete()
def clear_existing_answers(self, q, details):
if details.get("skip_clear_existing", None) is None:
Response.objects.filter(question=q, response_type=self.rt).delete()

def popuplate_council_lookup(self):
df = self.get_df("council names", {})
Expand Down Expand Up @@ -630,7 +664,7 @@ def handle_sheet(self, sheet, details, user):
self.print_info("")
df = self.get_df(sheet, details)
q = self.get_question(details)
self.clear_existing_answers(q)
self.clear_existing_answers(q, details)
self.add_options(q, details)
self.import_answers(user, self.rt, df, q, details)

Expand All @@ -646,13 +680,15 @@ def handle(self, quiet: bool = False, only_sheet: str = None, *args, **kwargs):
self.popuplate_council_lookup()
user, _ = User.objects.get_or_create(username="National_Importer")
self.rt = ResponseType.objects.get(type="Audit")
for sheet, details in self.sheets.items():
for details in self.sheets:
sheet = details["sheet"]
if only_sheet is not None and sheet != only_sheet:
continue

self.handle_sheet(sheet, details, user)

for sheet, details in self.ca_sheets.items():
for details in self.ca_sheets:
sheet = details["sheet"]
if only_sheet is not None and sheet != only_sheet:
continue

Expand Down

0 comments on commit edbaeea

Please sign in to comment.