Skip to content

Commit

Permalink
add no marks reasons as options if available
Browse files Browse the repository at this point in the history
  • Loading branch information
@struan
struan committed Jun 20, 2024
1 parent f9364a8 commit c6e1d4b
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 47 deletions.
95 changes: 72 additions & 23 deletions crowdsourcer/management/commands/ceuk_import_questions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ class Command(BaseCommand):
"question_no",
"topic",
"question",
"no_mark_options",
"criteria",
"clarifications",
"how_marked",
Expand Down Expand Up @@ -59,6 +60,18 @@ def add_arguments(self, parser):
"--column_list", action="store", help="file with list of column names"
)

parser.add_argument(
"--delete_old_no_mark_options",
action="store_true",
help="remove old options with zero marks",
)

parser.add_argument(
"--delete_options",
action="store_true",
help="remove all options and recreate",
)

def get_column_names(self, **kwargs):
column_list = kwargs.get("column_list", None)
column_list = settings.BASE_DIR / "data" / column_list
Expand Down Expand Up @@ -140,7 +153,6 @@ def handle(self, quiet: bool = False, *args, **kwargs):
df = df.dropna(axis="index", how="all")
drop_cols = [
"Climate Justice/Adaptation Tag",
"Drop down box options for no mark awarded (internal)",
"Is this question or criteria changing?",
"Change proposed",
"New Criteria",
Expand All @@ -154,10 +166,14 @@ def handle(self, quiet: bool = False, *args, **kwargs):
for col in drop_cols:
if col in df.columns:
df = df.drop(col, axis=1)
df = df.iloc[:, :14]
# df = df.iloc[:, :15]

columns = list(self.column_names)
options = len(df.columns) - len(self.column_names) + 1

if "Drop down box options for no mark awarded (internal)" not in df.columns:
columns.remove("no_mark_options")

options = len(df.columns) - len(columns) + 1
for i in range(1, options):
columns.append(f"option_{i}")

Expand Down Expand Up @@ -254,37 +270,70 @@ def handle(self, quiet: bool = False, *args, **kwargs):
if kwargs["text_only"] or kwargs["weighting_only"]:
continue

if kwargs["delete_old_no_mark_options"]:
Option.objects.filter(question=q, score=0).delete()

if kwargs["delete_options"]:
Option.objects.filter(question=q).delete()

if row.get("no_mark_options") is not None:
no_mark_options = row["no_mark_options"].splitlines()
else:
no_mark_options = []

no_mark_options.extend(
[
"No Penalty Mark",
"No evidence found",
"Evidence does not meet criteria",
"No response from FOI",
]
)

if q.question_type in ["select_one", "tiered", "multiple_choice"]:
o, c = Option.objects.update_or_create(
question=q,
description="None",
defaults={"score": 0, "ordering": 100},
)
if not no_mark_options:
o, c = Option.objects.update_or_create(
question=q,
description="None",
defaults={"score": 0, "ordering": 100},
)
no_mark_seen = 0
for i in range(1, options):
desc = row[f"option_{i}"]
desc = str(row[f"option_{i}"]).strip()
if pd.isna(desc) or desc == "":
continue
score = 1
ordering = i
if q.question_type == "tiered":
score = i
if not pd.isna(desc):
o, c = Option.objects.update_or_create(
question=q,
description=desc,
defaults={"score": score, "ordering": ordering},
)
elif q.question_type == "yes_no":
for desc in ["Yes", "No"]:
ordering = 1
score = 1
if desc == "No":
score = i - no_mark_seen
if desc in no_mark_options:
no_mark_seen = no_mark_seen + 1
score = 0
ordering = 2
o, c = Option.objects.update_or_create(
question=q,
description=desc,
defaults={"score": score, "ordering": ordering},
)
elif q.question_type == "yes_no":
o, c = Option.objects.update_or_create(
question=q,
description="Yes",
defaults={"score": 1, "ordering": 1},
)
if no_mark_options:
for option in no_mark_options:
o, c = Option.objects.update_or_create(
question=q,
description=option,
defaults={"score": 0, "ordering": 100},
)
else:
o, c = Option.objects.update_or_create(
question=q,
description="No",
defaults={"score": 0, "ordering": 2},
)

for col, group in q_groups.items():
if row[col] == "Yes":
if row[col] in ["Yes", "Y", "All"]:
q.questiongroup.add(group)
92 changes: 68 additions & 24 deletions crowdsourcer/management/commands/import_combined_authority_questions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ class Command(BaseCommand):
"question_no",
"topic",
"question",
"no_mark_options",
"criteria",
"clarifications",
"how_marked",
Expand Down Expand Up @@ -54,6 +55,18 @@ def add_arguments(self, parser):
"--column_list", action="store", help="file with list of column names"
)

parser.add_argument(
"--delete_old_no_mark_options",
action="store_true",
help="remove old options with zero marks",
)

parser.add_argument(
"--delete_options",
action="store_true",
help="remove all options and recreate",
)

def get_column_names(self, **kwargs):
column_list = kwargs.get("column_list", None)
column_list = settings.BASE_DIR / "data" / column_list
Expand Down Expand Up @@ -138,7 +151,6 @@ def handle(self, quiet: bool = False, *args, **kwargs):
df = df.dropna(axis="index", how="all")
drop_cols = [
"Climate Justice/Adaptation Tag",
"Drop down box options for no mark awarded (internal)",
"Is this question or criteria changing?",
"Change proposed",
"New Criteria",
Expand All @@ -158,7 +170,11 @@ def handle(self, quiet: bool = False, *args, **kwargs):
df = df.drop(col, axis=1)

columns = list(self.column_names)
options = len(df.columns) - len(self.column_names) + 1

if "Drop down box options for no mark awarded (internal)" not in df.columns:
columns.remove("no_mark_options")

options = len(df.columns) - len(columns) + 1
for i in range(1, options):
columns.append(f"option_{i}")

Expand Down Expand Up @@ -253,40 +269,68 @@ def handle(self, quiet: bool = False, *args, **kwargs):
if kwargs["text_only"]:
continue

if q.question_type in ["select_one", "tiered", "multiple_choice"]:
is_no = False
for i in range(1, options):
desc = row[f"option_{i}"]
score = 1
ordering = i
if q.question_type == "tiered":
score = i
if not pd.isna(desc):
if desc == "No":
is_no = True
o, c = Option.objects.update_or_create(
question=q,
description=desc,
defaults={"score": score, "ordering": ordering},
)
if kwargs["delete_old_no_mark_options"]:
Option.objects.filter(question=q, score=0).delete()

if not is_no and q.question_type == "tiered":
if kwargs["delete_options"]:
Option.objects.filter(question=q).delete()

if row.get("no_mark_options") is not None:
no_mark_options = row["no_mark_options"].splitlines()
else:
no_mark_options = []

no_mark_options.extend(
[
"No Penalty Mark",
"No evidence found",
"Evidence does not meet criteria",
"No response from FOI",
]
)

if q.question_type in ["select_one", "tiered", "multiple_choice"]:
if not no_mark_options:
o, c = Option.objects.update_or_create(
question=q,
description="None",
defaults={"score": 0, "ordering": 100},
)
elif q.question_type == "yes_no":
for desc in ["Yes", "No"]:
ordering = 1
no_mark_seen = 0
for i in range(1, options):
desc = str(row[f"option_{i}"]).strip()
if pd.isna(desc) or desc == "":
continue
score = 1
if desc == "No":
ordering = i
if q.question_type == "tiered":
score = i - no_mark_seen
if desc in no_mark_options:
no_mark_seen = no_mark_seen + 1
score = 0
ordering = 2
o, c = Option.objects.update_or_create(
question=q,
description=desc,
defaults={"score": score, "ordering": ordering},
)
elif q.question_type == "yes_no":
o, c = Option.objects.update_or_create(
question=q,
description="Yes",
defaults={"score": 1, "ordering": 1},
)
if no_mark_options:
for option in no_mark_options:
o, c = Option.objects.update_or_create(
question=q,
description=option,
defaults={"score": 0, "ordering": 100},
)
else:
o, c = Option.objects.update_or_create(
question=q,
description="No",
defaults={"score": 0, "ordering": 2},
)

q.questiongroup.add(group)

0 comments on commit c6e1d4b

Please sign in to comment.