fix: Serialize list problem_check results as JSON (#366)

* fix: Serialize list problem_check results as JSON

Prior to this fix, TinCan would use the repr of the list, which
made some responses effectively un-parseable downstream.

CHANGELOG.rst

@@ -13,31 +13,47 @@ Change Log
 
 Unreleased
 ~~~~~~~~~~
+
+[7.0.2]
+~~~~~~~
+
+* Ensure lists of answers in problem_check are properly serialized to JSON so they
+  can be parsed downstream
+
+  **Note: Old events cannot be updated, the log must be replayed (if possible).**
+
 [7.0.1]
+~~~~~~~
 
 * Do not send events for unknown courses
 
 [7.0.0]
+~~~~~~~
 
 * Multi-question problem_check tracking log statements will now be split into one xAPI statement for each question
 
 [6.2.0]
+~~~~~~~
 
 * Add support for completion events
 
 [6.1.0]
+~~~~~~~
 
 * Add support for exam attempts events
 
 [6.0.0]
+~~~~~~~
 
 * Do not send events for unknown users
 
 [5.5.6]
+~~~~~~~
 
 * upgrading deprecated `djfernet` with `django-fernet-fields-v2`
 
 [5.4.0]
+~~~~~~~
 
 * Add support for the ``edx.course.enrollment.mode_changed`` event
 

event_routing_backends/__init__.py

@@ -2,4 +2,4 @@
 Various backends for receiving edX LMS events..
 """
 
-__version__ = '7.0.1'
+__version__ = '7.0.2'

event_routing_backends/processors/tests/fixtures/current/problem_check(server).list_answers.json

@@ -0,0 +1,187 @@
+{
+  "name": "problem_check",
+  "context": {
+    "course_id": "course-v1:edX+DemoX+Demo_Course",
+    "course_user_tags": {},
+    "user_id": 6,
+    "path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4/handler/xmodule_handler/problem_check",
+    "org_id": "edX",
+    "enterprise_uuid": "",
+    "module": {
+      "display_name": "Multiple Choice Questions",
+      "usage_key": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4"
+    },
+    "asides": {}
+  },
+  "username": "tacotuesday",
+  "session": "9885dd163f65eed1fe88759c186b4ac3",
+  "ip": "192.168.1.1",
+  "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/119.0",
+  "host": "local.overhang.io",
+  "referer": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@54bb9b142c6c4c22afc62bcb628f0e68?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view&format=Homework",
+  "accept_language": "en-US,en;q=0.5",
+  "event": {
+    "state": {
+      "seed": 1,
+      "student_answers": {
+        "a0effb954cca4759994f1ac9e9434bf4_4_1": [
+          "choice_0",
+          "choice_1",
+          "choice_2"
+        ],
+        "a0effb954cca4759994f1ac9e9434bf4_2_1": "blue",
+        "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2"
+      },
+      "has_saved_answers": false,
+      "correct_map": {
+        "a0effb954cca4759994f1ac9e9434bf4_2_1": {
+          "correctness": "correct",
+          "npoints": null,
+          "msg": "",
+          "hint": "",
+          "hintmode": null,
+          "queuestate": null,
+          "answervariable": null
+        },
+        "a0effb954cca4759994f1ac9e9434bf4_3_1": {
+          "correctness": "correct",
+          "npoints": null,
+          "msg": "",
+          "hint": "",
+          "hintmode": null,
+          "queuestate": null,
+          "answervariable": null
+        },
+        "a0effb954cca4759994f1ac9e9434bf4_4_1": {
+          "correctness": "incorrect",
+          "npoints": null,
+          "msg": "",
+          "hint": "",
+          "hintmode": null,
+          "queuestate": null,
+          "answervariable": null
+        }
+      },
+      "input_state": {
+        "a0effb954cca4759994f1ac9e9434bf4_2_1": {},
+        "a0effb954cca4759994f1ac9e9434bf4_3_1": {},
+        "a0effb954cca4759994f1ac9e9434bf4_4_1": {},
+        "a0effb954cca4759994f1ac9e9434bf4_5_1": {}
+      },
+      "done": true
+    },
+    "problem_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
+    "answers": {
+      "a0effb954cca4759994f1ac9e9434bf4_4_1": [
+        "choice_0",
+        "choice_1",
+        "choice_2"
+      ],
+      "a0effb954cca4759994f1ac9e9434bf4_5_1": [
+        "choice_0",
+        "choice_1",
+        "choice_2",
+        "choice_3",
+        "choice_4",
+        "choice_5"
+      ],
+      "a0effb954cca4759994f1ac9e9434bf4_2_1": "blue",
+      "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2"
+    },
+    "grade": 2,
+    "max_grade": 4,
+    "correct_map": {
+      "a0effb954cca4759994f1ac9e9434bf4_2_1": {
+        "correctness": "correct",
+        "npoints": null,
+        "msg": "",
+        "hint": "",
+        "hintmode": null,
+        "queuestate": null,
+        "answervariable": null
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_3_1": {
+        "correctness": "correct",
+        "npoints": null,
+        "msg": "",
+        "hint": "",
+        "hintmode": null,
+        "queuestate": null,
+        "answervariable": null
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_4_1": {
+        "correctness": "incorrect",
+        "npoints": null,
+        "msg": "",
+        "hint": "",
+        "hintmode": null,
+        "queuestate": null,
+        "answervariable": null
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_5_1": {
+        "correctness": "incorrect",
+        "npoints": null,
+        "msg": "",
+        "hint": "",
+        "hintmode": null,
+        "queuestate": null,
+        "answervariable": null
+      }
+    },
+    "success": "incorrect",
+    "attempts": 9,
+    "submission": {
+      "a0effb954cca4759994f1ac9e9434bf4_4_1": {
+        "question": "",
+        "answer": [
+          "a piano",
+          "a tree",
+          "a guitar"
+        ],
+        "response_type": "choiceresponse",
+        "input_type": "checkboxgroup",
+        "correct": false,
+        "variant": "",
+        "group_label": ""
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_5_1": {
+        "question": "",
+        "answer": [
+          "Un emprunt \u00e0 l'anglais ou anglicisme",
+          "Un type de demande",
+          "Quelque chose de rapide, d'instantan\u00e9",
+          "Une question structur\u00e9e et pr\u00e9cise",
+          "Un type de poisson",
+          "I\"M SWP' \"SDF\""
+        ],
+        "response_type": "choiceresponse",
+        "input_type": "checkboxgroup",
+        "correct": false,
+        "variant": "",
+        "group_label": ""
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_2_1": {
+        "question": "",
+        "answer": "blue",
+        "response_type": "optionresponse",
+        "input_type": "optioninput",
+        "correct": true,
+        "variant": "",
+        "group_label": ""
+      },
+      "a0effb954cca4759994f1ac9e9434bf4_3_1": {
+        "question": "",
+        "answer": "a chair",
+        "response_type": "multiplechoiceresponse",
+        "input_type": "choicegroup",
+        "correct": true,
+        "variant": "",
+        "group_label": ""
+      }
+    }
+  },
+  "time": "2023-11-03T16:31:50.023274+00:00",
+  "event_type": "problem_check",
+  "event_source": "server",
+  "page": "x_module"
+}

event_routing_backends/processors/tests/transformers_test_mixin.py

@@ -119,6 +119,7 @@ def test_event_transformer(self, event_filename, mocked_uuid4):
                 try:
                     self.compare_events(actual_transformed_event, expected_event)
                 except Exception as e:  # pragma: no cover
+                    print("Comparison failed, writing output to test_output for debugging")
                     with open(f"test_output/generated.{event_filename}.json", "w") as actual_transformed_event_file:
                         try:
                             actual_transformed_event_file.write(actual_transformed_event.to_json())

event_routing_backends/processors/xapi/event_transformers/problem_interaction_events.py

@@ -1,6 +1,8 @@
 """
 Transformers for problem interaction events.
 """
+import json
+
 from tincan import Activity, ActivityDefinition, Extensions, LanguageMap, Result
 
 from event_routing_backends.helpers import get_problem_block_id
@@ -315,6 +317,15 @@ def get_result(self):
             response = submission["answer"]
             correct = submission.get("correct")
         else:
+            # The submission key didn't exist until March 2014, prior to that
+            # there was usually a version of the answer in the "answers" key,
+            # but it is very flaky (sometimes containing xml, and without the
+            # appended variant identifier that we get for free in the
+            # submission. We don't attempt to work around those issues here due
+            # to how few and how old those events are and how complicated the
+            # parsing is. Should we ever find it necessary to make a better
+            # parser for them, Insights had a good effort here:
+            # https://github.com/openedx/edx-analytics-pipeline/blob/8d96f93/edx/analytics/tasks/insights/answer_dist.py#L260C36-L260C36
             response = event_data.get('answers', None)
             correct = self.get_data('success') == 'correct'
 
@@ -328,7 +339,17 @@ def get_result(self):
             else:
                 scaled = 0
 
-        return Result(
+        # Some problems can provide a list of responses answers, but
+        # the Result type wants a string for "response". So we dump those
+        # to JSON here to provide a parsable version of the response instead
+        # of getting the __repr__ of the list, which is what Result will
+        # generate by default.
+        if isinstance(response, list):
+            cls = JSONEncodedResult
+        else:
+            cls = Result
+
+        return cls(
             success=correct,
             score={
                 'min': 0,
@@ -455,3 +476,30 @@ def get_result(self):
         submission = self._get_submission() or {}
         result.response = submission.get('answer')
         return result
+
+
+class JSONEncodedResult(Result):
+    """
+    This is a workaround for a TinCan issue where it will coerce a value passed
+    in for a `response` to str. This breaks our ability to serialize list
+    responses into JSON, so we override it here.
+    """
+    @property
+    def response(self):
+        """Response for Result
+
+        :setter: Tries to JSON dump the list value.
+        :setter type: list
+        :rtype: str
+        """
+        return self._response
+
+    @response.setter
+    def response(self, value):
+        """
+        Ensures the list is serialized as JSON.
+        """
+        if not isinstance(value, list):
+            raise ValueError(f"JSONEncodedResult only accepts lists, {type(value)} given.")
+
+        self._response = json.dumps(value)

event_routing_backends/processors/xapi/tests/fixtures/expected/problem_check(server).json

@@ -39,7 +39,7 @@
         "objectType": "Activity"
     },
     "result": {
-        "response": "['a correct answer', 'an incorrect answer']",
+        "response": "[\"a correct answer\", \"an incorrect answer\"]",
         "score": {
             "max": 1,
             "min": 0,