Merge pull request #370 from RTIInternational/irr_limit_coders

Handle IRR assignments

backend/django/core/utils/utils_annotate.py

@@ -96,15 +96,19 @@ def get_assignments(profile, project, num_assignments):
         for i in range(num_assignments):
             # if there is IRR, with some probability get an IRR item
             rand_choice = randrange(0, 101)
+
+            assigned_datum = None
             if project.percentage_irr > 0 and rand_choice <= project.percentage_irr:
                 assigned_datum = assign_datum(profile, project, type="irr")
                 if assigned_datum is None:
                     # no irr data found
                     assigned_datum = assign_datum(profile, project)
-            else:
+
+            # if we didn't assign an IRR data for some reason, check non-irr
+            if assigned_datum is None and project.percentage_irr < 100:
                 # get normal data
                 assigned_datum = assign_datum(profile, project)
-                if assigned_datum is None:
+                if assigned_datum is None and project.percentage_irr > 0:
                     # no non-irr data found so checking for irr
                     assigned_datum = assign_datum(profile, project, type="irr")
             if assigned_datum is None:

backend/django/core/utils/utils_queue.py

@@ -306,12 +306,23 @@ def pop_first_nonempty_queue(project, profile=None, type="normal"):
                 .exclude(data__in=skipped_data)
             )
 
+            final_data = []
+            for d in assigned_unlabeled:
+                # for each data item, need to check if the count of other assignments+labels+logs exceeds num_irr, don't return
+                if (
+                    DataLabel.objects.filter(data=d.data).count()
+                    + IRRLog.objects.filter(data=d.data).count()
+                    + AssignedData.objects.filter(data=d.data).count()
+                    < project.num_users_irr
+                ):
+                    final_data.append(d)
+
             # if there are no elements, return none
-            if len(assigned_unlabeled) == 0:
+            if len(final_data) == 0:
                 return (None, None)
             else:
                 # else, get the first element off the group and return it
-                datum = Data.objects.get(pk=assigned_unlabeled[0].data.pk)
+                datum = Data.objects.get(pk=final_data[0].data.pk)
                 return (queue, datum)
     if len(eligible_queue_ids) == 0:
         return (None, None)

backend/django/core/views/api.py

@@ -156,7 +156,13 @@ def download_irr_log(request, project_pk):
     for log in logs:
         label_name = log.label.name if log.label else ""
         writer.writerow(
-            [log.data.upload_id, log.data.text, label_name, log.profile.user, log.timestamp]
+            [
+                log.data.upload_id,
+                log.data.text,
+                label_name,
+                log.profile.user,
+                log.timestamp,
+            ]
         )
 
     return response

backend/django/test/test_irr.py

@@ -79,8 +79,9 @@ def test_annotate_irr(
     datum2 = assign_datum(test_profile2, project, "irr")
     assert datum.pk == datum2.pk
 
+    # since this is a 2-label IRR project, we shouldn't get the same item assigned to profile 3
     datum3 = assign_datum(test_profile3, project, "irr")
-    assert datum.pk == datum3.pk
+    assert datum.pk != datum3.pk
 
     # let other user label the same datum. It should now be in datatable with
     # creater=profile, be in IRRLog (twice), not be in IRRQueue
@@ -90,25 +91,19 @@ def test_annotate_irr(
     assert IRRLog.objects.filter(data=datum2).count() == 2
     assert DataQueue.objects.filter(data=datum2, queue=irr_queue).count() == 0
 
-    # let a third user label the first data something else. It should be in
-    # IRRLog but not overwrite the label from before
-    label_data(test_labels_half_irr[0], datum3, test_profile3, 3)
-    assert IRRLog.objects.filter(data=datum3).count() == 3
-    assert DataLabel.objects.filter(data=datum3).count() == 1
-    assert DataLabel.objects.get(data=datum3).profile.pk == project.creator.pk
+    # now assign a new item to the first profile. It should be the same as datum3
+    second_datum = assign_datum(test_profile, project, "irr")
 
+    # should be a new datum
+    assert datum3.pk == second_datum.pk
     # let two users disagree on a datum. It should be in the admin queue,
     # not in irr queue, not in datalabel, in irrlog twice
-    second_datum = assign_datum(test_profile, project, "irr")
-    # should be a new datum
-    assert datum.pk != second_datum.pk
-    second_datum2 = assign_datum(test_profile2, project, "irr")
     label_data(test_labels_half_irr[0], second_datum, test_profile, 3)
-    label_data(test_labels_half_irr[1], second_datum2, test_profile2, 3)
-    assert DataQueue.objects.filter(data=second_datum2, queue=admin_queue).count() == 1
-    assert DataQueue.objects.filter(data=second_datum2, queue=irr_queue).count() == 0
-    assert DataLabel.objects.filter(data=second_datum2).count() == 0
-    assert IRRLog.objects.filter(data=second_datum2).count() == 2
+    label_data(test_labels_half_irr[1], datum3, test_profile3, 3)
+    assert DataQueue.objects.filter(data=datum3, queue=admin_queue).count() == 1
+    assert DataQueue.objects.filter(data=datum3, queue=irr_queue).count() == 0
+    assert DataLabel.objects.filter(data=datum3).count() == 0
+    assert IRRLog.objects.filter(data=datum3).count() == 2
 
 
 def test_skip_irr(
@@ -158,15 +153,16 @@ def test_skip_irr(
     assert second_datum.pk != datum.pk
     assert second_datum.pk == second_datum2.pk
     second_datum3 = assign_datum(test_profile3, project, "irr")
-    assert second_datum2.pk == second_datum3.pk
+    assert second_datum2.pk != second_datum3.pk
 
     label_data(test_labels_half_irr[0], second_datum, test_profile, 3)
     label_data(test_labels_half_irr[0], second_datum2, test_profile2, 3)
     skip_data(second_datum3, test_profile3)
     assert DataQueue.objects.filter(data=second_datum3, queue=admin_queue).count() == 0
-    assert DataQueue.objects.filter(data=second_datum3, queue=irr_queue).count() == 0
-    assert IRRLog.objects.filter(data=second_datum3).count() == 3
-    assert DataLabel.objects.filter(data=second_datum3).count() == 1
+    assert DataQueue.objects.filter(data=second_datum3, queue=irr_queue).count() == 1
+    assert IRRLog.objects.filter(data=second_datum3).count() == 1
+    assert IRRLog.objects.filter(data=second_datum2).count() == 2
+    assert DataLabel.objects.filter(data=second_datum2).count() == 1
 
 
 def test_queue_refill(

frontend/src/containers/card_container.jsx

@@ -20,7 +20,7 @@ const CardContainer = (props) => {
             <DataCard 
                 data={{ ...props.cards[0], num_cards_left: props.cards.length }}
                 page={PAGES.ANNOTATE_DATA} 
-                actions={{ onSelectLabel: props.annotateCard, onAdjudicate: props.passCard, onSkip: props.unassignCard }} /> : null
+                actions={{ onSelectLabel: props.annotateCard, onAdjudicate: props.passCard, onSkip: props.unassignCard }} /> : <p>Available data for coding will populate here. If nothing appears after a minute or so then either your project has no data left to code or all data is currently being worked on by other coders.</p>
     );
 };