add a whole bunch of category fixes

backend/django/core/forms.py

@@ -155,9 +155,11 @@ def __init__(self, *args, **kwargs):
         else:
             start_val = "None"
         data_metadata = project.metadatafields.values_list("field_name", flat=True)
+        data_metadata = [m.lower().capitalize() for m in data_metadata]
         label_metadata = project.labelmetadatafields.values_list(
             "field_name", flat=True
         )
+        label_metadata = [lm.lower().capitalize() for lm in label_metadata]
         options = list(set(data_metadata) & set(label_metadata)) + ["None"]
 
         percentage_irr = kwargs.pop("percentage_irr")

backend/django/core/models.py

@@ -108,6 +108,12 @@ def coder_count(self):
     def labeled_data_count(self):
         return self.data_set.all().filter(datalabel__isnull=False).count()
 
+    def first_100_labels(self):
+        if self.labels.count() > 100:
+            return self.labels.all()[:100]
+        else:
+            return self.labels.all()
+
     def unverified_labeled_data_count(self):
         return (
             self.data_set.all()
@@ -430,6 +436,8 @@ class Meta:
     value = models.TextField(null=True, blank=True)
 
     def __str__(self):
+        if str(self.value) == "nan":
+            return ""
         return f"{str(self.label_metadata_field)}: {self.value}"
 
 

backend/django/core/serializers.py

@@ -61,11 +61,14 @@ class Meta:
 
     def to_representation(self, obj):
         base_representation = super().to_representation(obj)
-        if len(base_representation["labelmetadata"]) > 0:
+        if str(base_representation["description"]) == "nan":
+            base_representation["description"] = ""
+
+        metadata = [m for m in base_representation["labelmetadata"] if str(m) != ""]
+
+        if len(metadata) > 0:
             base_representation["description"] = (
-                base_representation["description"]
-                + " | "
-                + " | ".join(base_representation["labelmetadata"])
+                base_representation["description"] + " | " + " | ".join(metadata)
             )
         return base_representation
 

backend/django/core/templates/projects/create/create_wizard_advanced.html

@@ -11,12 +11,18 @@
 {% endblock %}
 
 {% block content %}
+<div class="overlay">
+  <div class="overlay_contents">
+      <div id="second_progress"></div>
+      <span id="loading_text"></span>
+  </div>
+</div>
 <div class="error-messages">{{ wizard.non_form_errors }}</div>
 <div class="row">
   <div class="col-md-8 col-md-offset-2">
     <div class="card">
       <div class="cardface">
-        <form action="." method="post" enctype="multipart/form-data">
+        <form action="." method="post" enctype="multipart/form-data" onsubmit="addASpinner()">
           {% csrf_token %}
           {{ wizard.management_form }}
 
@@ -262,5 +268,14 @@ <h5 class="panel-title">
     irr_or_not_box_disabled.hide();
   }
 });
+
+function addASpinner() {
+        $('.overlay').show();
+        $('#loading_text').html("Please wait. Creating project...");
+        $("#second_progress").addClass("loader");
+        setTimeout(function() {
+            $('#loading_text').html("Still processing data...");
+        }, 30000)
+    }
 </script>
 {% endblock %}

backend/django/core/templates/projects/create/create_wizard_labels.html

@@ -22,7 +22,7 @@
   <div class="col-md-8 col-md-offset-2">
     <div class="card">
       <div class="cardface">
-        <form action="." method="post" enctype="multipart/form-data">
+        <form action="." method="post" enctype="multipart/form-data" onsubmit="addASpinner()">
           {% csrf_token %}
           {{ wizard.management_form }}
 
@@ -61,5 +61,15 @@ <h3>Instructions</h3>
 {% endblock %}
 
 {% block scripts_body %}
-<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-csv/0.71/jquery.csv-0.71.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-csv/0.71/jquery.csv-0.71.min.js">
+  function addASpinner() {
+        $('.overlay').show();
+        $('#loading_text').html("Please wait. Updating labels...");
+        $("#second_progress").addClass("loader");
+        //after 30 seconds, update the message
+        setTimeout(function() {
+            $('#loading_text').html("Still processing data...");
+        }, 30000)
+    }
+</script>
 {% endblock %}

backend/django/core/templates/projects/detail.html

@@ -168,7 +168,7 @@ <h5 class="panel-title">
                 <li class="list-group-item">
                   <dt>Metadata Fields Displayed with Data</dt>
                   {% for meta in project.metadatafields.all %}
-                      {% if meta.field_name == project.category.field_name %}
+                      {% if meta.field_name.lower == project.category.field_name.lower %}
                         <dd>- [CATEGORY] {{ meta.field_name }} </dd>
                       {% else %}
                         <dd>- {{ meta.field_name }}</dd>
@@ -180,7 +180,7 @@ <h5 class="panel-title">
                 <li class="list-group-item">
                   <dt>Metadata Fields Displayed with Labels</dt>
                   {% for meta in project.labelmetadatafields.all %}
-                    {% if meta.field_name == project.category.field_name %}
+                    {% if meta.field_name.lower == project.category.field_name.lower %}
                       <dd>- [CATEGORY] {{ meta.field_name }} </dd>
                     {% else %}
                       <dd>- {{ meta.field_name }}</dd>
@@ -217,7 +217,7 @@ <h5 class="panel-title">
           <div id="label-panel" class="panel-collapse collapse">
             <div class="panel-body">
               <ul class="list-group-flush">
-                {% for label in project.labels.all %}
+                {% for label in project.first_100_labels %}
                   <li class="list-group-item">
                     <dt>{{ label.name }}</dt>
                     <dd>{{ label.description }} {% for meta in label.labelmetadata.all %} | {{ meta }} {% endfor %}</dd>

backend/django/core/templates/projects/update/labels.html

@@ -4,12 +4,18 @@
 {% load render_bundle from webpack_loader %}
 
 {% block content %}
+<div class="overlay">
+  <div class="overlay_contents">
+      <div id="second_progress"></div>
+      <span id="loading_text"></span>
+  </div>
+</div>
 <div class="row">
   <div class="col-md-8 col-md-offset-2">
     <div class="card">
       <div class="cardface">
         {{ form.media.css }}
-        <form action="." method="post" enctype="multipart/form-data">
+        <form action="." method="post" enctype="multipart/form-data" onsubmit="addASpinner()">
           {% csrf_token %}
           <h1>Update Label Descriptions/Metadata</h1>
           <p>This section lets you update your label descriptions or add/update metadata. Please upload a csv or excel file containing the fields 'Label', 'Description',
@@ -20,6 +26,7 @@ <h1>Update Label Descriptions/Metadata</h1>
           <div class="form-group">
             <label class="control-label" for="{{ form.data.id_for_label }}">{{ form.data.label }}</label>
             <input class="form-control" id="{{ form.data.id_for_label }}" maxlength="30" name="{{ form.data.html_name }}" type="file" placeholder="{{ form.data.label }}" />
+            <div class="error-messages">{{ form.non_field_errors }}</div>
             <div class="error-messages">{{ form.data.errors }}</div>
             <div class="error-messages">{{ form.all.errors }}</div>
           </div>
@@ -30,3 +37,16 @@ <h1>Update Label Descriptions/Metadata</h1>
   </div>
 </div>
 {% endblock %}
+
+{% block scripts_body %}
+<script type="text/javascript">
+      function addASpinner() {
+        $('.overlay').show();
+        $('#loading_text').html("Please wait. Uploading labels...");
+        $("#second_progress").addClass("loader");
+        setTimeout(function() {
+            $('#loading_text').html("STILL Processing labels...");
+        }, 30000)
+    }
+</script>
+{% endblock %}

backend/django/core/utils/util.py

@@ -847,33 +847,31 @@ def create_label_metadata(project, label_data, label_list):
             )
 
 
-def create_or_update_project_category(project, label_metadata, data_metadata):
+def create_or_update_project_category(project, new_category):
     """This function takes a field which overlaps between the label and data metadata
     and sets it to the project category to filter the suggestions by."""
-    metadata_both = list(set(data_metadata) & set(label_metadata))
-    if len(metadata_both) > 0:
-        # by default just pick the first overlap
-        if Category.objects.filter(project=project).exists():
-            Category.objects.filter(project=project).update(
-                field_name=metadata_both[0],
-                label_metadata_field=LabelMetaDataField.objects.get(
-                    field_name=metadata_both[0], project=project
-                ),
-                data_metadata_field=MetaDataField.objects.get(
-                    field_name=metadata_both[0], project=project
-                ),
-            )
-        else:
-            Category.objects.create(
-                project=project,
-                field_name=metadata_both[0],
-                label_metadata_field=LabelMetaDataField.objects.get(
-                    field_name=metadata_both[0], project=project
-                ),
-                data_metadata_field=MetaDataField.objects.get(
-                    field_name=metadata_both[0], project=project
-                ),
-            )
+    # by default just pick the first overlap
+    if Category.objects.filter(project=project).exists():
+        Category.objects.filter(project=project).update(
+            field_name=new_category,
+            label_metadata_field=LabelMetaDataField.objects.get(
+                field_name__iexact=new_category, project=project
+            ),
+            data_metadata_field=MetaDataField.objects.get(
+                field_name__iexact=new_category, project=project
+            ),
+        )
+    else:
+        Category.objects.create(
+            project=project,
+            field_name=new_category,
+            label_metadata_field=LabelMetaDataField.objects.get(
+                field_name__iexact=new_category, project=project
+            ),
+            data_metadata_field=MetaDataField.objects.get(
+                field_name__iexact=new_category, project=project
+            ),
+        )
 
 
 def update_label_descriptions_metadata(project, new_data):
@@ -934,4 +932,9 @@ def update_label_descriptions_metadata(project, new_data):
     data_metadata = MetaDataField.objects.filter(project=project).values_list(
         "field_name", flat=True
     )
-    create_or_update_project_category(project, label_metadata, data_metadata)
+    metadata_both = list(
+        set([m.lower() for m in data_metadata])
+        & set([m.lower() for m in label_metadata])
+    )
+    if len(metadata_both) > 0:
+        create_or_update_project_category(project, metadata_both[0])

backend/django/core/views/api_annotate.py

@@ -79,11 +79,17 @@ def get_queryset(self):
         filter_text = self.request.GET.get("searchString")
 
         label_category = self.request.GET.get("category")
-        if label_category:
-            category_label_list = LabelMetaData.objects.filter(
-                label_metadata_field=project.category.label_metadata_field,
-                value=label_category,
-            ).values_list("label__pk", flat=True)
+        if label_category and label_category != "all":
+            if label_category == "None":
+                category_label_list = LabelMetaData.objects.filter(
+                    label_metadata_field=project.category.label_metadata_field,
+                    value="nan",
+                ).values_list("label__pk", flat=True)
+            else:
+                category_label_list = LabelMetaData.objects.filter(
+                    label_metadata_field=project.category.label_metadata_field,
+                    value=label_category,
+                ).values_list("label__pk", flat=True)
             return Label.objects.filter(
                 project=project, pk__in=category_label_list
             ).filter(
@@ -120,12 +126,15 @@ def get_label_categories(request, project_pk, data_pk):
                 metadata_field=data_metadata_field, data=data_options
             ).value
             if data_category not in category_options:
-                data_category = ""
+                data_category = "all"
         else:
-            data_category = ""
+            data_category = "all"
+
+        category_options = [c if str(c) != "nan" else "None" for c in category_options]
         return Response(
             {
-                "label_category_options": [
+                "label_category_options": [{"value": "all", "label": "Category: all"}]
+                + [
                     {"value": key, "label": f"Category: {key}"}
                     for key in category_options
                 ],
@@ -150,9 +159,10 @@ def get_labels(request, project_pk):
     total_labels = Label.objects.filter(project=project).count()
 
     # If the number of labels is > 100, just return the first 100
-    serialized_labels = LabelSerializer(labels, many=True).data
-    if len(serialized_labels) > 100:
-        serialized_labels = serialized_labels[:100]
+    if total_labels < 100:
+        serialized_labels = LabelSerializer(labels, many=True).data
+    else:
+        serialized_labels = []
 
     return Response({"labels": serialized_labels, "total_labels": total_labels})
 
@@ -383,6 +393,13 @@ def annotate_data(request, data_pk):
         )
         return Response(response)
 
+    if not data.irr_ind and DataLabel.objects.filter(data=data).exists():
+        response["error"] = (
+            "ERROR: this card has been labeled by someone else in another tab."
+            "Please refresh the page to get new items to annotate."
+        )
+        return Response(response)
+
     num_history = IRRLog.objects.filter(data=data).count()
 
     if RecycleBin.objects.filter(data=data).count() > 0:
@@ -720,6 +737,7 @@ def data_unlabeled_table(request, project_pk):
     ]
     return Response({"data": data})
 
+
 @api_view(["POST"])
 @permission_classes([AllowAny])
 def embeddings_calculations(request):

backend/django/core/views/frontend.py

@@ -34,7 +34,6 @@
     Project,
     TrainingSet,
     Category,
-    LabelMetaDataField,
 )
 from core.templatetags import project_extras
 from core.utils.util import (
@@ -402,7 +401,12 @@ def done(self, form_list, form_dict, **kwargs):
             label_metadata = [
                 c for c in label_data if c not in ["Label", "Description"]
             ]
-            create_or_update_project_category(proj_obj, label_metadata, metadata_fields)
+            metadata_both = list(
+                set([m.lower() for m in metadata_fields])
+                & set([m.lower() for m in label_metadata])
+            )
+            if len(metadata_both) > 0:
+                create_or_update_project_category(proj_obj, metadata_both[0])
 
             add_queue(project=proj_obj, length=2000000, type="admin")
             irr_queue = add_queue(project=proj_obj, length=2000000, type="irr")
@@ -498,35 +502,7 @@ def form_valid(self, form):
                 if project_category.exists():
                     project_category.delete()
             else:
-                if project_category.exists():
-                    existing_category = Category.objects.get(project=self.object)
-                    if existing_category.field_name != chosen_category:
-                        # update the category
-                        label_field = LabelMetaDataField.objects.get(
-                            project=self.object, field_name=chosen_category
-                        )
-                        data_field = MetaDataField.objects.get(
-                            project=self.object, field_name=chosen_category
-                        )
-                        Category.objects.filter(project=self.object).update(
-                            field_name=chosen_category,
-                            label_metadata_field=label_field,
-                            data_metadata_field=data_field,
-                        )
-                else:
-                    label_field = LabelMetaDataField.objects.get(
-                        project=self.object, field_name=chosen_category
-                    )
-                    data_field = MetaDataField.objects.get(
-                        project=self.object, field_name=chosen_category
-                    )
-                    Category.objects.create(
-                        project=self.object,
-                        field_name=chosen_category,
-                        label_metadata_field=label_field,
-                        data_metadata_field=data_field,
-                    )
-
+                create_or_update_project_category(self.object, chosen_category)
             return redirect(self.get_success_url())
         else:
             return self.render_to_response(context)