diff --git a/stats/static/stats/global_assignments.js b/stats/static/stats/global_assignments.js index 30fb86a2..b4ca227f 100644 --- a/stats/static/stats/global_assignments.js +++ b/stats/static/stats/global_assignments.js @@ -71,7 +71,7 @@ $(function () { createGaugeChart(data[i].ns_country_code + '_unassigned',data[i].unassigned,'Unassigned reports','unassigned'); createGaugeChart(data[i].ns_country_code + '_progress',data[i].progress,'Reports in progress','progress'); createGaugeChart(data[i].ns_country_code + '_pending',data[i].pending,'Pending reports','pending'); - createGaugeChart(data[i].ns_country_code + '_nsblocked',data[i].blocked_ns,'NS queue','ns queue'); + createGaugeChart(data[i].ns_country_code + '_reserved',data[i].reserved,'Reserved for supervisor only','reserved'); } diff --git a/stats/templates/stats/global_assignments.html b/stats/templates/stats/global_assignments.html index 37a5edd2..d97e2264 100644 --- a/stats/templates/stats/global_assignments.html +++ b/stats/templates/stats/global_assignments.html @@ -53,34 +53,35 @@
+ {% with is_superexpert=user.userstat.is_superexpert %}

Category description:

+

Reserved reports

+

Reports reserved for national supervisors to validate first based on country-specific exclusivity days.

Unassigned reports

-

Unassigned reports are reports that have not been yet assigned to any expert. Reports which are hidden or missing pictures are not included in this count.

+

Unassigned reports pertain to those not yet allocated to any expert, excluding those reserved for the supervisor. Additionally, reports with hidden or missing pictures are not counted in this category.

Reports in progress

Reports in progress are reports which have not yet been assigned to at least 3 experts. Therefore, they are considered work in progress. This count excludes unassigned reports.

Pending reports

This is the total pending (assigned but not yet validated) reports for the users of the current country. These users include the national supervisor of the current country.

-

Reports in National Supervisor queue

-

Reports that are currently in the exclusivity period for the National Supervisor (and therefore not available to regular users), and NS has not yet claimed them

- {% if user.userstat.is_superexpert %} + {% if is_superexpert %}

Blocked reports

Number of reports which have been claimed by experts and not validated for more than {{ days }} days.

{% endif %}

Summary stats:

+

Total reports reserved for supervisors:

+

{{ summary.total_reserved }}

Total unassigned reports:

{{ summary.total_unassigned }}

Total in progress reports:

{{ summary.total_progress }}

Total pending reports:

{{ summary.total_pending }}

-

Total reports in National Supervisor queue:

-

{{ summary.total_blocked_ns }}

- {% if user.userstat.is_superexpert %} + {% if is_superexpert %}

Total blocked reports:

{{ summary.total_blocked }} {% if summary.total_blocked > 0 %} Manage blocked for all countries {% endif %}

{% endif %} @@ -93,51 +94,50 @@

{{ d.ns_country_name }}

National supervisor - {{ d.ns_username }}

-

Data last update - {{ d.last_update }} (UTC)

-
- {% if user.userstat.is_superexpert %} +
+ {% if is_superexpert %} {% endif %}
-
- {% if user.userstat.is_superexpert %} +
+ {% if is_superexpert %} {% endif %}
-
- {% if user.userstat.is_superexpert %} +
+ {% if is_superexpert %} {% endif %}
-
- {% if user.userstat.is_superexpert %} +
+ {% if is_superexpert %} {% endif %}
- {% if user.userstat.is_superexpert %} + {% if is_superexpert %} {% if d.blocked > 0 %}

Blocked reports

@@ -151,5 +151,6 @@

{{ d.blocked }}

{% endfor %} + {% endwith %} \ No newline at end of file diff --git a/stats/views.py b/stats/views.py index d7bc91ad..03506558 100644 --- a/stats/views.py +++ b/stats/views.py @@ -29,11 +29,8 @@ import math from django.urls import reverse from django.urls.exceptions import NoReverseMatch -from tigacrafting.report_queues import get_unassigned_available_reports, get_progress_available_reports, get_ns_locked_reports -from tigacrafting.views import get_blocked_reports_by_country -from django.db.models import Sum from django.db.models.functions import Extract, Trunc -import pytz + @xframe_options_exempt # @cache_page(60 * 15) @@ -158,12 +155,11 @@ def workload_available_reports(request): user_ids_arr = user_ids_string_to_int_array(user_ids_str) if len(user_ids_arr) > 0: user_id_filter = user_ids_arr - current_pending = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos=None).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations=0) - current_progress = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos=None).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=3).exclude(n_annotations=0) - overall_pending = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult') - current_pending = filter_reports(current_pending) - current_progress = filter_reports(current_progress) - data = { 'current_pending_n' : len(current_pending), 'current_progress_n' : len(current_progress), 'overall_pending': overall_pending.count()} + current_pending = Report.objects.queued().unassigned() + current_progress = Report.objects.in_progress() + + overall_pending = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False) + data = { 'current_pending_n' : current_pending.count(), 'current_progress_n' : current_progress.count(), 'overall_pending': overall_pending.count()} return Response(data) def compute_speedmeter_params(): @@ -762,179 +758,123 @@ def report_stats_ccaa(request): return render(request, 'stats/report_stats_ccaa.html', context) -def get_blocked_count(data): - n = 0 - for item in data.items(): - n = n + len(item[1]) - return n - @login_required def global_assignments(request): this_user = request.user - this_user_is_superexpert = this_user.groups.filter(name='superexpert').exists() - this_user_is_national_supervisor = this_user.userstat.is_national_supervisor() - lock_period = settings.ENTOLAB_LOCK_PERIOD - if this_user_is_superexpert: - assignments = GlobalAssignmentStat.objects.select_related('country').all().order_by('country__name_engl') - data = [] - total_blocked = 0 - for assignment in assignments: - blocked = get_blocked_reports_by_country(assignment.country) - n_blocked = get_blocked_count(blocked) - try: - userstat = UserStat.objects.get(national_supervisor_of=assignment.country) - ns_id = userstat.user.id - ns_username = userstat.user.username - except UserStat.DoesNotExist: - ns_id = 'N/A' - ns_username = 'N/A' - data.append( - { - "ns_id": ns_id, - "ns_username": ns_username, - "ns_country_id": assignment.country.gid, - "ns_country_code": assignment.country.iso3_code, - "ns_country_name": assignment.country.name_engl, - "unassigned": assignment.unassigned_reports, - "progress": assignment.in_progress_reports, - "pending": assignment.pending_reports, - "blocked": n_blocked, - "blocked_ns": assignment.nsqueue_reports, - "last_update": assignment.last_update.strftime('%d/%m/%Y - %H:%M:%S') - } - ) - total_blocked += n_blocked - totals = GlobalAssignmentStat.objects.aggregate(Sum('unassigned_reports'),Sum('in_progress_reports'),Sum('pending_reports'),Sum('nsqueue_reports')) - total_unassigned = totals['unassigned_reports__sum'] - total_progress = totals['in_progress_reports__sum'] - total_pending = totals['pending_reports__sum'] - total_blocked_ns = totals['nsqueue_reports__sum'] - summary = { 'total_unassigned':total_unassigned, 'total_progress':total_progress, 'total_pending':total_pending, 'total_blocked':total_blocked, 'total_blocked_ns': total_blocked_ns } - context = {'data': data, 'encoded_data': json.dumps(data), 'summary': summary, 'days': lock_period} - return render(request, 'stats/global_assignments.html', context) - elif this_user_is_national_supervisor: - data = [] - total_unassigned = 0 - total_progress = 0 - total_pending = 0 - total_blocked_ns = 0 - current_country = this_user.userstat.national_supervisor_of.gid - unassigned_filtered = get_unassigned_available_reports(this_user.userstat.national_supervisor_of) - progress_filtered = get_progress_available_reports(this_user.userstat.national_supervisor_of) - blocked_ns = get_ns_locked_reports(this_user.userstat.national_supervisor_of) - user_id_filter = UserStat.objects.filter(native_of__gid=current_country).values('user__id') - pending = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult').values('report') - n_unassigned = unassigned_filtered.count() - n_progress = progress_filtered.count() - n_pending = pending.count() - n_blocked_ns = 0 if blocked_ns is None else blocked_ns.count() - now = datetime.datetime.now(pytz.utc) - now_string = now.strftime('%d/%m/%Y - %H:%M:%S') + + is_national_supervisor = this_user.userstat.is_national_supervisor() + is_super_expert = this_user.userstat.is_superexpert() + + if not (is_national_supervisor or is_super_expert): + return HttpResponse("You need to be logged in as superexpert or be a national supervisor to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + + report_qs = Report.objects + if is_national_supervisor: + # Only allow stats from its country + report_qs = report_qs.filter(country=this_user.userstat.national_supervisor_of) + + n_unassigned = report_qs.queued().unassigned().in_supervisor_exclusivity_period(state=False).values('country').annotate( + total_count=Count(1) + ).values('country','total_count') + + n_in_exclusivity_period = report_qs.queued().in_supervisor_exclusivity_period(state=True).values('country').annotate( + total_count=Count(1) + ).values('country','total_count') + + n_progress = report_qs.in_progress().values('country').annotate( + total_count=Count(1) + ).values('country','total_count') + + n_pending = report_qs.with_pending_validation_to_finish().values('country').annotate( + total_count=Count(1) + ).values('country','total_count') + + n_blocked = report_qs.with_blocked_validations().values('country').annotate( + total_count=Count(1) + ).values('country','total_count') + + # Create a dictionary to store merged data + stats_country_data = {} + + # Merge data from each list of dicts + for data_list, fieldname in [(n_unassigned, "unassigned"), (n_progress, "progress"), (n_pending, "pending"), (n_blocked, "blocked"), (n_in_exclusivity_period, "reserved")]: + for entry in data_list: + country = entry['country'] or 'N/A' + if country not in stats_country_data: + stats_country_data[country] = {} + stats_country_data[country][fieldname] = entry['total_count'] + + country_info = list(EuropeCountry.objects.values('gid', 'iso3_code', 'name_engl', 'supervisors__user', 'supervisors__user__username')) + + # Append case when country is None + country_info.append({'gid': 'N/A', 'iso3_code': 'Other', 'name_engl': 'Other', 'supervisors__user': None, 'supervisors__user__username': None}) + + data = [] + for country in country_info: + stats = stats_country_data.get(country['gid']) data.append( { - "ns_id": this_user.id, - "ns_username": this_user.username, - "ns_country_id": this_user.userstat.national_supervisor_of.gid, - "ns_country_code": this_user.userstat.national_supervisor_of.iso3_code, - "ns_country_name": this_user.userstat.national_supervisor_of.name_engl, - "unassigned": n_unassigned, - "progress": n_progress, - "pending": n_pending, - "blocked_ns": n_blocked_ns, - "last_update": now_string + "ns_id": country['supervisors__user'], + "ns_username": country['supervisors__user__username'], + "ns_country_id":country['gid'], + "ns_country_code": country['iso3_code'], + "ns_country_name":country['name_engl'], + "unassigned": stats.get('unassigned', 0) if stats else 0, + "progress": stats.get('progress', 0) if stats else 0, + "pending": stats.get('pending', 0) if stats else 0, + "blocked": stats.get('blocked', 0) if stats else 0, + "reserved": stats.get('reserved', 0) if stats else 0, } ) - total_unassigned += n_unassigned - total_progress += n_progress - total_pending += n_pending - total_blocked_ns += n_blocked_ns - try: - g = GlobalAssignmentStat.objects.get(country=current_country) - g.unassigned_reports = n_unassigned - g.in_progress_reports = n_progress - g.pending_reports = n_pending - g.nsqueue_reports = n_blocked_ns - g.last_update = now - except GlobalAssignmentStat.DoesNotExist: - g = GlobalAssignmentStat( - country=current_country, - unassigned_reports=n_unassigned, - in_progress_reports=n_progress, - pending_reports=n_pending, - nsqueue_reports=n_blocked_ns, - last_update=now - ) - g.save() - summary = {'total_unassigned': total_unassigned, 'total_progress': total_progress, 'total_pending': total_pending, 'total_blocked_ns': total_blocked_ns} - context = {'data': data, 'encoded_data': json.dumps(data), 'summary': summary} - return render(request, 'stats/global_assignments.html', context) - else: - return HttpResponse("You need to be logged in as superexpert or be a national supervisor to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + summary = { + 'total_unassigned': sum(item['unassigned'] for item in data), + 'total_progress': sum(item['progress'] for item in data), + 'total_pending': sum(item['pending'] for item in data), + 'total_blocked': sum(item['blocked'] for item in data), + 'total_reserved': sum(item['reserved'] for item in data) + } + context = {'data': data, 'encoded_data': json.dumps(data), 'summary': summary, 'days': settings.ENTOLAB_LOCK_PERIOD} + return render(request, 'stats/global_assignments.html', context) @login_required def global_assignments_list(request, country_code=None, status=None): - countryID = EuropeCountry.objects.filter(iso3_code=country_code) - - for c in countryID: - countryGID = c.gid - countryName = c.name_engl - - report_id_table = {} - listas = [] - reportStatus = '' - - if status != 'pending': - if status == 'unassigned': - unassigned = get_unassigned_available_reports( EuropeCountry.objects.get(pk=countryGID) ) - reportList = unassigned - reportStatus = 'Unassigned' - elif status == 'progress': - progress = get_progress_available_reports( EuropeCountry.objects.get(pk=countryGID) ) - reportList = progress - reportStatus = 'In progress' - elif status == 'nsblocked': - nsblocked = get_ns_locked_reports( EuropeCountry.objects.get(pk=countryGID) ) - reportList = nsblocked - reportStatus = 'NS Queue' - - i = 0 - while i < len(reportList): - t = reportList[i].get_expert_recipients_names() - - expNames = t.split("+") - expNamesJoined = ' / '.join(expNames) - - listas.append({ - 'idReports': reportList[i].__unicode__(), - 'experts': expNamesJoined, - }) - i += 1 - - elif status == 'pending': - if countryGID == 17: - user_id_filter = settings.USERS_IN_STATS - reportList = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult') - else: - user_id_filter = UserStat.objects.filter(native_of__gid=countryGID).values('user__id') - reportList = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult') - - reportStatus = 'Pending' - i = 0 - while i < len(reportList): - - t = reportList[i].report.get_expert_recipients_names() + if country_code == 'Other': + country = None + else: + country = get_object_or_404(EuropeCountry, iso3_code=country_code) + + qs = Report.objects.filter(country=country) + + if status == 'pending': + qs = qs.with_pending_validation_to_finish() + title = 'Pending' + elif status == 'unassigned': + qs = qs.queued().unassigned() + title = 'Unassigned' + elif status == 'progress': + qs = qs.in_progress() + title = 'In progress' + elif status == 'reserved': + qs = qs.queued().in_supervisor_exclusivity_period(state=True) + title = 'Reserved for national supervisors' + else: + return HttpResponse('status not found.') - expNames = t.split("+") - expNamesJoined = ' / '.join(expNames) + result = [] + for obj in qs.order_by('-server_upload_time').prefetch_related('expert_report_annotations').iterator(): + names = obj.get_expert_recipients_names() - listas.append({ - 'idReports': reportList[i].report.__unicode__(), - 'experts': expNamesJoined - }) - i += 1 + expNames = names.split("+") + expNamesJoined = ' / '.join(expNames) + result.append( + { + 'idReports': obj.pk, + 'experts': expNamesJoined, + } + ) - context = {'data': listas, 'country': country_code, 'status': status, 'countryName': countryName, 'reportStatus': reportStatus} + context = {'data': result, 'country': country_code, 'status': status, 'countryName': country.name_engl, 'reportStatus': title} return render(request, 'stats/global_assignments_list.html', context) diff --git a/tigacrafting/managers.py b/tigacrafting/managers.py new file mode 100644 index 00000000..fbe9029b --- /dev/null +++ b/tigacrafting/managers.py @@ -0,0 +1,21 @@ +from datetime import timedelta + +from django.conf import settings +from django.db import models +from django.utils import timezone + +class ExpertReportAnnotationQuerySet(models.QuerySet): + + def blocked(self, days: int = settings.ENTOLAB_LOCK_PERIOD): + from tigaserver_app.models import Report + + return self.exclude( + user__groups__name='superexpert', + ).filter( + created__lt=timezone.now() - timedelta(days=days), + validation_complete=False + ).exclude( + report__in=Report.objects.with_finished_validation() + ) + +ExpertReportAnnotationManager = models.Manager.from_queryset(ExpertReportAnnotationQuerySet) \ No newline at end of file diff --git a/tigacrafting/migrations/0022_expertreportannotation_validation_complete_indexing.py b/tigacrafting/migrations/0022_expertreportannotation_validation_complete_indexing.py new file mode 100644 index 00000000..e341e32f --- /dev/null +++ b/tigacrafting/migrations/0022_expertreportannotation_validation_complete_indexing.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.7 on 2024-02-08 09:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tigacrafting', '0021_auto_20240125_1557'), + ] + + operations = [ + migrations.AlterField( + model_name='expertreportannotation', + name='validation_complete', + field=models.BooleanField(db_index=True, default=False, help_text='Mark this when you have completed your review and are ready for your annotation to be displayed to public.'), + ), + migrations.AlterField( + model_name='expertreportannotation', + name='validation_complete_executive', + field=models.BooleanField(db_index=True, default=False, help_text='Available only to national supervisor. Causes the report to be completely validated, with the final classification decided by the national supervisor'), + ), + ] diff --git a/tigacrafting/models.py b/tigacrafting/models.py index 20626438..e27e6999 100644 --- a/tigacrafting/models.py +++ b/tigacrafting/models.py @@ -1,8 +1,7 @@ -from typing import Optional, Literal +from typing import List, Optional, Literal -from django.db import models - -from django.db import models +from django.conf import settings +from django.db import models, transaction from django.contrib.auth import get_user_model from datetime import datetime, timedelta from django.core.validators import MaxValueValidator, MinValueValidator @@ -11,8 +10,8 @@ from django.db.models.signals import post_save from django.dispatch import receiver import tigacrafting.html_utils as html_utils -import pytz +from .managers import ExpertReportAnnotationManager from .messages import other_insect_msg_dict, albopictus_msg_dict, albopictus_probably_msg_dict, culex_msg_dict, notsure_msg_dict User = get_user_model() @@ -185,16 +184,16 @@ class MoveLabAnnotation(models.Model): SITE_CATEGORIES = ((2, 'Definitely a breeding site'), (1, 'Probably a breeding site'), (0, 'Not sure'), (-1, 'Probably not a breeding site'), (-2, 'Definitely not a breeding site')) -STATUS_CATEGORIES = ((1, 'public'), (0, 'flagged'), (-1, 'hidden')) - class ExpertReportAnnotation(models.Model): - MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT = 3 - VALIDATION_CATEGORY_DEFINITELY = 2 VALIDATION_CATEGORY_PROBABLY = 1 - VALIDATION_CATEGORIES = ((VALIDATION_CATEGORY_DEFINITELY, 'Definitely'), (VALIDATION_CATEGORY_PROBABLY, 'Probably')) + STATUS_HIDDEN = -1 + STATUS_FLAGGED = 0 + STATUS_PUBLIC = 1 + STATUS_CATEGORIES = ((STATUS_PUBLIC, 'public'), (STATUS_FLAGGED, 'flagged'), (STATUS_HIDDEN, 'hidden')) + user = models.ForeignKey(User, related_name="expert_report_annotations", on_delete=models.PROTECT, ) report = models.ForeignKey('tigaserver_app.Report', related_name='expert_report_annotations', on_delete=models.CASCADE, ) tiger_certainty_category = models.IntegerField('Tiger Certainty', choices=TIGER_CATEGORIES, default=None, blank=True, null=True, help_text='Your degree of belief that at least one photo shows a tiger mosquito') @@ -207,7 +206,7 @@ class ExpertReportAnnotation(models.Model): status = models.IntegerField('Status', choices=STATUS_CATEGORIES, default=1, help_text='Whether report should be displayed on public map, flagged for further checking before public display), or hidden.') #last_modified = models.DateTimeField(auto_now=True, default=datetime.now()) last_modified = models.DateTimeField(default=timezone.now) - validation_complete = models.BooleanField(default=False, help_text='Mark this when you have completed your review and are ready for your annotation to be displayed to public.') + validation_complete = models.BooleanField(default=False, db_index=True, help_text='Mark this when you have completed your review and are ready for your annotation to be displayed to public.') revise = models.BooleanField(default=False, help_text='For superexperts: Mark this if you want to substitute your annotation for the existing Expert annotations. Make sure to also complete your annotation form and then mark the "validation complete" box.') best_photo = models.ForeignKey('tigaserver_app.Photo', related_name='expert_report_annotations', null=True, blank=True, on_delete=models.SET_NULL, ) linked_id = models.CharField('Linked ID', max_length=10, help_text='Use this field to add any other ID that you want to associate the record with (e.g., from some other database).', blank=True) @@ -218,7 +217,9 @@ class ExpertReportAnnotation(models.Model): complex = models.ForeignKey('tigacrafting.Complex', related_name='expert_report_annotations', null=True, blank=True, help_text='Complex category assigned by expert or superexpert. Mutually exclusive with category. If this field has value, there should not be a validation value', on_delete=models.SET_NULL, ) validation_value = models.IntegerField('Validation Certainty', choices=VALIDATION_CATEGORIES, default=None, blank=True, null=True, help_text='Certainty value, 1 for probable, 2 for sure, 0 for none') other_species = models.ForeignKey('tigacrafting.OtherSpecies', related_name='expert_report_annotations', null=True, blank=True, help_text='Additional info supplied if the user selected the Other species category', on_delete=models.SET_NULL, ) - validation_complete_executive = models.BooleanField(default=False, help_text='Available only to national supervisor. Causes the report to be completely validated, with the final classification decided by the national supervisor') + validation_complete_executive = models.BooleanField(default=False, db_index=True, help_text='Available only to national supervisor. Causes the report to be completely validated, with the final classification decided by the national supervisor') + + objects = ExpertReportAnnotationManager() class Meta: constraints = [ @@ -233,17 +234,9 @@ def is_expert(self): @property def is_on_ns_executive_validation_period(self): - utc = pytz.UTC - if self.report.country is not None: - if UserStat.objects.filter(national_supervisor_of=self.report.country).exists(): - expiration_period = self.report.country.national_supervisor_report_expires_in - if expiration_period is None: - expiration_period = 14 - date_now = datetime.now().replace(tzinfo=utc) - date_expiration = date_now - timedelta(days=expiration_period) - if self.report.server_upload_time >= date_expiration: - return True - return False + from tigaserver_app.models import Report + + return Report.objects.in_supervisor_exclusivity_period().filter(pk=self.report).exist() @classmethod def _get_auto_message(cls, category: 'Categories', validation_value: int, locale: str = 'en') -> Optional[str]: @@ -286,9 +279,9 @@ def create_replicas(self): username_replicas = ["innie", "minnie", "manny"] report_annotations = ExpertReportAnnotation.objects.filter(report=self.report).count() - if report_annotations >= self.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT: + if report_annotations >= settings.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT: return - num_missing_annotations = self.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT - report_annotations + num_missing_annotations = settings.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT - report_annotations fieldnames = [ field.name for field in ExpertReportAnnotation._meta.fields if field.name not in ('id', 'user', 'report') @@ -515,7 +508,7 @@ def _can_be_simplified(self) -> bool: total_completed_annotations_qs = self.report.expert_report_annotations.all() return ( total_completed_annotations_qs.filter(simplified_annotation=False).exists() or - total_completed_annotations_qs.count() < self.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT - 1 + total_completed_annotations_qs.count() < settings.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT - 1 ) def save(self, *args, **kwargs): @@ -594,6 +587,43 @@ def pending_annotations(self): def num_pending_annotations(self) -> int: return self.pending_annotations.count() + @transaction.atomic + def assign_reports(self, country: Optional['EuropeCountry'] = None) -> List[Optional[ExpertReportAnnotation]]: + # Import here to avoid circular import error. + from tigaserver_app.models import Report + + report_queue = Report.objects.queued(user_prioritized=self.user) + if country is not None: + report_queue = report_queue.filter(country=country) + + if not self.is_superexpert(): + # Only assign until reaching the maximum allowed. + current_pending = self.num_pending_annotations + if current_pending >= settings.MAX_N_OF_PENDING_REPORTS: + return + + num_to_assign = settings.MAX_N_OF_PENDING_REPORTS - current_pending + report_queue = report_queue.all()[:num_to_assign] + + result = [] + for report in report_queue: + result.append( + ExpertReportAnnotation.objects.create( + report=report, + user=self.user + ) + ) + self.grabbed_reports += 1 + + self.save() + + return result + + def assign_crisis_report(self, country: 'EuropeCountry') -> List[Optional[ExpertReportAnnotation]]: + # NOTE: self.save() is called in assign_reports + self.last_emergency_mode_grab = country + return self.assign_reports(country=country) + def has_accepted_license(self): return self.license_accepted @@ -615,12 +645,6 @@ def is_team_not_bcn(self): def is_team_everywhere(self): return self.user.groups.exclude(name="team_not_bcn").exclude(name="team_bcn").exists() - def n_completed_annotations(self): - return self.user.expert_report_annotations.filter(validation_complete=True).count() - - def n_pending_annotations(self): - return self.user.expert_report_annotations.filter(validation_complete=False).count() - def is_bb_user(self): if self.native_of is not None: return self.native_of.is_bounding_box diff --git a/tigacrafting/report_queues.py b/tigacrafting/report_queues.py deleted file mode 100644 index 7e6217df..00000000 --- a/tigacrafting/report_queues.py +++ /dev/null @@ -1,653 +0,0 @@ -from tigaserver_app.models import Report, ExpertReportAnnotation, EuropeCountry -from django.core.exceptions import ObjectDoesNotExist -from tigacrafting.models import UserStat -from django.contrib.auth.models import User -from django.db.models import Count -from django.db.models import Q -from django.utils import timezone -from datetime import date, datetime, timedelta -import logging -import operator -import functools -from django.db import IntegrityError - -logger_report_assignment = logging.getLogger('mosquitoalert.report.assignment') -logger_duplicate_assignation = logging.getLogger('mosquitoalert.report.duplicateassignment') - -MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT = 3 -MAX_N_OF_PENDING_REPORTS = 5 -BCN_BB = {'min_lat': 41.321049, 'min_lon': 2.052380, 'max_lat': 41.468609, 'max_lon': 2.225610} - - -def get_base_adults_qs(): - #return Report.objects.exclude(creation_time__year=2014).exclude(creation_time__year=2015).exclude(note__icontains="#345").exclude(hide=True).exclude(photos__isnull=True).filter(type='adult') - return Report.objects.filter(creation_time__year__gt=2021).exclude(note__icontains="#345").exclude(hide=True).exclude(photos__isnull=True).filter(type='adult') - - -# def get_deleted_adult_reports(qs): -# return qs.filter(version_number=-1).values('report_id').distinct() -# This is not used anymore, because it's too naive and returns as deleted some reports which are in fact not deleted -# This happens because there is a lot more collisions between report_id -# The solution goes through this query -# -# select "version_UUID" -# from -# tigaserver_app_report r, -# ( -# select report_id, user_id, count("version_UUID") -# from -# tigaserver_app_report -# where -# type = 'adult' and report_id in -# (select distinct report_id from tigaserver_app_report where version_number = -1) -# group by report_id, user_id having count("version_UUID") >1 -# ) as deleted -# where r.report_id = deleted.report_id and r.user_id = deleted.user_id -# -# The internal count subquery gives a list of report_id, user_id of truly deleted reports. These are reports with the same -# report_id, belonging to the same user. The subclause in this query applies an additional requirement: there must be a -1 in -# the versions. This way we obtain the list of report_id, user_id for reports which contain a -1 in the series and belong to -# the same user. This is important, and avoids the fact of considering deleted a report with the same report_id as one marked -1, -# but belonging to a different user - - -def filter_reports_for_superexpert(reports): - # not deleted, last version, completely validated by at least three experts - #deleted_adult_reports = get_deleted_adult_reports(reports) - # not deleted - reports_qs = reports.filter(type=Report.TYPE_ADULT).non_deleted() - # fully validated - experts = User.objects.filter(groups__name='expert') - fully_validated = ExpertReportAnnotation.objects.filter( - report__in=reports_qs, - user__in=experts, - validation_complete=True - ).values('report').annotate( - n_validations=Count('report') - ).filter( - n_validations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT - ) - - report_ids = set(fully_validated.values_list('report', flat=True)) - filtered_reports = reports_qs.filter(version_UUID__in=report_ids) - #filtered_reports = filter(lambda x: len(list(filter(lambda y: y.is_expert() and y.validation_complete, x.expert_report_annotations.all()))) >= 3, latest_versions) - return filtered_reports - - -def filter_reports(reports): - #not deleted, last version - return reports.filter(type=Report.TYPE_ADULT).non_deleted() - - -def assign_reports_to_national_supervisor(this_user): - logger_report_assignment.debug('User {0} is national supervisor, assigning reports'.format(this_user, )) - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() - supervised_country = this_user.userstat.national_supervisor_of - logger_report_assignment.debug('User {0} is national supervisor for {1}'.format(this_user, supervised_country.name_engl)) - if current_pending < MAX_N_OF_PENDING_REPORTS: - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - n_to_get = MAX_N_OF_PENDING_REPORTS - current_pending - logger_report_assignment.debug('Getting {0} reports for user {1}'.format(n_to_get, this_user)) - country_with_supervisor = UserStat.objects.filter(national_supervisor_of__isnull=False).values('national_supervisor_of__gid').distinct() - bounding_boxes = EuropeCountry.objects.filter(is_bounding_box=True) - reports_supervised_country = get_base_adults_qs().filter(country__gid=supervised_country.gid).annotate(n_annotations=Count('expert_report_annotations')) - - # These are unassigned reports from supervised country, should be prioritized - reports_supervised_country = reports_supervised_country.filter(n_annotations=0).exclude( Q(country=supervised_country) & ~Q(server_upload_time__gte=timezone.now() - timedelta(days=supervised_country.national_supervisor_report_expires_in)) ) - - # list of countries with supervisor (excluding present) - country_with_supervisor_other_than_this = EuropeCountry.objects.filter(gid__in=country_with_supervisor).exclude(gid=supervised_country.gid) - reserved_for_supervised_countries_operator_list = [] - for country in country_with_supervisor_other_than_this: - if country.national_supervisor_report_expires_in is not None: - expiration_period = country.national_supervisor_report_expires_in - else: - expiration_period = 14 - reserved_for_supervised_countries_operator_list.append(Q(country=country) & Q(server_upload_time__gte=timezone.now() - timedelta(days=country.national_supervisor_report_expires_in))) - new_reports_unfiltered = get_base_adults_qs().exclude(country__in=bounding_boxes).exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - #reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude(functools.reduce(operator.or_, reserved_for_supervised_countries_operator_list)) - for condition in reserved_for_supervised_countries_operator_list: - new_reports_unfiltered = new_reports_unfiltered.exclude(condition) - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.filter(~Q(country__gid=17) & Q(country__gid__isnull=False)) - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter(user__userstat__national_supervisor_of__in=country_with_supervisor).filter(report__type='adult').filter(validation_complete=False).values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = reports_unfiltered_excluding_reserved_ns.exclude(version_UUID__in=blocked_by_experts) - - reports_unfiltered_excluding_reserved_ns_own_country = reports_unfiltered_excluding_reserved_ns.filter(country__gid=supervised_country.gid) - reports_unfiltered_excluding_reserved_ns_other_countries = reports_unfiltered_excluding_reserved_ns.exclude(country__gid=supervised_country.gid) - - country_filtered_reports = filter_reports(reports_supervised_country.order_by('creation_time')) - #these reports are from the ns country, but expired - non_executive_own_country_filtered_reports = filter_reports(reports_unfiltered_excluding_reserved_ns_own_country.order_by('creation_time')) - other_countries_filtered_reports = filter_reports(reports_unfiltered_excluding_reserved_ns_other_countries.order_by('creation_time')) - - currently_taken = 0 - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - - # for national supervisor, reports from supervised country are never simplified if granted first - for this_report in country_filtered_reports: - logger_report_assignment.debug('* Assigned Reserved report {0} to user {1}'.format(this_report.version_UUID, this_user)) - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - grabbed_reports += 1 - try: - new_annotation.save() - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - for this_report in non_executive_own_country_filtered_reports: - logger_report_assignment.debug('* Assigned Non Reserved own country report {0} to user {1}'.format(this_report.version_UUID, this_user)) - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - # No one has the report, is simplified - new_annotation.simplified_annotation = True - grabbed_reports += 1 - try: - new_annotation.save() - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - for this_report in other_countries_filtered_reports: - logger_report_assignment.debug('* Assigned Non Reserved other country report {0} to user {1}'.format(this_report.version_UUID,this_user)) - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - # No one has the report, is simplified - new_annotation.simplified_annotation = True - grabbed_reports += 1 - try: - new_annotation.save() - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - # reports_unfiltered = reports_supervised_country.order_by('creation_time') | reports_unfiltered_excluding_reserved_ns.order_by('creation_time') - # if reports_unfiltered: - # new_reports = filter_reports(reports_unfiltered) - # reports_to_take = new_reports[0:n_to_get] - # user_stats = None - # try: - # user_stats = UserStat.objects.get(user_id=this_user.id) - # except ObjectDoesNotExist: - # pass - # grabbed_reports = -1 - # if user_stats: - # grabbed_reports = user_stats.grabbed_reports - # for this_report in reports_to_take: - # if not this_report.user_has_report(this_user): - # new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - # who_has_count = this_report.get_who_has_count() - # if who_has_count == 0 or who_has_count == 1: - # # No one has the report, is simplified - # new_annotation.simplified_annotation = True - # grabbed_reports += 1 - # new_annotation.save() - # if grabbed_reports != -1 and user_stats: - # user_stats.grabbed_reports = grabbed_reports - # user_stats.save() - - -def get_progress_available_reports(country): - # expiration_period_days = 14 - expiration_period_days = country.national_supervisor_report_expires_in - if not expiration_period_days: - expiration_period_days = 14 - new_reports_unfiltered = get_base_adults_qs().filter(country=country).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=3).exclude(n_annotations=0) - if UserStat.objects.filter(national_supervisor_of=country).exists(): - if country.national_supervisor_report_expires_in is not None: - expiration_period_days = country.national_supervisor_report_expires_in - new_reports_unfiltered = new_reports_unfiltered.exclude(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period_days)) - # exclude reports assigned to supervisor but not yet validated - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter( - user__userstat__national_supervisor_of=country).filter(report__type='adult').filter(validation_complete=False) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.exclude(Q(report__country=country) & Q(report__server_upload_time__lt=timezone.now() - timedelta(days=expiration_period_days))) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude(version_UUID__in=blocked_by_experts) - available_reports = filter_reports(reports_unfiltered_excluding_reserved_ns.order_by('creation_time')) - return available_reports - -def get_ns_locked_reports(country): - expiration_period_days = country.national_supervisor_report_expires_in - if not expiration_period_days: - expiration_period_days = 14 - if UserStat.objects.filter(national_supervisor_of=country).exists(): - new_reports_unfiltered = get_base_adults_qs().filter(country=country).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations=0) - # reports in ns exclusion period - new_reports_unfiltered = new_reports_unfiltered.filter(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period_days)) - available_reports = filter_reports(new_reports_unfiltered.order_by('creation_time')) - return available_reports - else: - return None - - -def get_unassigned_available_reports(country): - #expiration_period_days = 14 - expiration_period_days = country.national_supervisor_report_expires_in - if not expiration_period_days: - expiration_period_days = 14 - new_reports_unfiltered = get_base_adults_qs().filter(country=country).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations=0) - if UserStat.objects.filter(national_supervisor_of=country).exists(): - if country.national_supervisor_report_expires_in is not None: - expiration_period_days = country.national_supervisor_report_expires_in - new_reports_unfiltered = new_reports_unfiltered.exclude(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period_days)) - # exclude reports assigned to supervisor but not yet validated - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter(user__userstat__national_supervisor_of=country).filter(report__type='adult').filter(validation_complete=False) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.exclude(Q(report__country=country) & Q(report__server_upload_time__lt=timezone.now() - timedelta(days=expiration_period_days))) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude(version_UUID__in=blocked_by_experts) - available_reports = filter_reports(reports_unfiltered_excluding_reserved_ns.order_by('creation_time')) - return available_reports - -def get_crisis_report_available_reports(country): - # expiration_period_days = 14 - expiration_period_days = country.national_supervisor_report_expires_in - if not expiration_period_days: - expiration_period_days = 14 - new_reports_unfiltered = get_base_adults_qs().filter(country=country).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - if UserStat.objects.filter(national_supervisor_of=country).exists(): - if country.national_supervisor_report_expires_in is not None: - expiration_period_days = country.national_supervisor_report_expires_in - new_reports_unfiltered = new_reports_unfiltered.exclude(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period_days)) - # exclude reports assigned to supervisor but not yet validated - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter(user__userstat__national_supervisor_of=country).filter(report__type='adult').filter(validation_complete=False) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.exclude(Q(report__country=country) & Q(report__server_upload_time__lt=timezone.now() - timedelta(days=expiration_period_days))) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude(version_UUID__in=blocked_by_experts) - available_reports = filter_reports(reports_unfiltered_excluding_reserved_ns.order_by('creation_time')) - return available_reports - - -def assign_crisis_report(this_user, country): - summary = {} - # expiration_period_days = 14 - expiration_period_days = country.national_supervisor_report_expires_in - if not expiration_period_days: - expiration_period_days = 14 - logger_report_assignment.debug('Assigning crisis report for User {0} in country {1}'.format(this_user, country.name_engl)) - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() - summary['current_pending'] = current_pending - if current_pending < MAX_N_OF_PENDING_REPORTS: - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - n_to_get = MAX_N_OF_PENDING_REPORTS - current_pending - summary['n_to_get'] = n_to_get - new_reports_unfiltered = get_base_adults_qs().filter(country=country).exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - if UserStat.objects.filter(national_supervisor_of=country).exists(): - if country.national_supervisor_report_expires_in is not None: - expiration_period_days = country.national_supervisor_report_expires_in - new_reports_unfiltered = new_reports_unfiltered.exclude(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period_days)) - #exclude reports assigned to supervisor but not yet validated - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter(user__userstat__national_supervisor_of=country).filter(report__type='adult').filter(validation_complete=False) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.exclude( Q(report__country=country) & Q(report__server_upload_time__lt=timezone.now() - timedelta(days=expiration_period_days)) ) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude(version_UUID__in=blocked_by_experts) - if reports_unfiltered_excluding_reserved_ns: - new_reports = filter_reports(reports_unfiltered_excluding_reserved_ns.order_by('creation_time')) - reports_to_take = new_reports[0:n_to_get] - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - for this_report in reports_to_take: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - # No one has the report, is simplified - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - return summary - - -def this_report_can_be_simplified(report): - who_has_count = report.get_who_has_count() - there_is_full_report = ExpertReportAnnotation.objects.filter(report=report).filter(simplified_annotation=False).exists() - if there_is_full_report: - return True - if who_has_count == 0 or who_has_count == 1: - return True - return False - - -def _do_assign(report_list, this_user, grabbed_reports, currently_taken): - if report_list: - for this_report in report_list: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - if this_report_can_be_simplified(this_report): - # No one has the report, is simplified - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug( - 'Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - return { 'grabbed_reports': grabbed_reports, 'currently_taken': currently_taken } - - -def assign_reports_to_regular_user(this_user): - logger_report_assignment.debug('User {0} is regular user, assigning reports'.format(this_user, )) - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() - if current_pending < MAX_N_OF_PENDING_REPORTS: - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - n_to_get = MAX_N_OF_PENDING_REPORTS - current_pending - country_with_supervisor = UserStat.objects.filter(national_supervisor_of__isnull=False).values('national_supervisor_of__gid').distinct() - bounding_boxes = EuropeCountry.objects.filter(is_bounding_box=True) - supervised_countries = EuropeCountry.objects.filter(gid__in=country_with_supervisor) - reserved_for_supervised_countries_operator_list = [] - for country in supervised_countries: - if country.national_supervisor_report_expires_in is not None: - expiration_period = country.national_supervisor_report_expires_in - else: - expiration_period = 14 - reserved_for_supervised_countries_operator_list.append( Q(country=country) & Q(server_upload_time__gte=timezone.now() - timedelta(days=expiration_period)) ) - new_reports_unfiltered = get_base_adults_qs().exclude(country__in=bounding_boxes).exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - #reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.exclude( functools.reduce(operator.or_, reserved_for_supervised_countries_operator_list) ) - for condition in reserved_for_supervised_countries_operator_list: - new_reports_unfiltered = new_reports_unfiltered.exclude(condition) - ''' - if this_user.groups.filter(name='eu_group_europe').exists(): #Europe - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.filter( ~Q(country__gid=17) & Q(country__gid__isnull=False) ) - else: #Spain - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered.filter( Q(country__gid=17) | Q(country__gid__isnull=True) ) - ''' - # We don't segment anymore by country, this must be done later - reports_unfiltered_excluding_reserved_ns = new_reports_unfiltered - #exclude reports assigned to ANY supervisor but not yet validated - ### !!!! ### - reports_assigned_to_supervisor_not_yet_validated = ExpertReportAnnotation.objects.filter(user__userstat__national_supervisor_of__in=country_with_supervisor).filter(report__type='adult').filter(validation_complete=False) - for country in supervised_countries: - country_supervisor = User.objects.get( userstat__national_supervisor_of=country ) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.exclude( Q(report__country=country) & Q(report__server_upload_time__lt=timezone.now() - timedelta(days=country.national_supervisor_report_expires_in)) ) - # this weird second filter is to ensure that reports assigned to a national supervisor but not from their own country are blocked - # if this wasn't here, a French report assigned to the NS of Austria could be considered blocked - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.filter( Q(user=country_supervisor) & Q(report__country=country) ) - reports_assigned_to_supervisor_not_yet_validated = reports_assigned_to_supervisor_not_yet_validated.values('report').distinct() - blocked_by_experts = get_base_adults_qs().filter(version_UUID__in=reports_assigned_to_supervisor_not_yet_validated) - reports_unfiltered_excluding_reserved_ns = reports_unfiltered_excluding_reserved_ns.exclude(version_UUID__in=blocked_by_experts) - reports_validated_by_ns_non_exec = [] - - this_user_country = this_user.userstat.native_of - this_user_ns = None - try: - if this_user_country is not None: - this_user_ns_userstat = UserStat.objects.get(national_supervisor_of=this_user_country) - this_user_ns = this_user_ns_userstat.user - except UserStat.DoesNotExist: - pass - - if this_user_ns is not None: - annotation_non_exec_validated_ns = ExpertReportAnnotation.objects.filter(user=this_user_ns).filter(Q(validation_complete=True) & Q(validation_complete_executive=False)).values('report').distinct() - reports_validated_by_ns_non_exec = Report.objects.exclude(country__in=bounding_boxes).filter(version_UUID__in=annotation_non_exec_validated_ns).exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - - if reports_unfiltered_excluding_reserved_ns or reports_validated_by_ns_non_exec: - new_reports = filter_reports(reports_unfiltered_excluding_reserved_ns.order_by('creation_time')) - if not this_user.groups.filter(name='eu_group_europe').exists(): #Spain - this_user_nuts2 = this_user.userstat.nuts2_assignation - reports_own_region = None - reports_other_region = None - if this_user_nuts2 is not None: - reports_own_region = new_reports.filter(Q(country__gid=17) & Q(nuts_2=this_user_nuts2.nuts_id)) - reports_other_region = new_reports.filter(Q(country__gid=17) & ~Q(nuts_2=this_user_nuts2.nuts_id)) - else: - reports_own_region = new_reports.filter(country__gid=17) - reports_other_region = None - reports_europe = new_reports.filter( ~Q(country__gid=17) & Q(country__gid__isnull=False) ) - reports_other = new_reports.filter( country__gid__isnull=True ) - - currently_taken = 0 - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - - result = _do_assign(reports_own_region, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(reports_other_region, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(reports_europe, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(reports_other, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - - - ''' - reports_to_take = new_reports[0:n_to_get] - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - for this_report in reports_to_take: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - ''' - - else: #Europe -> prioritize reports from own country - new_reports_own_country = new_reports.filter(country=this_user.userstat.native_of) - new_reports_other_countries = new_reports.exclude(country=this_user.userstat.native_of) - new_reports_other_countries_not_null = new_reports_other_countries.exclude(country__gid__isnull=True) - new_reports_other_country_null = new_reports_other_countries.filter(country__gid__isnull=True) - - currently_taken = 0 - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - - ''' - for this_report in reports_validated_by_ns_non_exec: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - if this_report_can_be_simplified(this_report): - # No one has the report, is simplified - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug( - 'Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - ''' - result = _do_assign(reports_validated_by_ns_non_exec,this_user,grabbed_reports,currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(new_reports_own_country,this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - ''' - for this_report in new_reports_own_country: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - if this_report_can_be_simplified(this_report): - # No one has the report, is simplified - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - ''' - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(new_reports_other_countries_not_null, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - - if currently_taken < MAX_N_OF_PENDING_REPORTS: - result = _do_assign(new_reports_other_country_null, this_user, grabbed_reports, currently_taken) - grabbed_reports = result['grabbed_reports'] - currently_taken = result['currently_taken'] - ''' - for this_report in new_reports_other_countries: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - # No one has the report, is simplified - new_annotation.simplified_annotation = True - grabbed_reports += 1 - try: - new_annotation.save() - currently_taken += 1 - if currently_taken >= MAX_N_OF_PENDING_REPORTS: - break - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - ''' - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - - -def assign_superexpert_reports(this_user): - logger_report_assignment.debug('User {0} is superexpert, assigning reports'.format(this_user, )) - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - new_reports_unfiltered = get_base_adults_qs().exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__gte=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - if new_reports_unfiltered and this_user.groups.filter(name='team_bcn').exists(): - new_reports_unfiltered = new_reports_unfiltered.filter(Q(location_choice='selected', selected_location_lon__range=(BCN_BB['min_lon'],BCN_BB['max_lon']),selected_location_lat__range=(BCN_BB['min_lat'], BCN_BB['max_lat'])) | Q(location_choice='current', current_location_lon__range=(BCN_BB['min_lon'],BCN_BB['max_lon']),current_location_lat__range=(BCN_BB['min_lat'], BCN_BB['max_lat']))) - if new_reports_unfiltered and this_user.groups.filter(name='team_not_bcn').exists(): - new_reports_unfiltered = new_reports_unfiltered.exclude(Q(location_choice='selected', selected_location_lon__range=(BCN_BB['min_lon'],BCN_BB['max_lon']),selected_location_lat__range=(BCN_BB['min_lat'], BCN_BB['max_lat'])) | Q(location_choice='current', current_location_lon__range=(BCN_BB['min_lon'],BCN_BB['max_lon']),current_location_lat__range=(BCN_BB['min_lat'], BCN_BB['max_lat']))) - if this_user.id == 25: # it's roger, don't assign reports from barcelona prior to 03/10/2017 - new_reports_unfiltered = new_reports_unfiltered.exclude(Q(Q(location_choice='selected', selected_location_lon__range=(BCN_BB['min_lon'], BCN_BB['max_lon']),selected_location_lat__range=(BCN_BB['min_lat'], BCN_BB['max_lat'])) | Q(location_choice='current',current_location_lon__range=(BCN_BB['min_lon'],BCN_BB['max_lon']),current_location_lat__range=(BCN_BB['min_lat'],BCN_BB['max_lat']))) & Q(creation_time__date__lte=date(2017, 3, 10))) - new_reports = filter_reports_for_superexpert(new_reports_unfiltered) - for this_report in new_reports: - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - try: - new_annotation.save() - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - - -def assign_bb_reports(this_user): - logger_report_assignment.debug('User {0} is bounding box user, assigning reports'.format(this_user, )) - logger_report_assignment.debug('Bounding box for user {0} is {1}'.format(this_user, this_user.userstat.native_of.name_engl)) - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() - if current_pending < MAX_N_OF_PENDING_REPORTS: - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - n_to_get = MAX_N_OF_PENDING_REPORTS - current_pending - logger_report_assignment.debug('Getting {0} reports for user {1}'.format(n_to_get, this_user )) - new_reports_unfiltered = get_base_adults_qs().exclude(version_UUID__in=my_reports).annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) - new_reports_unfiltered = new_reports_unfiltered.filter(country=this_user.userstat.native_of) - logger_report_assignment.debug('{0} reports potentially assignable for user {1}'.format(len(new_reports_unfiltered), this_user)) - if new_reports_unfiltered: - new_reports = filter_reports(new_reports_unfiltered.order_by('creation_time')) - reports_to_take = new_reports[0:n_to_get] - user_stats = None - try: - user_stats = UserStat.objects.get(user_id=this_user.id) - except ObjectDoesNotExist: - pass - grabbed_reports = -1 - if user_stats: - grabbed_reports = user_stats.grabbed_reports - for this_report in reports_to_take: - logger_report_assignment.debug('* Assigned report {0} to user {1}'.format(this_report.version_UUID, this_user)) - new_annotation = ExpertReportAnnotation(report=this_report, user=this_user) - who_has_count = this_report.get_who_has_count() - exists_long = this_report.get_expert_has_been_assigned_long_validation() - if exists_long or who_has_count == 0 or who_has_count == 1: - # No one has the report, is simplified - new_annotation.simplified_annotation = True - try: - new_annotation.save() - grabbed_reports += 1 - except IntegrityError as e: - logger_duplicate_assignation.debug('Tried to assign twice report {0} to user {1}'.format(this_report, this_user, )) - if grabbed_reports != -1 and user_stats: - user_stats.grabbed_reports = grabbed_reports - user_stats.save() - - -def assign_reports(this_user): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: #is regular user - assign_reports_to_regular_user(this_user) diff --git a/tigacrafting/templates/tigacrafting/expert_status.html b/tigacrafting/templates/tigacrafting/expert_status.html index 96ac53a0..d8d12d53 100644 --- a/tigacrafting/templates/tigacrafting/expert_status.html +++ b/tigacrafting/templates/tigacrafting/expert_status.html @@ -65,7 +65,7 @@

{{ user.first_name }} {{ user.last_name }}

Pending Reports - {{ user.userstat.n_pending_annotations }} + {{ user.userstat.num_pending_annotations }}

@@ -101,7 +101,7 @@

{{ user.first_name }} {{ user.last_name }}

Complete Reports - {{ user.userstat.n_completed_annotations }} + {{ user.userstat.num_completed_annotations }}

diff --git a/tigacrafting/tests/tests.py b/tigacrafting/tests/tests.py index dc3ab597..40baffc2 100644 --- a/tigacrafting/tests/tests.py +++ b/tigacrafting/tests/tests.py @@ -1,23 +1,16 @@ # Create your tests here. +from datetime import timedelta from django.test import TestCase from django.utils.translation import activate, deactivate, gettext as _ from tigaserver_app.models import NutsEurope, EuropeCountry, TigaUser, Report, ExpertReportAnnotation, Photo, NotificationContent, Notification -from tigacrafting.models import UserStat, ExpertReportAnnotation, Categories, Complex +from tigacrafting.models import ExpertReportAnnotation, Categories, Complex from tigacrafting.views import must_be_autoflagged from django.contrib.auth.models import User, Group from django.utils import timezone -from django.db.models import Count, Q -from django.core.exceptions import ObjectDoesNotExist -from operator import attrgetter +from django.db.models import Q from tigacrafting.messaging import send_finished_validation_notification import tigaserver_project.settings as conf from django.utils import timezone -import pytz -from tigacrafting.report_queues import * - -### HELPER STUFF ######################################################################## - -BCN_BB = {'min_lat': 41.321049, 'min_lon': 2.052380, 'max_lat': 41.468609, 'max_lon': 2.225610} def user_summary(user): @@ -544,6 +537,9 @@ def create_report_pool(self): p.save() a = a + 1 + # NOTE: report queue filters by server_upload_time + Report.objects.all().update(server_upload_time=non_naive_time) + def create_small_region_team(self): spain_group = Group.objects.create(name='eu_group_spain') spain_group.save() @@ -709,16 +705,7 @@ def test_last_assignment(self): self.create_report_pool() # they are all regular users but ... for this_user in User.objects.exclude(id=24): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() for r in Report.objects.all(): annos = ExpertReportAnnotation.objects.filter(report=r) if annos.count() == 3: @@ -741,16 +728,7 @@ def test_assign_reports(self): self.create_report_pool() for this_user in User.objects.exclude(id=24): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() #get all national supervisors for this_user in User.objects.filter(userstat__national_supervisor_of__isnull=False): @@ -842,16 +820,7 @@ def test_simplified_assignation_two_experts_and_ns_report_from_not_supervised_co c = EuropeCountry.objects.get(pk=23) #France report = create_report(0, "1", t, c) for this_user in User.objects.exclude(id=24): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() # There should be three assignations n = ExpertReportAnnotation.objects.all().count() self.assertTrue( n == 3, "There should be {0} annotations, {1} found".format( 3, n ) ) @@ -874,16 +843,7 @@ def test_simplified_assignation_two_experts_and_ns_report_from_supervised_countr # we create an austrian report, with current time. That means it's locked by ns report = create_report(0, "1", t, c) for this_user in User.objects.exclude(id=24): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() # There should be ONE assignation n = ExpertReportAnnotation.objects.all().count() self.assertTrue(n == 1, "There should be {0} annotations, {1} found".format(1, n)) @@ -893,16 +853,7 @@ def test_simplified_assignation_two_experts_and_ns_report_from_supervised_countr ns_validation.save() # Now report it's validated AND NO LONGER LOCKED, reassign for this_user in User.objects.exclude(id=24): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() n = ExpertReportAnnotation.objects.all().count() # it should now be assigned to 3 experts (ns, and two regulars) self.assertTrue(n == 3, "There should be {0} annotations, {1} found".format(1, n)) @@ -923,13 +874,13 @@ def test_report_outdate(self): self.create_outdated_report_pool() # assign reports to regular user. All assigned reports should be from isle of man user = User.objects.get(pk=10) - assign_reports_to_regular_user(user) + user.userstat.assign_reports() # user should have been assigned 2 outdated reports assigned_reports = ExpertReportAnnotation.objects.filter(user=user) self.assertTrue( assigned_reports.count() == 2, "User {0} has been assigned {1} reports, should have been assigned {2}".format( user.username, assigned_reports.count(), 5 ) ) national_supervisor_isleofman = User.objects.get(pk=3) - assign_reports_to_national_supervisor(national_supervisor_isleofman) + national_supervisor_isleofman.userstat.assign_reports() server_upload_time_first_report = ExpertReportAnnotation.objects.filter(user=national_supervisor_isleofman).order_by('id')[0].report.server_upload_time server_upload_time_first_report_str = server_upload_time_first_report.strftime('%Y-%m-%d') self.assertTrue( server_upload_time_first_report_str == timezone.now().strftime('%Y-%m-%d'), "Server upload time of first assigned report should be {0}, is {1}".format( timezone.now().strftime('%Y-%m-%d'), server_upload_time_first_report_str ) ) @@ -979,25 +930,8 @@ def test_assign_stlouis_reports(self): self.create_stlouis_team() self.create_stlouis_report_pool() - number_of_assignments_to_regular_user = 0 - number_of_assignments_to_bb_stlouis = 0 - for this_user in User.objects.exclude(id__in=[24,25]): - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - number_of_assignments_to_bb_stlouis += 1 - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) - number_of_assignments_to_regular_user += 1 - - self.assertEqual( number_of_assignments_to_regular_user, 2, "Assigned reports to regular users {0} times, should be 2".format( number_of_assignments_to_regular_user ) ) - self.assertEqual( number_of_assignments_to_bb_stlouis, 3, "Assigned reports to bb stlouis users {0} times, should be 3".format( number_of_assignments_to_bb_stlouis ) ) + this_user.userstat.assign_reports() #all stlouis experts id=2,3,4 should be assigned 5 reports for i in [2, 3, 4]: @@ -1096,7 +1030,7 @@ def test_outdated_assign(self): #Now assign reports to Faroes native. Should receive report with uuid 1 faroes_native_regular_user = User.objects.get(username='expert_9_eu') - assign_reports_to_regular_user(faroes_native_regular_user) + faroes_native_regular_user.userstat.assign_reports() #should have been assigned the Faroes report, since the report is outdated and therefore no longer blocked by NS n_assigned_to_faroes_user = ExpertReportAnnotation.objects.filter(user=faroes_native_regular_user).filter(report=r).count() @@ -1180,16 +1114,7 @@ def test_spanish_regionalization(self): self.assertTrue(users[0].id == 3, "First expert to be assigned should be eu, is not") for this_user in users: - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() # All reports assigned to catalan expert should be from Catalonia catalan = User.objects.get(pk=1) @@ -1234,16 +1159,7 @@ def test_regionalization_priority_queues(self): self.assertTrue(users.count() == 2, "There should be 2 experts, there are {0}".format(users.count())) for this_user in users: - if this_user.userstat.is_superexpert(): - assign_superexpert_reports(this_user) - else: - if this_user.userstat.is_bb_user(): - assign_bb_reports(this_user) - else: - if this_user.userstat.is_national_supervisor(): - assign_reports_to_national_supervisor(this_user) - else: # is regular user - assign_reports_to_regular_user(this_user) + this_user.userstat.assign_reports() extremadura = NutsEurope.objects.get(name_latn='Extremadura') extremaduran = User.objects.get(pk=1) @@ -1263,4 +1179,4 @@ def test_regionalization_priority_queues(self): self.assertTrue(generic_assignments.filter(report__country__gid=17).count() == 4, "Expert should be assigned 4 spanish reports, has been assigned {0}".format( generic_assignments.filter(report__country__gid=17).count() )) n_european = generic_assignments.exclude(report__country__gid=17).exclude(report__country__isnull=True).count() self.assertTrue(n_european == 1,"Expert should be 1 european report, has been assigned {0}".format(n_european)) - # print( generic_assignments.exclude(report__country__gid=17).exclude(report__country__isnull=True).first().report.country.name_engl ) \ No newline at end of file + # print( generic_assignments.exclude(report__country__gid=17).exclude(report__country__isnull=True).first().report.country.name_engl ) diff --git a/tigacrafting/views.py b/tigacrafting/views.py index f1141188..951a8b5b 100644 --- a/tigacrafting/views.py +++ b/tigacrafting/views.py @@ -1,6 +1,6 @@ # coding=utf-8 from pydoc import visiblename -from django.shortcuts import render +from django.shortcuts import render, get_object_or_404 from django.core.exceptions import ObjectDoesNotExist import requests import json @@ -45,7 +45,6 @@ import functools import operator import math -from tigacrafting.report_queues import get_crisis_report_available_reports #----------Metadades fotos----------# @@ -55,7 +54,6 @@ from tigaserver_project.settings import * from rest_framework.response import Response import re -from tigacrafting.report_queues import assign_reports, get_base_adults_qs from django.utils import timezone #-----------------------------------# @@ -226,16 +224,6 @@ def filter_tasks(tasks): tasks_filtered = filter(lambda x: not x.photo.report.deleted, tasks) return tasks_filtered -def fast_filter_reports(reports): - #these are all the report_id in reports, with its max version number and the number of versions - #the values at the end drops everything into a dictionary - keys = Report.objects.exclude(creation_time__year=2014).filter(type='adult').values('report_id').annotate(Max('version_number')).annotate(Min('version_number')).annotate(Count('version_number')) - report_id_table = {} - for row in keys: - report_id_table[row['report_id']] = {'max_version':row['version_number__max'],'min_version':row['version_number__min'],'num_versions': row['version_number__count']} - reports_filtered = filter( lambda x: report_id_table[x.report_id]['num_versions'] == 1 or (report_id_table[x.report_id]['min_version'] != -1 and x.version_number == report_id_table[x.report_id]['max_version']), reports ) - return reports_filtered - def filter_reports(reports, sort=True): reports_filtered = reports.non_deleted() @@ -447,9 +435,6 @@ def movelab_annotation_pending(request, scroll_position='', tasks_per_page='50', else: return HttpResponse("You need to be logged in as a MoveLab member to view this page.") - -BCN_BB = {'min_lat': 41.321049, 'min_lon': 2.052380, 'max_lat': 41.468609, 'max_lon': 2.225610} - ITALY_GEOMETRY = GEOSGeometry('{"type": "Polygon","coordinates": [[[7.250976562499999,43.70759350405294],[8.96484375,44.071800467511565],[10.96435546875,41.95131994679697],[7.822265625000001,41.0130657870063],[8.1298828125,38.788345355085625],[11.865234375,37.735969208590504],[14.1064453125,36.70365959719456],[14.985351562499998,36.31512514748051],[18.80859375,40.27952566881291],[12.45849609375,44.5278427984555],[13.86474609375,45.413876460821086],[14.04052734375,46.51351558059737],[12.238769531249998,47.264320080254805],[6.8994140625,46.10370875598026],[6.43798828125,45.120052841530544],[7.250976562499999,43.70759350405294]]]}') @@ -504,8 +489,7 @@ def predefined_messages(request): def update_pending_data(country): - data = get_crisis_report_available_reports(country) - country.pending_crisis_reports = data.count() + country.pending_crisis_reports = Report.objects.queued().filter(country=country).count() country.last_crisis_report_n_update = timezone.now() country.save() @@ -542,244 +526,189 @@ def expert_geo_report_assign(request): return HttpResponse("You don't have emergency mode permissions, so you can't see this page. Please contact your administrator.") -def get_blocked_reports_by_country(country_id): - lock_period = settings.ENTOLAB_LOCK_PERIOD - superexperts = User.objects.filter(groups__name='superexpert') - if country_id == 17: #Spain - country_experts = User.objects.filter(id__in=settings.USERS_IN_STATS) - else: - country_experts = User.objects.filter(userstat__native_of=country_id) - annos = ExpertReportAnnotation.objects.filter(validation_complete=False).filter(user__in=country_experts).exclude(user__in=superexperts).order_by('user__username', 'report') - data = {} - for anno in annos: - elapsed = (datetime.now(timezone.utc) - anno.created).days - if elapsed > lock_period: - try: - data[anno.user.username] - except KeyError: - data[anno.user.username] = [] - data[anno.user.username].append({'annotation': anno, 'days': elapsed}) - return data - - -def sort_by_days(elem): - return elem[1][0]['days'] - - @login_required def report_expiration(request, country_id=None): this_user = request.user - this_user_is_superexpert = this_user.groups.filter(name='superexpert').exists() + + if not this_user.userstat.is_superexpert(): + return HttpResponse("You need to be logged in as superexpert to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + + lock_period = settings.ENTOLAB_LOCK_PERIOD + qs = ExpertReportAnnotation.objects.blocked(days=lock_period) + country = None if country_id is not None: - country = EuropeCountry.objects.get(pk=country_id) - if this_user_is_superexpert: - lock_period = settings.ENTOLAB_LOCK_PERIOD - superexperts = User.objects.filter(groups__name='superexpert') - if country_id is not None: - if country_id == 17: # Spain - country_experts = User.objects.filter(id__in=settings.USERS_IN_STATS) - else: - country_experts = User.objects.filter(userstat__native_of=country_id) - annos = ExpertReportAnnotation.objects.filter(validation_complete=False).filter(user__in=country_experts).exclude(user__in=superexperts).order_by('user__username', 'report') - else: - annos = ExpertReportAnnotation.objects.filter(validation_complete=False).exclude(user__in=superexperts).order_by('user__username', 'report') - data = { } - sorted_data = [] - for anno in annos: - elapsed = (datetime.now(timezone.utc) - anno.created).days - if elapsed > lock_period: - try: - data[anno.user.username] - except KeyError: - data[anno.user.username] = [] - data[anno.user.username].append({'annotation': anno, 'days': elapsed}) - sorted_data = sorted( data.items(), key=sort_by_days, reverse=True ) + country = get_object_or_404(EuropeCountry, pk=country_id) + qs = qs.filter(report__country=country) + + data_dict = {} + for annotation in qs.select_related('user').iterator(): + if annotation.user.username not in data_dict: + data_dict[annotation.user.username] = [] + + data_dict[annotation.user.username].append( + { + 'annotation': annotation, + 'days': (timezone.now() - annotation.created).days + } + ) - return render(request, 'tigacrafting/report_expiration.html', { 'data':sorted_data, 'lock_period': lock_period , 'country': country}) - else: - return HttpResponse("You need to be logged in as superexpert to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + sorted_data = sorted(data_dict.items(), key=lambda x: x[1][0]['days'], reverse=True) + + return render(request, 'tigacrafting/report_expiration.html', { 'data':sorted_data, 'lock_period': settings.ENTOLAB_LOCK_PERIOD , 'country': country}) @transaction.atomic @login_required def expert_report_annotation(request, scroll_position='', tasks_per_page='10', note_language='es', load_new_reports='F', year='all', orderby='date', tiger_certainty='all', site_certainty='all', pending='na', checked='na', status='all', final_status='na', max_pending=5, max_given=3, version_uuid='na', linked_id='na', ns_exec='all', edit_mode='off', tags_filter='na',loc='na'): this_user = request.user - if getattr(settings, 'SHOW_USER_AGREEMENT_ENTOLAB', False) == True: + + if settings.SHOW_USER_AGREEMENT_ENTOLAB: if this_user.userstat: if not this_user.userstat.has_accepted_license(): return HttpResponseRedirect(reverse('entolab_license_agreement')) else: return HttpResponse("There is a problem with your current user. Please contact the EntoLab admin at " + settings.ENTOLAB_ADMIN) - this_user_is_expert = this_user.groups.filter(name='expert').exists() - this_user_is_superexpert = this_user.groups.filter(name='superexpert').exists() - this_user_is_team_bcn = this_user.groups.filter(name='team_bcn').exists() - this_user_is_team_not_bcn = this_user.groups.filter(name='team_not_bcn').exists() - this_user_is_team_venezuela = this_user.groups.filter(name='team_venezuela').exists() - this_user_is_team_stlouis = this_user.groups.filter(name='team_stlouis').exists() - #this_user_is_team_italy = this_user.groups.filter(name='team_italy').exists() - #this_user_is_team_not_italy = this_user.groups.filter(name='team_not_italy').exists() - this_user_is_reritja = (this_user.id == 25) - - if this_user_is_expert or this_user_is_superexpert: - args = {} - args.update(csrf(request)) - args['scroll_position'] = scroll_position - if this_user_is_superexpert: - AnnotationFormset = modelformset_factory(ExpertReportAnnotation, form=SuperExpertReportAnnotationForm, extra=0) - else: - AnnotationFormset = modelformset_factory(ExpertReportAnnotation, form=ExpertReportAnnotationForm, extra=0) - if request.method == 'POST': - scroll_position = request.POST.get("scroll_position", '0') - orderby = request.POST.get('orderby', orderby) - tiger_certainty = request.POST.get('tiger_certainty', tiger_certainty) - site_certainty = request.POST.get('site_certainty', site_certainty) - pending = request.POST.get('pending', pending) - status = request.POST.get('status', status) - final_status = request.POST.get('final_status', final_status) - version_uuid = request.POST.get('version_uuid', version_uuid) - linked_id = request.POST.get('linked_id', linked_id) - ns_exec = request.POST.get('ns_exec', ns_exec) - tags_filter = request.POST.get('tags_filter', tags_filter) - checked = request.POST.get('checked', checked) - loc = request.POST.get('loc', loc) - tasks_per_page = request.POST.get('tasks_per_page', tasks_per_page) - note_language = request.GET.get('note_language', "es") - load_new_reports = request.POST.get('load_new_reports', load_new_reports) - save_formset = request.POST.get('save_formset', "F") - if save_formset == "T": - formset = AnnotationFormset(request.POST) - if formset.is_valid(): - for f in formset: - one_form = f.save(commit=False) - auto_flag = must_be_autoflagged(one_form,one_form.validation_complete) - if auto_flag: - one_form.status = 0 - one_form.save() - f.save_m2m() - if(this_user_is_reritja and one_form.validation_complete == True): - send_finished_validation_notification(one_form) - if auto_flag: - autoflag_others(one_form.id) - else: - return render(request, 'tigacrafting/formset_errors.html', {'formset': formset}) - page = request.POST.get('page') - if not page: - page = '1' - return HttpResponseRedirect(reverse('expert_report_annotation') + '?page='+page+'&tasks_per_page='+tasks_per_page+'¬e_language=' + note_language + '&loc=' + loc + '&scroll_position='+scroll_position+(('&pending='+pending) if pending else '') + (('&checked='+checked) if checked else '') + (('&final_status='+final_status) if final_status else '') + (('&version_uuid='+version_uuid) if version_uuid else '') + (('&linked_id='+linked_id) if linked_id else '') + (('&orderby='+orderby) if orderby else '') + (('&tiger_certainty='+tiger_certainty) if tiger_certainty else '') + (('&site_certainty='+site_certainty) if site_certainty else '') + (('&status='+status) if status else '') + (('&load_new_reports='+load_new_reports) if load_new_reports else '') + (('&tags_filter=' + urllib.parse.quote_plus(tags_filter)) if tags_filter else '')) - else: - tasks_per_page = request.GET.get('tasks_per_page', tasks_per_page) - note_language = request.GET.get('note_language', note_language) - scroll_position = request.GET.get('scroll_position', scroll_position) - orderby = request.GET.get('orderby', orderby) - tiger_certainty = request.GET.get('tiger_certainty', tiger_certainty) - site_certainty = request.GET.get('site_certainty', site_certainty) - pending = request.GET.get('pending', pending) - status = request.GET.get('status', status) - final_status = request.GET.get('final_status', final_status) - version_uuid = request.GET.get('version_uuid', version_uuid) - linked_id = request.GET.get('linked_id', linked_id) - ns_exec = request.GET.get('ns_exec', ns_exec) - tags_filter = request.GET.get('tags_filter', tags_filter) - checked = request.GET.get('checked', checked) - loc = request.GET.get('loc', loc) - load_new_reports = request.GET.get('load_new_reports', load_new_reports) - edit_mode = request.GET.get('edit_mode', edit_mode) - - #current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).count() - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() - #my_reports = ExpertReportAnnotation.objects.filter(user=this_user).values('report') - my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - - ''' - if loc == 'spain': - all_annotations = all_annotations.filter(Q(report__country__isnull=True) | Q(report__country__gid=17)) - elif loc == 'europe': - all_annotations = all_annotations.filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)) - else: - pass - ''' - if loc == 'spain': - hidden_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=-1).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).values_list('report', flat=True).distinct() - flagged_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=0).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).exclude(report__version_UUID__in=hidden_final_reports_superexpert).values_list('report', flat=True).distinct() - public_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=1).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).exclude(report__version_UUID__in=hidden_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).values_list('report', flat=True).distinct() - hidden_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True, status=-1).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).values_list('report', flat=True).distinct() - flagged_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True, status=0).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).values_list('report', flat=True).distinct() - public_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True, status=1).exclude(report__version_UUID__in=flagged_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).values_list('report', flat=True).distinct() - elif loc == 'europe': - hidden_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=-1).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).values_list('report', flat=True).distinct() - flagged_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=0).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).exclude(report__version_UUID__in=hidden_final_reports_superexpert).values_list('report', flat=True).distinct() - public_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=1).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).exclude(report__version_UUID__in=hidden_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).values_list('report', flat=True).distinct() - hidden_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=-1).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).values_list('report', flat=True).distinct() - flagged_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=0).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).values_list('report', flat=True).distinct() - public_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=1).exclude(report__version_UUID__in=flagged_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).values_list('report', flat=True).distinct() - else: - hidden_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=-1).values_list('report', flat=True).distinct() - flagged_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=0).exclude(report__version_UUID__in=hidden_final_reports_superexpert).values_list('report', flat=True).distinct() - public_final_reports_superexpert = ExpertReportAnnotation.objects.filter(user__groups__name='superexpert', validation_complete=True,revise=True, status=1).exclude(report__version_UUID__in=hidden_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).values_list('report', flat=True).distinct() - hidden_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=-1).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=flagged_final_reports_superexpert).values_list('report', flat=True).distinct() - flagged_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=0).exclude(report__version_UUID__in=public_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).values_list('report', flat=True).distinct() - public_final_reports_expert = ExpertReportAnnotation.objects.filter(user__groups__name='expert', validation_complete=True,status=1).exclude(report__version_UUID__in=flagged_final_reports_superexpert).exclude(report__version_UUID__in=hidden_final_reports_superexpert).values_list('report', flat=True).distinct() + this_user_is_expert = this_user.userstat.is_expert() + this_user_is_superexpert = this_user.userstat.is_superexpert() + + this_user_is_reritja = this_user.id == 25 + + if not (this_user_is_expert or this_user_is_superexpert): + return HttpResponse("You need to be logged in as an expert member to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + + args = {} + args.update(csrf(request)) + args['scroll_position'] = scroll_position - # hidden_final_reports = set(list(hidden_final_reports_superexpert) + list(hidden_final_reports_expert)) - # flagged_final_reports = set(list(flagged_final_reports_superexpert) + list(flagged_final_reports_expert)) - # public_final_reports = set(list(public_final_reports_superexpert) + list(public_final_reports_expert)) - hidden_final_reports = hidden_final_reports_superexpert | hidden_final_reports_expert - flagged_final_reports = flagged_final_reports_superexpert | flagged_final_reports_expert - public_final_reports = public_final_reports_superexpert | public_final_reports_expert + if this_user_is_superexpert: + AnnotationFormset = modelformset_factory(ExpertReportAnnotation, form=SuperExpertReportAnnotationForm, extra=0, can_order=False) + else: + AnnotationFormset = modelformset_factory(ExpertReportAnnotation, form=ExpertReportAnnotationForm, extra=0, can_order=False) - if load_new_reports == 'T': - assign_reports(this_user) - elif this_user_is_superexpert: - assign_reports(this_user) + if request.method == 'POST': + scroll_position = request.POST.get("scroll_position", '0') + orderby = request.POST.get('orderby', orderby) + tiger_certainty = request.POST.get('tiger_certainty', tiger_certainty) + site_certainty = request.POST.get('site_certainty', site_certainty) + pending = request.POST.get('pending', pending) + status = request.POST.get('status', status) + final_status = request.POST.get('final_status', final_status) + version_uuid = request.POST.get('version_uuid', version_uuid) + linked_id = request.POST.get('linked_id', linked_id) + ns_exec = request.POST.get('ns_exec', ns_exec) + tags_filter = request.POST.get('tags_filter', tags_filter) + checked = request.POST.get('checked', checked) + loc = request.POST.get('loc', loc) + tasks_per_page = request.POST.get('tasks_per_page', tasks_per_page) + note_language = request.GET.get('note_language', "es") + load_new_reports = request.POST.get('load_new_reports', load_new_reports) + save_formset = request.POST.get('save_formset', "F") + if save_formset == "T": + formset = AnnotationFormset(request.POST) + if formset.is_valid(): + for f in formset: + one_form = f.save(commit=False) + auto_flag = must_be_autoflagged(one_form,one_form.validation_complete) + if auto_flag: + one_form.status = 0 + one_form.save() + f.save_m2m() + if(this_user_is_reritja and one_form.validation_complete == True): + send_finished_validation_notification(one_form) + if auto_flag: + autoflag_others(one_form.id) + else: + return render(request, 'tigacrafting/formset_errors.html', {'formset': formset}) + page = request.POST.get('page') + if not page: + page = '1' + return HttpResponseRedirect(reverse('expert_report_annotation') + '?page='+page+'&tasks_per_page='+tasks_per_page+'¬e_language=' + note_language + '&loc=' + loc + '&scroll_position='+scroll_position+(('&pending='+pending) if pending else '') + (('&checked='+checked) if checked else '') + (('&final_status='+final_status) if final_status else '') + (('&version_uuid='+version_uuid) if version_uuid else '') + (('&linked_id='+linked_id) if linked_id else '') + (('&orderby='+orderby) if orderby else '') + (('&tiger_certainty='+tiger_certainty) if tiger_certainty else '') + (('&site_certainty='+site_certainty) if site_certainty else '') + (('&status='+status) if status else '') + (('&load_new_reports='+load_new_reports) if load_new_reports else '') + (('&tags_filter=' + urllib.parse.quote_plus(tags_filter)) if tags_filter else '')) + else: + tasks_per_page = request.GET.get('tasks_per_page', tasks_per_page) + note_language = request.GET.get('note_language', note_language) + scroll_position = request.GET.get('scroll_position', scroll_position) + orderby = request.GET.get('orderby', orderby) + tiger_certainty = request.GET.get('tiger_certainty', tiger_certainty) + site_certainty = request.GET.get('site_certainty', site_certainty) + pending = request.GET.get('pending', pending) + status = request.GET.get('status', status) + final_status = request.GET.get('final_status', final_status) + version_uuid = request.GET.get('version_uuid', version_uuid) + linked_id = request.GET.get('linked_id', linked_id) + ns_exec = request.GET.get('ns_exec', ns_exec) + tags_filter = request.GET.get('tags_filter', tags_filter) + checked = request.GET.get('checked', checked) + loc = request.GET.get('loc', loc) + load_new_reports = request.GET.get('load_new_reports', load_new_reports) + edit_mode = request.GET.get('edit_mode', edit_mode) + + # TODO: filter by adults? + report_qs = Report.objects.filter( + type=Report.TYPE_ADULT + ) + spain_country = EuropeCountry.objects.spain() + if loc == 'spain': + report_qs = report_qs.filter(country=spain_country) + elif loc == 'europe': + report_qs = report_qs.filter( + country__isnull=False + ).exclude( + country=spain_country + ) - all_annotations = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult') + public_final_reports = report_qs.filter_by_status(status=ExpertReportAnnotation.STATUS_PUBLIC) + flagged_final_reports = report_qs.filter_by_status(status=ExpertReportAnnotation.STATUS_FLAGGED) + hidden_final_reports = report_qs.filter_by_status(status=ExpertReportAnnotation.STATUS_HIDDEN) + if load_new_reports == 'T' or this_user_is_superexpert: + this_user.userstat.assign_reports() - my_linked_ids = all_annotations.values('linked_id').distinct() + user_assigned_reports = report_qs.filter(expert_report_annotations__user=this_user) if this_user_is_expert: if (version_uuid == 'na' and linked_id == 'na' and tags_filter == 'na') and (not pending or pending == 'na'): pending = 'pending' + if this_user_is_superexpert: if (version_uuid == 'na' and linked_id == 'na' and tags_filter == 'na') and (not final_status or final_status == 'na'): final_status = 'public' if (version_uuid == 'na' and linked_id == 'na' and tags_filter == 'na') and (not checked or checked == 'na'): checked = 'unchecked' - n_flagged = all_annotations.filter(report__in=flagged_final_reports).count() - n_hidden = all_annotations.filter(report__in=hidden_final_reports).count() - n_public = all_annotations.filter(report__in=public_final_reports).exclude(report__in=flagged_final_reports).exclude(report__in=hidden_final_reports).count() - - if loc == 'spain': - n_unchecked = all_annotations.filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).filter(validation_complete=False).count() - n_confirmed = all_annotations.filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).filter(validation_complete=True, revise=False).count() - n_revised = all_annotations.filter(Q(report__country__isnull=True) | Q(report__country__gid=17)).filter(validation_complete=True, revise=True).count() - elif loc == 'europe': - n_unchecked = all_annotations.filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).filter(validation_complete=False).count() - n_confirmed = all_annotations.filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).filter(validation_complete=True, revise=False).count() - n_revised = all_annotations.filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17)).filter(validation_complete=True, revise=True).count() - else: - n_unchecked = all_annotations.filter(validation_complete=False).count() - n_confirmed = all_annotations.filter(validation_complete=True, revise=False).count() - n_revised = all_annotations.filter(validation_complete=True, revise=True).count() - - - n_spain = all_annotations.filter( Q(report__country__isnull=True) | Q(report__country__gid=17) ).count() - n_europe = all_annotations.filter(Q(report__country__isnull=False) & ~Q(report__country__gid=17) ).count() - - args['n_flagged'] = n_flagged - args['n_hidden'] = n_hidden - args['n_public'] = n_public - args['n_unchecked'] = n_unchecked - args['n_confirmed'] = n_confirmed - args['n_revised'] = n_revised - args['n_loc_spain'] = n_spain - args['n_loc_europe'] = n_europe - args['n_loc_all'] = n_spain + n_europe + # NOTE: dict where status is key and count is value + report_status_count_qs = user_assigned_reports.annotate_final_status().values('final_status').annotate(count=models.Count(1)) + report_status_count = { + item['final_status']: item['count'] + for item in report_status_count_qs + } + args['n_flagged'] = report_status_count.get(ExpertReportAnnotation.STATUS_FLAGGED, 0) + args['n_hidden'] = report_status_count.get(ExpertReportAnnotation.STATUS_HIDDEN, 0) + args['n_public'] = report_status_count.get(ExpertReportAnnotation.STATUS_PUBLIC, 0) + + report_user_stats_qs = user_assigned_reports.values( + 'expert_report_annotations__validation_complete', + 'expert_report_annotations__revise' + ).annotate( + validation_complete=models.F('expert_report_annotations__validation_complete'), + revise=models.F('expert_report_annotations__revise'), + total_count=models.Count(1) + ).values('validation_complete', 'revise', 'total_count') + report_user_stats = list(report_user_stats_qs) + + args['n_unchecked'] = sum(item['total_count'] for item in report_user_stats if item['validation_complete'] is False) + args['n_confirmed'] = sum(item['total_count'] for item in report_user_stats if item['validation_complete'] is True and item['revise'] is False) + args['n_revised'] = sum(item['total_count'] for item in report_user_stats if item['validation_complete'] is True and item['revise'] is True) + + args['n_loc_spain'] = user_assigned_reports.filter(country=spain_country).count() + args['n_loc_europe'] = user_assigned_reports.filter(country__isnull=False).exclude(country=spain_country).count() + args['n_loc_all'] = user_assigned_reports.count() + + + all_annotations = this_user.expert_report_annotations.all() if version_uuid and version_uuid != 'na': - all_annotations = all_annotations.filter(report__version_UUID=version_uuid) + all_annotations = all_annotations.filter(report__pk=version_uuid) if linked_id and linked_id != 'na': all_annotations = all_annotations.filter(linked_id=linked_id) if tags_filter and tags_filter != 'na' and tags_filter!='': @@ -787,9 +716,8 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n # we must go up to Report to filter tags, because you don't want to filter only your own tags but the tag that # any expert has put on the report # these are all (not only yours, but also) the reports that contain the filtered tag - everyones_tagged_reports = ExpertReportAnnotation.objects.filter(tags__name__in=tags_array).values('report').distinct() # we want the annotations of the reports which contain the tag(s) - all_annotations = all_annotations.filter(report__in=everyones_tagged_reports) + all_annotations = all_annotations.filter(tags__name__in=tags_array) if (not version_uuid or version_uuid == 'na') and (not linked_id or linked_id == 'na') and (not tags_filter or tags_filter == 'na' or tags_filter==''): if year and year != 'all': try: @@ -800,7 +728,6 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n if tiger_certainty and tiger_certainty != 'all': try: this_certainty = int(tiger_certainty) - #all_annotations = all_annotations.filter(tiger_certainty_category=this_certainty) all_annotations = all_annotations.filter(category__id=this_certainty) except ValueError: pass @@ -813,8 +740,7 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n if ns_exec and ns_exec != 'all': try: this_exec = int(ns_exec) - annotated_by_exec = ExpertReportAnnotation.objects.filter(validation_complete_executive=True).filter(user_id=this_exec).values('report').distinct() - all_annotations = all_annotations.filter(report_id__in=annotated_by_exec) + all_annotations = all_annotations.filter(validation_complete_executive=True, user_id=this_exec) except ValueError: pass @@ -823,8 +749,7 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n elif pending == 'pending': all_annotations = all_annotations.filter(validation_complete=False) elif pending == 'favorite': - my_favorites = FavoritedReports.objects.filter(user=this_user).values('report__version_UUID') - all_annotations = all_annotations.filter(report__version_UUID__in=my_favorites) + all_annotations = all_annotations.filter(report__favorites__user=this_user) if status == "flagged": all_annotations = all_annotations.filter(status=0) elif status == "hidden": @@ -840,49 +765,35 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n elif checked == "revised": all_annotations = all_annotations.filter(validation_complete=True, revise=True) elif checked == "favorite": - my_favorites = FavoritedReports.objects.filter(user=this_user).values('report__version_UUID') - all_annotations = all_annotations.filter(report__version_UUID__in=my_favorites) + all_annotations = all_annotations.filter(report__favorites__user=this_user) if final_status == "flagged": all_annotations = all_annotations.filter(report__in=flagged_final_reports) elif final_status == "hidden": all_annotations = all_annotations.filter(report__in=hidden_final_reports) elif final_status == "public": - all_annotations = all_annotations.filter(report__in=public_final_reports).exclude(report__in=flagged_final_reports).exclude(report__in=hidden_final_reports) + all_annotations = all_annotations.filter(report__in=public_final_reports) if loc == 'spain': - all_annotations = all_annotations.filter( Q(report__country__isnull=True) | Q(report__country__gid=17) ) + all_annotations = all_annotations.filter(report__country=spain_country) elif loc == 'europe': - all_annotations = all_annotations.filter( Q(report__country__isnull=False) & ~Q(report__country__gid=17) ) - else: - pass + all_annotations = all_annotations.filter(report__country__isnull=False).exclude(report__country=spain_country) + + all_annotations = all_annotations.order_by('-report__server_upload_time') + if orderby == "tiger_score": + all_annotations = all_annotations.order_by('category__name') - if all_annotations: - all_annotations = all_annotations.order_by('-report__creation_time') - if orderby == "tiger_score": - all_annotations = all_annotations.order_by('category__name') - # if orderby == "site_score": - # all_annotations = all_annotations.order_by('site_certainty_category') - # elif orderby == "tiger_score": - # all_annotations = all_annotations.order_by('tiger_certainty_category') paginator = Paginator(all_annotations, int(tasks_per_page)) - page = request.GET.get('page', 1) - try: - objects = paginator.page(page) - except PageNotAnInteger: - objects = paginator.page(1) - except EmptyPage: - objects = paginator.page(paginator.num_pages) - page_query = all_annotations.filter(id__in=[object.id for object in objects]) - this_formset = AnnotationFormset(queryset=page_query) + page_num = request.GET.get('page', 1) + page_obj = paginator.get_page(page_num) + + this_formset = AnnotationFormset(queryset=page_obj.object_list) args['formset'] = this_formset - args['objects'] = objects - args['pages'] = range(1, objects.paginator.num_pages+1) - #current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).count() - current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() + args['objects'] = page_obj + args['pages'] = paginator.page_range + current_pending = this_user.userstat.num_pending_annotations args['n_pending'] = current_pending - #n_complete = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=True).count() - n_complete = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=True).filter(report__type='adult').count() + n_complete = this_user.userstat.num_completed_annotations args['n_complete'] = n_complete n_favorites = FavoritedReports.objects.filter(user=this_user).count() args['n_favorites'] = n_favorites @@ -920,8 +831,7 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n else: args['exec_validated_label'] = 'N/A' args['tags_filter'] = tags_filter - #args['my_version_uuids'] = my_version_uuids - args['my_linked_ids'] = my_linked_ids + args['my_linked_ids'] = this_user.expert_report_annotations.filter(report__type='adult').values('linked_id').distinct() args['tasks_per_page'] = tasks_per_page args['note_language'] = note_language args['scroll_position'] = scroll_position @@ -941,8 +851,6 @@ def expert_report_annotation(request, scroll_position='', tasks_per_page='10', n args['ns_list'] = User.objects.filter(userstat__national_supervisor_of__isnull=False).order_by('userstat__national_supervisor_of__name_engl') return render(request, 'tigacrafting/expert_report_annotation.html' if this_user_is_expert else 'tigacrafting/superexpert_report_annotation.html', args) - else: - return HttpResponse("You need to be logged in as an expert member to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") @login_required def single_report_view(request,version_uuid): @@ -966,34 +874,30 @@ def expert_report_status(request, reports_per_page=10, version_uuid=None, linked if this_user.groups.filter(Q(name='superexpert') | Q(name='movelab')).exists(): version_uuid = request.GET.get('version_uuid', version_uuid) reports_per_page = request.GET.get('reports_per_page', reports_per_page) - #all_reports_version_uuids = Report.objects.filter(type__in=['adult', 'site']).values('version_UUID') - #all_reports_version_uuids = Report.objects.filter(type='adult').values('version_UUID') - #these_reports = Report.objects.exclude(creation_time__year=2014).exclude(hide=True).exclude(photos__isnull=True).filter(type__in=['adult', 'site']) - these_reports = Report.objects.exclude(creation_time__year=2014).exclude(hide=True).exclude(photos__isnull=True).filter(type='adult').order_by('-creation_time') + if version_uuid and version_uuid != 'na': - reports = Report.objects.filter(version_UUID=version_uuid) + reports = Report.objects.filter(pk=version_uuid) n_reports = 1 elif linked_id and linked_id != 'na': reports = Report.objects.filter(linked_id=linked_id) n_reports = 1 else: - #reports = list(filter_reports(these_reports, sort=False)) - reports = list(fast_filter_reports(these_reports)) + reports = Report.objects.queueable().filter(type=Report.TYPE_ADULT).order_by('-server_upload_time') n_reports = len(reports) + paginator = Paginator(reports, int(reports_per_page)) - page = request.GET.get('page', 1) - try: - objects = paginator.page(page) - except PageNotAnInteger: - objects = paginator.page(1) - except EmptyPage: - objects = paginator.page(paginator.num_pages) - paged_reports = Report.objects.filter(version_UUID__in=[object.version_UUID for object in objects]).order_by('-creation_time') + page_num = request.GET.get('page', 1) + page_obj = paginator.get_page(page_num) + reports_per_page_choices = range(0, min(1000, n_reports)+1, 25) - #context = {'reports': paged_reports, 'all_reports_version_uuids': all_reports_version_uuids, 'version_uuid': version_uuid, 'reports_per_page_choices': reports_per_page_choices} - context = {'reports': paged_reports, 'version_uuid': version_uuid, 'reports_per_page_choices': reports_per_page_choices} - context['objects'] = objects - context['pages'] = range(1, objects.paginator.num_pages+1) + + context = { + 'reports': page_obj, + 'version_uuid': version_uuid, + 'reports_per_page_choices': reports_per_page_choices, + 'objects': page_obj, + 'pages': range(1, paginator.num_pages+1) + } return render(request, 'tigacrafting/expert_report_status.html', context) else: @@ -1074,45 +978,23 @@ def expert_report_complete(request): return Response(context) def get_reports_unfiltered_sites_embornal(reports_imbornal): - return Report.objects.filter( + return Report.objects.queueable().unassigned().filter( type=Report.TYPE_SITE, - breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN, - creation_time__year__gt=2014, - ).exclude( - note__icontains='#345' - ).has_photos().annotate( - n_annotations=Count('expert_report_annotations') - ).filter( - n_annotations=0 + breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN ).order_by('-server_upload_time') def get_reports_unfiltered_sites_other(reports_imbornal): - return Report.objects.filter( - type=Report.TYPE_SITE, - creation_time__year__gt=2014, + return Report.objects.queueable().unassigned().filter( + type=Report.TYPE_SITE ).exclude( - models.Q(note__icontains='#345') | - models.Q(breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN) - ).has_photos().annotate( - n_annotations=Count('expert_report_annotations') - ).filter( - n_annotations=0 + breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN ).order_by('-server_upload_time') def get_reports_imbornal(): return Report.objects.filter(breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN) def get_reports_unfiltered_adults(): - return Report.objects.filter( - creation_time__year__gt=2014, - type=Report.TYPE_ADULT - ).exclude( - note__icontains='#345' - ).has_photos().annotate( - n_annotations=Count('expert_report_annotations') - ).filter( - n_annotations__lt=3 - ).order_by('-server_upload_time') + return Report.objects.filter(type=Report.TYPE_ADULT).in_progress().order_by('-server_upload_time') def auto_annotate_notsure(report: Report) -> None: auto_annotate( @@ -1187,36 +1069,39 @@ def picture_validation(request,tasks_per_page='300',visibility='visible', usr_no this_user = request.user this_user_is_coarse = this_user.groups.filter(name='coarse_filter').exists() super_movelab = User.objects.get(pk=24) - if this_user_is_coarse: - args = {} - args.update(csrf(request)) - PictureValidationFormSet = modelformset_factory(Report, form=PhotoGrid, extra=0, can_order=True) - if request.method == 'POST': - save_formset = request.POST.get('save_formset', "F") - tasks_per_page = request.POST.get('tasks_per_page', tasks_per_page) - if save_formset == "T": - formset = PictureValidationFormSet(request.POST) - if formset.is_valid(): - for f in formset: - report = f.save(commit=False) - #check that the report hasn't been assigned to anyone before saving, as a precaution to not hide assigned reports - who_has = report.get_who_has() - if who_has == '': - report.save() + + if not this_user_is_coarse: + return HttpResponse("You need to be logged in as an expert member to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + + args = {} + args.update(csrf(request)) + PictureValidationFormSet = modelformset_factory(Report, form=PhotoGrid, extra=0, can_order=False) + if request.method == 'POST': + save_formset = request.POST.get('save_formset', "F") + tasks_per_page = request.POST.get('tasks_per_page', tasks_per_page) + if save_formset == "T": + formset = PictureValidationFormSet(request.POST) + if formset.is_valid(): + for f in formset: + report = f.save(commit=False) + #check that the report hasn't been assigned to anyone before saving, as a precaution to not hide assigned reports + who_has = report.get_who_has() + if who_has == '': + report.save() ###############-------------------------------- FastUpload --------------------------------############### - #print(f.cleaned_data) - if f.cleaned_data['fastUpload']: - #check that annotation does not exist, to avoid duplicates - if not ExpertReportAnnotation.objects.filter(report=report).filter(user=super_movelab).exists(): - new_annotation = ExpertReportAnnotation(report=report, user=super_movelab) - photo = report.photos.first() - new_annotation.site_certainty_notes = 'auto' - new_annotation.best_photo_id = photo.id - new_annotation.validation_complete = True - new_annotation.revise = True - new_annotation.save() + #print(f.cleaned_data) + if f.cleaned_data['fastUpload']: + #check that annotation does not exist, to avoid duplicates + if not ExpertReportAnnotation.objects.filter(report=report).filter(user=super_movelab).exists(): + new_annotation = ExpertReportAnnotation(report=report, user=super_movelab) + photo = report.photos.first() + new_annotation.site_certainty_notes = 'auto' + new_annotation.best_photo_id = photo.id + new_annotation.validation_complete = True + new_annotation.revise = True + new_annotation.save() ###############------------------------------ FI FastUpload --------------------------------############### if f.cleaned_data['other_species']: @@ -1230,121 +1115,106 @@ def picture_validation(request,tasks_per_page='300',visibility='visible', usr_no if f.cleaned_data['not_sure']: auto_annotate_notsure(report) - - page = request.POST.get('page') - visibility = request.POST.get('visibility') - usr_note = request.POST.get('usr_note') - type = request.POST.get('type', type) - country = request.POST.get('country', country) - aithr = request.POST.get('aithr', aithr) - if aithr == '': - aithr = '0.75' - - if not page: - page = request.GET.get('page',"1") - return HttpResponseRedirect(reverse('picture_validation') + '?page=' + page + '&tasks_per_page='+tasks_per_page + '&visibility=' + visibility + '&usr_note=' + urllib.parse.quote_plus(usr_note) + '&type=' + type + '&country=' + country + '&aithr=' + aithr) - else: - tasks_per_page = request.GET.get('tasks_per_page', tasks_per_page) - type = request.GET.get('type', type) - country = request.GET.get('country', country) - visibility = request.GET.get('visibility', visibility) - usr_note = request.GET.get('usr_note', usr_note) - aithr = request.GET.get('aithr', aithr) - if aithr == '': - aithr = '0.75' - - new_reports_unfiltered_qs = Report.objects.filter( - creation_time__year__gt=2014, + page = request.POST.get('page') + visibility = request.POST.get('visibility') + usr_note = request.POST.get('usr_note') + type = request.POST.get('type', type) + country = request.POST.get('country', country) + aithr = request.POST.get('aithr', aithr) + if aithr == '': + aithr = '0.75' + + if not page: + page = request.GET.get('page',"1") + return HttpResponseRedirect(reverse('picture_validation') + '?page=' + page + '&tasks_per_page='+tasks_per_page + '&visibility=' + visibility + '&usr_note=' + urllib.parse.quote_plus(usr_note) + '&type=' + type + '&country=' + country + '&aithr=' + aithr) + else: + tasks_per_page = request.GET.get('tasks_per_page', tasks_per_page) + type = request.GET.get('type', type) + country = request.GET.get('country', country) + visibility = request.GET.get('visibility', visibility) + usr_note = request.GET.get('usr_note', usr_note) + aithr = request.GET.get('aithr', aithr) + if aithr == '': + aithr = '0.75' + + reports_qs = Report.objects.queueable().unassigned().order_by('-server_upload_time') + + if type == 'adult': + type_readable = "Adults" + reports_qs = reports_qs.filter( + type=Report.TYPE_ADULT + ) + elif type == 'site': + type_readable = "Breeding sites - Storm drains" + reports_qs = reports_qs.filter(type=Report.TYPE_SITE).filter( + type=Report.TYPE_SITE, + breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN + ) + elif type == 'site-o': + type_readable = "Breeding sites - Other" + reports_qs = reports_qs.filter( + type=Report.TYPE_SITE ).exclude( - note__icontains='#345' - ).non_deleted().has_photos().annotate( - n_annotations=Count('expert_report_annotations') - ).filter( - n_annotations=0 + breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN ) + elif type == 'all': + type_readable = "All" - if type == 'adult': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter( - type=Report.TYPE_ADULT, - ia_filter_1__lte=float(aithr) - ) - elif type == 'site': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter( - type=Report.TYPE_SITE, - breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN - ) - elif type == 'site-o': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter( - type=Report.TYPE_SITE - ).exclude( - breeding_site_type=Report.BREEDING_SITE_TYPE_STORM_DRAIN - ) - - if visibility == 'visible': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter(hide=False) - elif visibility == 'hidden': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter(hide=True) + if visibility == 'visible': + reports_qs = reports_qs.filter(hide=False) + elif visibility == 'hidden': + reports_qs = reports_qs.filter(hide=True) - if usr_note and usr_note != '': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter(note__icontains=usr_note) - if country and country != '' and country != 'all': - new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter(country__gid=int(country)) + if usr_note and usr_note != '': + reports_qs = reports_qs.filter(note__icontains=usr_note) - paginator = Paginator( - new_reports_unfiltered_qs.prefetch_related('photos').select_related('country').order_by('-server_upload_time'), - int(tasks_per_page) - ) - page = request.GET.get('page', 1) - try: - objects = paginator.page(page) - except PageNotAnInteger: - objects = paginator.page(1) - except EmptyPage: - objects = paginator.page(paginator.num_pages) - page_query = objects - page_query.ordered = True - this_formset = PictureValidationFormSet(queryset=page_query) - args['formset'] = this_formset - args['objects'] = objects - args['page'] = page - args['pages'] = range(1, objects.paginator.num_pages + 1) - args['new_reports_unfiltered'] = page_query - args['tasks_per_page'] = tasks_per_page - args['aithr'] = aithr - args['visibility'] = visibility - args['usr_note'] = usr_note - args['type'] = type - args['country'] = country - country_readable = '' + if country and country != '': if country == 'all': country_readable = 'All' - elif country is not None and country != '': - try: - country_readable = EuropeCountry.objects.get(pk=int(country)).name_engl - except EuropeCountry.DoesNotExist: - pass - args['country_readable'] = country_readable - args['countries'] = EuropeCountry.objects.all().order_by('name_engl') - type_readable = '' - if type == 'site': - type_readable = "Breeding sites - Storm drains" - elif type == 'site-o': - type_readable = "Breeding sites - Other" - elif type == 'adult': - type_readable = "Adults" - elif type == 'all': - type_readable = "All" - args['type_readable'] = type_readable - #n_query_records = new_reports_unfiltered.count() - n_query_records = new_reports_unfiltered_qs.count() - args['n_query_records'] = n_query_records - #args['tasks_per_page_choices'] = range(5, min(100, n_query_records) + 1, 5) - range_list = [ n for n in range(5, 101, 5) ] - args['tasks_per_page_choices'] = range_list + [200,300] - return render(request, 'tigacrafting/photo_grid.html', args) - else: - return HttpResponse("You need to be logged in as an expert member to view this page. If you have have been recruited as an expert and have lost your log-in credentials, please contact MoveLab.") + else: + country = get_object_or_404(EuropeCountry, int(country)) + country_readable = country.name_engl + reports_qs = reports_qs.filter(country=country) + + if aithr: + reports_qs = reports_qs.filter(ia_filter_1__lte=float(aithr)) + + reports_qs = reports_qs.prefetch_related('photos').select_related('country').order_by('-server_upload_time') + paginator = Paginator( + reports_qs, + int(tasks_per_page) + ) + page_num = request.GET.get('page', 1) + + objects = paginator.get_page(page_num) + page_query = objects + page_query.ordered = True + + this_formset = PictureValidationFormSet(queryset=page_query) + args['formset'] = this_formset + args['objects'] = objects + args['page'] = page_num + args['pages'] = range(1, paginator.num_pages + 1) + args['new_reports_unfiltered'] = page_query + args['tasks_per_page'] = tasks_per_page + args['aithr'] = aithr + args['visibility'] = visibility + args['usr_note'] = usr_note + args['type'] = type + args['country'] = country + + args['country_readable'] = country_readable + + args['countries'] = EuropeCountry.objects.all().order_by('name_engl') + + args['type_readable'] = type_readable + + args['n_query_records'] = reports_qs.count() + + range_list = [ n for n in range(5, 101, 5) ] + args['tasks_per_page_choices'] = range_list + [200,300] + return render(request, 'tigacrafting/photo_grid.html', args) @login_required def aimalog(request): diff --git a/tigaserver_app/managers.py b/tigaserver_app/managers.py index 92e96667..b3f4c2a9 100644 --- a/tigaserver_app/managers.py +++ b/tigaserver_app/managers.py @@ -1,7 +1,18 @@ +from datetime import timedelta +from typing import Tuple, Optional + +from django.conf import settings +from django.contrib.auth.models import User from django.db import models +from django.db.models import fields +from django.db.models.functions import Coalesce +from django.db.models.query import QuerySet +from django.utils import timezone from fcm_django.models import FCMDeviceQuerySet, FCMDeviceManager +from tigacrafting.models import ExpertReportAnnotation, UserStat + class ReportQuerySet(models.QuerySet): def has_photos(self, state: bool = True): from .models import Photo @@ -25,6 +36,298 @@ def non_deleted(self): def published(self, state: bool = True): return self.non_deleted().filter(map_aux_report__isnull=not state) + def with_finished_validation(self, state: bool = True) -> QuerySet: + # All assignations (maximum) has validated. + subquery = ExpertReportAnnotation.objects.filter( + validation_complete=True + ).values('report').annotate( + count=models.Count(1) + ).filter( + models.Q(count__gte=settings.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT) | + models.Q(validation_complete_executive=True), + ).filter(report=models.OuterRef('pk')) + + return self.annotate( + has_finished_validation=models.Exists(subquery.values('pk')) + ).filter( + has_finished_validation=state + ) + + def annotate_final_status(self): + return self.annotate( + superexpert_status=models.Subquery( + ExpertReportAnnotation.objects.filter( + report=models.OuterRef('pk'), + user__groups__name='superexpert', + validation_complete=True, + revise=True + ).order_by('-last_modified').values('status')[:1], + output_field=models.IntegerField() + ), + worst_expert_status=models.Subquery( + ExpertReportAnnotation.objects.filter( + report=models.OuterRef('pk'), + user__groups__name='expert', + validation_complete=True, + ).values('status').annotate( + worst_status=models.Case( + models.When( + validation_complete_executive=True, then=models.F('status') + ), + default=models.Min('status') + ) + ).values('worst_status')[:1], + output_field=models.IntegerField() + ), + final_status=models.Case( + models.When( + models.Q( + superexpert_status__isnull=False, + ), + then=models.F('superexpert_status') + ), + default=models.F("worst_expert_status"), + output_field=models.IntegerField() + ) + ) + + def filter_by_status(self, status): + return self.annotate_final_status().filter( + final_status=status + ) + + def with_blocked_validations(self, days=settings.ENTOLAB_LOCK_PERIOD): + return self.filter( + pk__in=ExpertReportAnnotation.objects.blocked(days=days).values('report') + ) + + def unassigned(self, state: bool = True) -> QuerySet: + # With no assignations at all. + return self.annotate( + has_annotations=models.Exists( + ExpertReportAnnotation.objects.filter( + report=models.OuterRef('pk') + ).values('pk') + ) + ).filter( + has_annotations=not state + ) + + def with_pending_validation_to_finish(self) -> QuerySet: + return self.with_max_annotations(state=True).with_finished_validation(state=False) + + def in_progress(self): + return self.queued().unassigned(state=False) + + def _filter_and_prioritize(self, user: Optional[User] = None) -> QuerySet: + from .models import EuropeCountry + + qs = self + + if not user: + # Nothing extra to prioritze. Return as is. + return qs + + # Remove reports already asigned. + qs = qs.exclude( + expert_report_annotations__user=user + ) + + supervised_countries = UserStat.objects.filter( + national_supervisor_of__isnull=False, + ).values('national_supervisor_of').distinct() + qs = qs.annotate( + # NOTE: if ever need to order_by country count. + # country_count=models.Window( + # expression=models.Count(1), + # partition_by=['country'] + # ), + in_supervised_country=models.Case( + models.When(country__in=supervised_countries, then=True), + default=False, + output_field=models.BooleanField() + ) + ).order_by("-in_supervised_country", "-server_upload_time") + + if user.userstat.is_superexpert(): + # Nothing extra to prioritze. Return as is. + return qs + + # Start pioritization: + # Summary: + # - Case user in bounding box: + # 1. Report.country is the bounding box + # - Other (expert and national supervisor): + # 1. Report.country is not bounding box + # 2. Report is not in the exclusivity period (except for its country if user is national supervisor) + # 3. Sort by: + # European user: + # 1. Report is in europe + # 2. Report is user's country + # 3. Append default ordering + # Spanish user: + # 1. Report is in spain and nuts2 region + # 2. Report is in spain + # 3. Report is in europe + # 4. Append default ordering + if user.userstat.is_bb_user(): + # Case user in a bounding box + qs = qs.filter( + country=user.userstat.native_of + ) + else: + # Case regular user (expert and national supervisors) + qs = qs.filter( + country__is_bounding_box=False + ) + + # Reports outside the exclusivity period + qs = qs.in_supervisor_exclusivity_period( + state=False, + for_user=user + ) + + # Prioritize for user + if user.groups.filter(name='eu_group_europe').exists(): + # Case European user: Prioritize reports from own country + qs = qs.annotate( + in_user_country=models.Case( + models.When( + country=user.userstat.national_supervisor_of or user.userstat.native_of, + then=True + ), + default=False, + output_field=models.BooleanField() + ) + ).order_by( + # NOTE: appending here all the field to be ordered_by since order_by function + # can not be chained, and each order_by cal will clear any previous ordering. + *( + ('-in_user_country',) + qs.query.order_by + ) + ) + else: + # Case Non European user: Prioritize reports from nuts2 assignation, then spain, the Europe. + spain_country = EuropeCountry.objects.spain() + qs = qs.annotate( + in_spain_own_region=models.Case( + models.When( + models.Q( + country=spain_country, + nuts_2=user.userstat.nuts2_assignation.nuts_id if user.userstat.nuts2_assignation else None + ), + then=True + ), + default=False, + output_field=models.BooleanField() + ), + # NOTE: can not name "is_spain" due to conflict with a @property named the same in the model class. + in_spain=models.Case( + models.When( + country=spain_country, + then=True + ), + default=False, + output_field=models.BooleanField() + ), + in_europe=models.Case( + models.When( + country__isnull=False, + then=True + ), + default=False, + output_field=models.BooleanField() + ) + ).order_by( + # NOTE: appending here all the field to be ordered_by since order_by function + # can not be chained, and each order_by cal will clear any previous ordering. + *( + ('-in_spain_own_region', '-in_spain', '-in_europe') + qs.query.order_by + ) + ) + + return qs + + def queueable(self) -> QuerySet: + return self.non_deleted().has_photos().exclude( + note__icontains="#345", + ).filter( + server_upload_time__year__gt=2021, + hide=False + ) + + def with_max_annotations(self, state: bool = True) -> QuerySet: + # Available to assign + subquery = ExpertReportAnnotation.objects.filter( + report=models.OuterRef('pk') + ).annotate( + count=models.Count(1) + ).filter( + count__gte=settings.MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT + ) + + return self.annotate( + has_max_annotations=models.Exists(subquery) + ).filter( + has_max_annotations=state, + ) + + def queued(self, user_prioritized: Optional[User] = None) -> QuerySet: + + qs = self.queueable().filter(type=self.model.TYPE_ADULT) + + if user_prioritized and user_prioritized.userstat and user_prioritized.userstat.is_superexpert(): + # Only get reports with all annotations finished. + qs = qs.with_finished_validation(state=True) + else: + # Available to assign + qs = qs.with_finished_validation(state=False).with_max_annotations(state=False) + + return qs._filter_and_prioritize(user=user_prioritized) + + def in_supervisor_exclusivity_period(self, state: bool = True, for_user: Optional[User] = None): + # Get reports that (meet all of): + # 1. Inside the exclusivity window + # 2. Country has supervisors + # 3. Supervisor has not validated yet. + lookup = models.Q( + models.Q( + country__isnull=False, + country_has_supervisors=True, + server_upload_time__gte=models.ExpressionWrapper( + timezone.now() - timedelta(days=1) * Coalesce( + models.F('country__national_supervisor_report_expires_in'), + settings.DEFAULT_EXPIRATION_DAYS + ), + output_field=fields.DateTimeField() + ), + supervisor_has_validated=False + ), + _negated=not state + ) + + if not state and for_user: + if for_user.userstat.national_supervisor_of: + # Supervisor can bypass the exclusivity period only in its country + lookup |= models.Q(country=for_user.userstat.national_supervisor_of) + + # Using Exists subquery since it's faster than using JOIN. + supervisor_has_validated_subquery = ExpertReportAnnotation.objects.filter( + report=models.OuterRef('pk'), + validation_complete=True, + user__userstat__national_supervisor_of__isnull=False, + user__userstat__national_supervisor_of=models.OuterRef('country'), + ) + country_has_supervisors_subquery = UserStat.objects.filter( + national_supervisor_of__isnull=False, + national_supervisor_of=models.OuterRef('country') + ) + + return self.annotate( + supervisor_has_validated=models.Exists(supervisor_has_validated_subquery.values('pk')), + country_has_supervisors=models.Exists(country_has_supervisors_subquery.values('pk')) + ).filter(lookup) + ReportManager = models.Manager.from_queryset(ReportQuerySet) class PhotoQuerySet(models.QuerySet): @@ -76,4 +379,10 @@ def deactivate_devices_with_error_results(self, *args, **kwargs): class DeviceManager(FCMDeviceManager): def get_queryset(self): - return DeviceQuerySet(self.model) \ No newline at end of file + return DeviceQuerySet(self.model) + +class EuropeCountryQuerySet(models.QuerySet): + def spain(self): + return self.get(gid=17) + +EuropeCountryManager = models.Manager.from_queryset(EuropeCountryQuerySet) \ No newline at end of file diff --git a/tigaserver_app/migrations/0074_indexing_for_refactor_queues.py b/tigaserver_app/migrations/0074_indexing_for_refactor_queues.py new file mode 100644 index 00000000..b6d04571 --- /dev/null +++ b/tigaserver_app/migrations/0074_indexing_for_refactor_queues.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.7 on 2024-11-12 16:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tigaserver_app', '0073_tigauser_lastlocation'), + ] + + operations = [ + migrations.AlterField( + model_name='europecountry', + name='is_bounding_box', + field=models.BooleanField(db_index=True, default=False, help_text='If true, this geometry acts as a bounding box. The bounding boxes act as little separate entolabs, in the sense that no reports located inside a bounding box should reach an expert outside this bounding box'), + ), + migrations.AlterField( + model_name='europecountry', + name='national_supervisor_report_expires_in', + field=models.IntegerField(db_index=True, default=14, help_text='Number of days that a report in the queue is exclusively available to the nagional supervisor. For example, if the field value is 6, after report_creation_time + 6 days a report will be available to all users'), + ), + migrations.AlterField( + model_name='historicalreport', + name='type', + field=models.CharField(choices=[('bite', 'Bite'), ('adult', 'Adult'), ('site', 'Breeding Site'), ('mission', 'Mission')], db_index=True, help_text="Type of report: 'adult', 'site', or 'mission'.", max_length=7), + ), + migrations.AlterField( + model_name='report', + name='hide', + field=models.BooleanField(db_index=True, default=False, help_text='Hide this report from public views?'), + ), + migrations.AlterField( + model_name='report', + name='type', + field=models.CharField(choices=[('bite', 'Bite'), ('adult', 'Adult'), ('site', 'Breeding Site'), ('mission', 'Mission')], db_index=True, help_text="Type of report: 'adult', 'site', or 'mission'.", max_length=7), + ), + migrations.DeleteModel( + name='GlobalAssignmentStat', + ), + ] diff --git a/tigaserver_app/models.py b/tigaserver_app/models.py index ea0693fc..ac342cbd 100644 --- a/tigaserver_app/models.py +++ b/tigaserver_app/models.py @@ -39,7 +39,7 @@ from taggit.managers import TaggableManager from taggit.models import GenericUUIDTaggedItemBase, TaggedItemBase -from tigacrafting.models import MoveLabAnnotation, ExpertReportAnnotation, Categories, STATUS_CATEGORIES +from tigacrafting.models import MoveLabAnnotation, ExpertReportAnnotation, Categories import tigacrafting.html_utils as html_utils import tigaserver_project.settings as conf @@ -47,6 +47,7 @@ from .managers import ReportManager, PhotoManager, NotificationManager, DeviceManager from .messaging import send_new_award_notification from .mixins import TimeZoneModelMixin +from .managers import ReportManager, EuropeCountryManager logger_report_geolocation = logging.getLogger('mosquitoalert.location.report_location') logger_notification = logging.getLogger('mosquitoalert.notification') @@ -665,12 +666,14 @@ class EuropeCountry(models.Model): x_max = models.FloatField(blank=True, null=True) y_min = models.FloatField(blank=True, null=True) y_max = models.FloatField(blank=True, null=True) - is_bounding_box = models.BooleanField(default=False, help_text='If true, this geometry acts as a bounding box. The bounding boxes act as little separate entolabs, in the sense that no reports located inside a bounding box should reach an expert outside this bounding box') - national_supervisor_report_expires_in = models.IntegerField(default=14, help_text='Number of days that a report in the queue is exclusively available to the nagional supervisor. For example, if the field value is 6, after report_creation_time + 6 days a report will be available to all users') + is_bounding_box = models.BooleanField(default=False, db_index=True, help_text='If true, this geometry acts as a bounding box. The bounding boxes act as little separate entolabs, in the sense that no reports located inside a bounding box should reach an expert outside this bounding box') + national_supervisor_report_expires_in = models.IntegerField(default=settings.DEFAULT_EXPIRATION_DAYS, db_index=True, help_text='Number of days that a report in the queue is exclusively available to the nagional supervisor. For example, if the field value is 6, after report_creation_time + 6 days a report will be available to all users') pending_crisis_reports = models.IntegerField(blank=True, null=True, help_text='Number of reports in country assignable to non-supervisors') last_crisis_report_n_update = models.DateTimeField(help_text="Last time count was updated", null=True, blank=True) + objects = EuropeCountryManager() + class Meta: managed = True ordering = ['name_engl'] @@ -682,13 +685,6 @@ def __unicode__(self): def __str__(self): return '{} - {}'.format(self.gid, self.name_engl) -class GlobalAssignmentStat(models.Model): - country = models.OneToOneField('tigaserver_app.EuropeCountry', primary_key=True, on_delete=models.CASCADE, ) - unassigned_reports = models.IntegerField(default=0) - in_progress_reports = models.IntegerField(default=0) - pending_reports = models.IntegerField(default=0) - nsqueue_reports = models.IntegerField(default=0) - last_update = models.DateTimeField(help_text="Last time stats were updated", null=True, blank=True) class NutsEurope(models.Model): gid = models.AutoField(primary_key=True) @@ -839,11 +835,12 @@ class Report(TimeZoneModelMixin, models.Model): type = models.CharField( max_length=7, choices=TYPE_CHOICES, + db_index=True, help_text="Type of report: 'adult', 'site', or 'mission'.", ) hide = models.BooleanField( - default=False, help_text="Hide this report from public views?" + default=False, db_index=True, help_text="Hide this report from public views?" ) location_choice = models.CharField( @@ -1193,7 +1190,7 @@ def deleted(self) -> bool: @property def is_spain(self) -> bool: - return self.country is None or self.country.gid == 17 + return self.country == EuropeCountry.objects.spain() @property def language(self) -> str: @@ -2630,7 +2627,7 @@ def get_final_expert_status(self): return 1 def get_final_expert_status_text(self): - return dict(STATUS_CATEGORIES)[self.get_final_expert_status()] + return dict(ExpertReportAnnotation.STATUS_CATEGORIES)[self.get_final_expert_status()] def get_final_expert_status_bootstrap(self): result = '' @@ -2706,12 +2703,10 @@ def get_superexpert_completed_recipients(self): def get_expert_recipients_names(self): result = [] - for ano in self.expert_report_annotations.all().select_related('user'): - if not ano.user.userstat.is_superexpert(): - if ano.user.first_name and ano.user.last_name: - result.append(ano.user.first_name + ' ' + ano.user.last_name) - else: - result.append(ano.user.username) + for ano in self.expert_report_annotations.exclude(user__groups__name='superexpert').select_related("user"): + result.append( + ano.user.get_full_name() or ano.user.username + ) return '+'.join(result) def get_who_has_bootstrap(self): diff --git a/tigaserver_app/views.py b/tigaserver_app/views.py index b5350fcb..c26c88bc 100644 --- a/tigaserver_app/views.py +++ b/tigaserver_app/views.py @@ -18,8 +18,7 @@ from operator import attrgetter from tigaserver_app.serializers import NotificationSerializer, NotificationContentSerializer, UserSerializer, ReportSerializer, MissionSerializer, PhotoSerializer, FixSerializer, ConfigurationSerializer, MapDataSerializer, SiteMapSerializer, CoverageMapSerializer, CoverageMonthMapSerializer, TagSerializer, NearbyReportSerializer, ReportIdSerializer, UserAddressSerializer, SessionSerializer, OWCampaignsSerializer, OrganizationPinsSerializer, AcknowledgedNotificationSerializer, UserSubscriptionSerializer, CoarseReportSerializer from tigaserver_app.models import Notification, NotificationContent, TigaUser, Mission, Report, Photo, Fix, Configuration, CoverageArea, CoverageAreaMonth, Session, ExpertReportAnnotation, OWCampaigns, OrganizationPin, SentNotification, AcknowledgedNotification, NotificationTopic, UserSubscription, EuropeCountry, Categories, ReportResponse, Device -from tigacrafting.models import FavoritedReports -from tigacrafting.report_queues import assign_crisis_report +from tigacrafting.models import FavoritedReports, UserStat from tigacrafting.views import auto_annotate from math import ceil from taggit.models import Tag @@ -808,12 +807,11 @@ def crisis_report_assign(request, user_id=None, country_id=None): return Response(status=status.HTTP_400_BAD_REQUEST) if country_id is None: return Response(status=status.HTTP_400_BAD_REQUEST) - user = get_object_or_404(User.objects.all(), pk=user_id) + userstat = get_object_or_404(UserStat.objects.all(), pk=user_id) country = get_object_or_404(EuropeCountry.objects.all(), pk=country_id) - retval = assign_crisis_report(user, country) - user.userstat.last_emergency_mode_grab = country - user.userstat.save() - return Response(data=retval, status=status.HTTP_200_OK) + + userstat.assign_crisis_report(country=country) + return Response(status=status.HTTP_200_OK) @api_view(['POST']) @@ -1153,13 +1151,12 @@ def token(request): @api_view(['GET']) @cache_page(60 * 5) def report_stats(request): - user_id = request.query_params.get('user_id', -1) - if user_id == -1: - r_count = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos__isnull=True).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__gte=3).count() - else: - user_reports = Report.objects.filter(user__user_UUID=user_id).filter(type='adult') - r_count = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos__isnull=True).filter(version_UUID__in=user_reports).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__gte=3).count() - content = {'report_count' : r_count} + user_id = request.query_params.get('user_id', None) + + qs = Report.objects.with_finished_validation() + if user_id: + qs = qs.filter(user=user_id) + content = {'report_count' : qs.count()} return Response(content) @@ -1582,44 +1579,20 @@ def nearby_reports_no_dwindow(request): data = cursor.fetchall() flattened_data = [element for tupl in data for element in tupl] - reports_adult = Report.objects.exclude(cached_visible=0)\ - .filter(version_UUID__in=flattened_data)\ - .exclude(creation_time__year=2014)\ - .exclude(note__icontains="#345")\ - .exclude(hide=True)\ - .exclude(photos__isnull=True)\ - .filter(type='adult')\ - .annotate(n_annotations=Count('expert_report_annotations'))\ - .filter(n_annotations__gte=3) - if user is not None: - reports_adult = reports_adult.exclude(user=user) - if show_hidden == 0: - reports_adult = reports_adult.non_deleted() - - reports_bite = Report.objects.exclude(cached_visible=0)\ - .filter(version_UUID__in=flattened_data)\ - .exclude(creation_time__year=2014)\ - .exclude(note__icontains="#345")\ - .exclude(hide=True) \ - .filter(type='bite') - if user is not None: - reports_bite = reports_bite.exclude(user=user) - if show_hidden == 0: - reports_bite = reports_bite.non_deleted() - - reports_site = Report.objects.exclude(cached_visible=0)\ - .filter(version_UUID__in=flattened_data)\ - .exclude(creation_time__year=2014)\ - .exclude(note__icontains="#345")\ - .exclude(hide=True) \ - .filter(type='site') + finished_reports_qs = Report.objects.with_finished_validation().exclude( + cached_visible=0 + ).filter(version_UUID__in=flattened_data) + if user is not None: - reports_site = reports_site.exclude(user=user) + finished_reports_qs = finished_reports_qs.exclude(user=user) if show_hidden == 0: - reports_site = reports_site.non_deleted() + finished_reports_qs = finished_reports_qs.non_deleted() - classified_reports_in_max_radius = filter(lambda x: x.show_on_map, reports_adult) + reports_adult = finished_reports_qs.filter(type=Report.TYPE_ADULT) + reports_bite = finished_reports_qs.filter(type=Report.TYPE_BITE) + reports_site = finished_reports_qs.filter(type=Report.TYPE_SITE) + classified_reports_in_max_radius = filter(lambda x: x.show_on_map, reports_adult) if user is not None: user_reports = Report.objects.filter(user=user) @@ -1684,16 +1657,13 @@ def nearby_reports_fast(request): data = cursor.fetchall() flattened_data = [element for tupl in data for element in tupl] - reports = Report.objects.exclude(cached_visible=0)\ - .filter(version_UUID__in=flattened_data)\ - .exclude(creation_time__year=2014)\ - .exclude(note__icontains="#345")\ - .exclude(hide=True)\ - .exclude(photos__isnull=True)\ - .filter(type='adult')\ - .annotate(n_annotations=Count('expert_report_annotations'))\ - .filter(n_annotations__gte=3)\ - .exclude(creation_time__lte=date_n_days_ago) + reports = Report.objects.with_finished_validation().filter( + type=Report.TYPE_ADULT, + version_UUID__in=flattened_data + ).exclude( + cached_visible=0, + server_upload_time__lte=date_n_days_ago + ) classified_reports_in_max_radius = filter(lambda x: x.simplified_annotation is not None and x.simplified_annotation['score'] > 0, reports) @@ -1813,7 +1783,10 @@ def nearby_reports(request): center_point_4326 = GEOSGeometry('SRID=4326;POINT(' + center_buffer_lon + ' ' + center_buffer_lat + ')') - all_reports = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos__isnull=True).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__gte=3).exclude(creation_time__lte=date_N_days_ago) + all_reports = Report.objects.with_finished_validation().filter( + type=Report.TYPE_ADULT, + server_upload_time__gt=date_N_days_ago + ) reports_in_max_radius = all_reports.filter(point__distance_lt=(center_point_4326,Distance(m=MAX_SEARCH_RADIUS))) classified_reports_in_max_radius = filter(lambda x: x.simplified_annotation is not None and x.simplified_annotation['score'] > 0,reports_in_max_radius) dst = distance_matrix(center_point_4326,classified_reports_in_max_radius) @@ -1842,71 +1815,59 @@ def nearby_reports(request): ''' def send_unblock_email(name, email): - lock_period = settings.ENTOLAB_LOCK_PERIOD - send_to = copy.copy(settings.ADDITIONAL_EMAIL_RECIPIENTS) - send_to.append(email) + + send_to = (settings.ADDITIONAL_EMAIL_RECIPIENTS or []) + [email] + subject = 'MOSQUITO ALERT - blocked report release warning' plaintext = get_template('tigaserver_app/report_release/report_release_template') context = { 'name': name, - 'n_days': lock_period + 'n_days': settings.ENTOLAB_LOCK_PERIOD } text_content = plaintext.render(context) email = EmailMessage(subject, text_content, to=send_to) email.send(fail_silently=True) +def delete_annotations_and_notify(annotations_qs): + recipients = [] + for obj in annotations_qs.select_related('user'): + if obj.user.email is not None and obj.user.email != '': + recipients.append(obj.user) + + obj.delete() + + for r in set(recipients): + name = r.first_name if r.first_name != '' else r.username + email = r.email + send_unblock_email(name, email) + + @api_view(['DELETE']) def clear_blocked_all(request): if request.method == 'DELETE': - lock_period = settings.ENTOLAB_LOCK_PERIOD - superexperts = User.objects.filter(groups__name='superexpert') - annos = ExpertReportAnnotation.objects.filter(validation_complete=False).exclude(user__in=superexperts).order_by('user__username', 'report') - to_delete = [] - recipients = [] - for anno in annos: - elapsed = (datetime.now(timezone.utc) - anno.created).days - if elapsed > lock_period: - to_delete.append(anno.id) - if anno.user.email is not None and anno.user.email != '': - if anno.user not in recipients: - recipients.append(anno.user) - ExpertReportAnnotation.objects.filter(id__in=to_delete).delete() - for r in recipients: - name = r.first_name if r.first_name != '' else r.username - email = r.email - send_unblock_email( name, email ) + delete_annotations_and_notify( + annotations_qs=ExpertReportAnnotation.objects.blocked() + ) + return Response(status=status.HTTP_200_OK) @api_view(['DELETE']) def clear_blocked(request, username, report=None): - lock_period = settings.ENTOLAB_LOCK_PERIOD if request.method == 'DELETE': if username is None: return Response(status=status.HTTP_400_BAD_REQUEST) - user = get_object_or_404(User.objects.all(), username=username) - actual_report = None - if report is not None: - actual_report = get_object_or_404(Report.objects.all(), pk=report) - if report is not None: - annotations_for_user = ExpertReportAnnotation.objects.filter(user=user).filter(validation_complete=False).filter(report=actual_report) - else: - annotations_for_user = ExpertReportAnnotation.objects.filter(user=user).filter(validation_complete=False) - to_delete = [] - recipients = [] - for anno in annotations_for_user: - elapsed = (datetime.now(timezone.utc) - anno.created).days - if elapsed > lock_period: - to_delete.append(anno.id) - if anno.user.email is not None and anno.user.email != '': - if anno.user not in recipients: - recipients.append(anno.user) - ExpertReportAnnotation.objects.filter(id__in=to_delete).delete() - for r in recipients: - name = r.first_name if r.first_name != '' else r.username - email = r.email - send_unblock_email( name, email ) + user = get_object_or_404(User, username=username) + + annotations_qs = user.userstat.pending_annotations.blocked() + + if report: + annotations_qs.filter(report=get_object_or_404(Report, pk=report)) + + delete_annotations_and_notify( + annotations_qs=annotations_qs + ) return Response(status=status.HTTP_200_OK) @@ -2287,6 +2248,8 @@ def annotate_coarse(request): category = get_object_or_404(Categories, pk=category_id) if validation_value == '' or not category.specify_certainty_level: validation_value = None + else: + validation_value = int(validation_value) annotation = auto_annotate(report, category, validation_value) return Response(data={'message':'success', 'opcode': 0}, status=status.HTTP_200_OK) @@ -2306,15 +2269,7 @@ def coarse_filter_reports(request): limit = request.query_params.get("limit", 300) offset = request.query_params.get("offset", 1) - new_reports_unfiltered_qs = Report.objects.filter( - creation_time__year__gt=2014, - ).exclude( - note__icontains='#345' - ).non_deleted().has_photos().annotate( - n_annotations=Count('expert_report_annotations') - ).filter( - n_annotations=0 - ) + new_reports_unfiltered_qs = Report.objects.queueable().unassigned().order_by('-server_upload_time') if type == 'adult': new_reports_unfiltered_qs = new_reports_unfiltered_qs.filter( diff --git a/tigaserver_project/settings.py b/tigaserver_project/settings.py index c75be530..47d9c03a 100644 --- a/tigaserver_project/settings.py +++ b/tigaserver_project/settings.py @@ -388,8 +388,6 @@ SITE_ID = 1 -# number of days after a report is considered blocked -ENTOLAB_LOCK_PERIOD = 14 # FCM Notification settings DRY_RUN_PUSH = True @@ -420,6 +418,23 @@ DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +# Definning Barcelona Bounding Box +BCN_BB = { + 'min_lat': 41.321049, + 'min_lon': 2.052380, + 'max_lat': 41.468609, + 'max_lon': 2.225610 +} + +# Entolab +MAX_N_OF_PENDING_REPORTS = 5 +MAX_N_OF_EXPERTS_ASSIGNED_PER_REPORT = 3 +DEFAULT_EXPIRATION_DAYS = 14 +# number of days after a report is considered blocked +ENTOLAB_LOCK_PERIOD = 14 + +ADDITIONAL_EMAIL_RECIPIENTS = [] + try: from tigaserver_project.settings_local import * except ModuleNotFoundError: diff --git a/tigaserver_project/urls.py b/tigaserver_project/urls.py index c0b50c22..8bcc0af6 100644 --- a/tigaserver_project/urls.py +++ b/tigaserver_project/urls.py @@ -170,7 +170,7 @@ path('webmap/coverage/', show_filterable_report_map, {'map_type': 'coverage'}, name='coverage_map'), path('webmap/', show_filterable_report_map, name='webmap.show_map_defaults'), path('', RedirectView.as_view(url='/static/tigapublic/spain.html#/es/', permanent=False)), - path('bcn/', show_filterable_report_map,{'min_lat': 41.321049, 'min_lon': 2.052380, 'max_lat': 41.468609, 'max_lon': 2.225610, 'min_zoom': 12,'max_zoom': 18}), + path('bcn/', show_filterable_report_map,{**settings.BCN_BB, **{'min_zoom': 12,'max_zoom': 18}}), path('single_report_map//', show_single_report_map, name='webmap.single_report'), path('single_simple//', single_report_map_simplified, name='single_report_map_simplified'), path('stats/', show_usage, name='show_usage'), diff --git a/util_scripts/check_in_progress_reports.py b/util_scripts/check_in_progress_reports.py index 13d0c2bc..501557eb 100644 --- a/util_scripts/check_in_progress_reports.py +++ b/util_scripts/check_in_progress_reports.py @@ -14,9 +14,7 @@ from django.db.models import Count from tigaserver_app.models import EuropeCountry, Report, ExpertReportAnnotation, Categories - -current_progress = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(hide=True).exclude(photos=None).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=3).exclude(n_annotations=0).order_by('-server_upload_time') -reports_filtered = current_progress.non_deleted() +current_progress = Report.objects.filter(type=Report.TYPE_ADULT).in_progress().order_by('-server_upload_time') for c in current_progress: country = 'None' if c.country is not None: diff --git a/util_scripts/free_reports.py b/util_scripts/free_reports.py index bc5101ac..2d5d6808 100644 --- a/util_scripts/free_reports.py +++ b/util_scripts/free_reports.py @@ -16,22 +16,18 @@ from django.db.models import Count -def report_is_validated(report): - return ExpertReportAnnotation.objects.filter(report=report, user__groups__name='expert', validation_complete=True).count() == 3 and ExpertReportAnnotation.objects.filter(report=report, user__groups__name='superexpert', validation_complete=True).count() == 1 - def free_report(report): ExpertReportAnnotation.objects.filter(report=report).delete() def free_reports(number=10): - reports = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(photos__isnull=True).exclude(hide=True).filter(type='adult').order_by('-creation_time') + reports = Report.objects.queueable().with_finished_validation(state=False).filter(type='adult').order_by('-server_upload_time') i = 0 for r in reports: - if report_is_validated(r): - free_report(r) - print("Freeing report {0}, created on {1}".format( r.version_UUID, r.creation_time )) - i +=1 - if i >= number: - print("Reached limit") - break + free_report(r) + print("Freeing report {0}, created on {1}".format(r.pk, r.creation_time)) + i +=1 + if i >= number: + print("Reached limit") + break free_reports() \ No newline at end of file diff --git a/util_scripts/queue_simulator.py b/util_scripts/queue_simulator.py index 52c65b4a..287cd7ee 100644 --- a/util_scripts/queue_simulator.py +++ b/util_scripts/queue_simulator.py @@ -41,7 +41,7 @@ def get_global_queue_regular_user(this_user): current_pending = ExpertReportAnnotation.objects.filter(user=this_user).filter(validation_complete=False).filter(report__type='adult').count() my_reports = ExpertReportAnnotation.objects.filter(user=this_user).filter(report__type='adult').values('report').distinct() - new_reports_unfiltered = Report.objects.exclude(creation_time__year=2014).exclude(note__icontains="#345").exclude(version_UUID__in=my_reports).exclude(photos__isnull=True).exclude(hide=True).filter(type='adult').annotate(n_annotations=Count('expert_report_annotations')).filter(n_annotations__lt=max_given) + new_reports_unfiltered = Report.objects.filter(type='adult').exclude(pk__in=my_reports).queueable().with_finished_validation(state=False) bounding_boxes = EuropeCountry.objects.filter(is_bounding_box=True) diff --git a/util_scripts/update_global_assignment_stats.py b/util_scripts/update_global_assignment_stats.py deleted file mode 100644 index 7c9379e6..00000000 --- a/util_scripts/update_global_assignment_stats.py +++ /dev/null @@ -1,85 +0,0 @@ -import django -from django.conf import settings -from django.db.models import F - -import csv -import os, sys - -proj_path = "/home/webuser/webapps/tigaserver/" -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tigaserver_project.settings") -sys.path.append(proj_path) - -django.setup() - -from tigaserver_app.models import GlobalAssignmentStat, EuropeCountry -from tigacrafting.models import UserStat, ExpertReportAnnotation -from tigacrafting.report_queues import get_unassigned_available_reports, get_progress_available_reports, get_ns_locked_reports -from stats.views import get_blocked_count -from tigacrafting.views import get_blocked_reports_by_country -import datetime -import pytz - -def do_update(): - countries_with_at_least_one_expert = UserStat.objects.exclude(native_of__isnull=True).exclude(native_of__gid=17).values('native_of').distinct() - countries = EuropeCountry.objects.filter(gid__in=countries_with_at_least_one_expert) - for current_country in countries: - unassigned_filtered = get_unassigned_available_reports(current_country) - progress_filtered = get_progress_available_reports(current_country) - blocked_ns = get_ns_locked_reports(current_country) - if current_country.gid == 17: - user_id_filter = settings.USERS_IN_STATS - else: - user_id_filter = UserStat.objects.filter(native_of=current_country).values('user__id') - pending = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult').values('report') - n_unassigned = unassigned_filtered.count() - n_progress = progress_filtered.count() - n_pending = pending.count() - n_blocked_ns = 0 if blocked_ns is None else blocked_ns.count() - try: - g = GlobalAssignmentStat.objects.get(country=current_country) - g.unassigned_reports = n_unassigned - g.in_progress_reports = n_progress - g.pending_reports = n_pending - g.nsqueue_reports = n_blocked_ns - g.last_update = datetime.datetime.now(pytz.utc) - except GlobalAssignmentStat.DoesNotExist: - g = GlobalAssignmentStat( - country=current_country, - unassigned_reports=n_unassigned, - in_progress_reports=n_progress, - pending_reports=n_pending, - nsqueue_reports=n_blocked_ns, - last_update=datetime.datetime.now(pytz.utc) - ) - g.save() - - country_spain = EuropeCountry.objects.get(pk=17) - unassigned_filtered = get_unassigned_available_reports(country_spain) - progress_filtered = get_progress_available_reports(country_spain) - blocked_ns = None - user_id_filter = settings.USERS_IN_STATS - pending = ExpertReportAnnotation.objects.filter(user__id__in=user_id_filter).filter(validation_complete=False).filter(report__type='adult').values('report') - n_unassigned = unassigned_filtered.count() - n_progress = progress_filtered.count() - n_pending = pending.count() - n_blocked_ns = 0 if blocked_ns is None else blocked_ns.count() - try: - g = GlobalAssignmentStat.objects.get(country=country_spain) - g.unassigned_reports = n_unassigned - g.in_progress_reports = n_progress - g.pending_reports = n_pending - g.nsqueue_reports = n_blocked_ns - g.last_update = datetime.datetime.now(pytz.utc) - except GlobalAssignmentStat.DoesNotExist: - g = GlobalAssignmentStat( - country=country_spain, - unassigned_reports=n_unassigned, - in_progress_reports=n_progress, - pending_reports=n_pending, - nsqueue_reports=n_blocked_ns, - last_update=datetime.datetime.now(pytz.utc) - ) - g.save() - -if __name__ == "__main__": - do_update() \ No newline at end of file