Merge pull request #5015 from open-formulieren/feature/4931-submissio…

…n-statistics-based-on-logs

Display form submission statistics based on logs

pyright.pyproject.toml

@@ -34,6 +34,9 @@ include = [
     "src/openforms/contrib/objects_api/",
     # Emails
     "src/openforms/emails/templatetags/cosign_information.py",
+    # Logging
+    "src/openforms/logging/logevent.py",
+    "src/openforms/logging/models.py",
     # Registrations
     "src/openforms/registrations/tasks.py",
     "src/openforms/registrations/contrib/email/",

requirements/base.in

@@ -36,6 +36,7 @@ schwifty
 # Framework libraries
 django ~= 4.2
 django-admin-index
+django-admin-rangefilter
 django-autoslug
 django-axes[ipware]
 django-camunda

requirements/base.txt

@@ -132,6 +132,8 @@ django==4.2.17
     #   zgw-consumers
 django-admin-index==3.1.1
     # via -r requirements/base.in
+django-admin-rangefilter==0.13.2
+    # via -r requirements/base.in
 django-appconf==1.0.4
     # via
     #   django-cookie-consent

requirements/ci.txt

@@ -214,6 +214,10 @@ django-admin-index==3.1.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+django-admin-rangefilter==0.13.2
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-appconf==1.0.4
     # via
     #   -c requirements/base.txt

requirements/dev.txt

@@ -235,6 +235,10 @@ django-admin-index==3.1.1
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+django-admin-rangefilter==0.13.2
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 django-appconf==1.0.4
     # via
     #   -c requirements/ci.txt

requirements/extensions.txt

@@ -202,6 +202,10 @@ django-admin-index==3.1.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+django-admin-rangefilter==0.13.2
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-appconf==1.0.4
     # via
     #   -c requirements/base.txt

requirements/type-checking.txt

@@ -230,6 +230,10 @@ django-admin-index==3.1.1
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+django-admin-rangefilter==0.13.2
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 django-appconf==1.0.4
     # via
     #   -c requirements/ci.txt

src/openforms/conf/base.py

@@ -189,6 +189,7 @@
     "import_export",
     "flags",
     "django_setup_configuration",
+    "rangefilter",
     # Project applications.
     "openforms.accounts",
     "openforms.analytics_tools",

src/openforms/fixtures/default_admin_index.json

@@ -70,7 +70,7 @@
             ],
             [
                 "forms",
-                "formstatistics"
+                "formsubmissionstatistics"
             ]
         ]
     }
@@ -300,6 +300,10 @@
                 "django_yubin",
                 "message"
             ],
+            [
+                "log_outgoing_requests",
+                "outgoingrequestslog"
+            ],
             [
                 "logging",
                 "avgtimelinelogproxy"
@@ -308,10 +312,6 @@
                 "logging",
                 "timelinelogproxy"
             ],
-            [
-                "log_outgoing_requests",
-                "outgoingrequestslog"
-            ],
             [
                 "submissions",
                 "temporaryfileupload"

src/openforms/forms/admin/__init__.py

@@ -3,8 +3,8 @@
     form,
     form_definition,
     form_logic,
-    form_statistics,
     form_step,
+    form_submission_statistics,
     form_variable,
     form_version,
 )
@@ -14,7 +14,7 @@
     "form",
     "form_definition",
     "form_logic",
-    "form_statistics",
+    "form_submission_statistics",
     "form_step",
     "form_variable",
     "form_version",

src/openforms/forms/admin/form_submission_statistics.py

@@ -0,0 +1,94 @@
+from django.contrib import admin
+from django.db.models import Count, ExpressionWrapper, F, IntegerField
+from django.template.response import TemplateResponse
+from django.urls import path
+from django.utils.translation import gettext_lazy as _
+
+from rangefilter.filters import DateRangeFilterBuilder
+
+from ..forms.form_statistics import (
+    get_first_of_previous_month,
+    get_last_of_previous_month,
+)
+from ..models import FormSubmissionStatistics
+from .views import ExportSubmissionStatisticsView
+
+
+@admin.register(FormSubmissionStatistics)
+class FormSubmissionStatisticsAdmin(admin.ModelAdmin):
+    """
+    Modified admin view to display the submission statistics for each form.
+
+    The table displays the form name, number of submission and when the first and last
+    submission happened _within the selected date range_. We do this by looking at the
+    logged events for each completed submission, stored in the django-timeline-logger
+    database table with particular metadata.
+
+    Filtering for submissions on date range is possible based on the timestamp.
+
+    If the log events are pruned, this affects the reported statistics. Filtering a date
+    range will also show the first/last timestamps within the selected range.
+
+    There are no detail views to click through, the table overview is all you get. If
+    you construct the URLs by hand, you end up in the standard timeline log detail page.
+    No permissions to create, delete or modify records are enabled.
+    """
+
+    list_filter = (
+        (
+            "timestamp",
+            DateRangeFilterBuilder(
+                title=_("submitted between"),
+                default_start=lambda *args: get_first_of_previous_month(),
+                default_end=lambda *args: get_last_of_previous_month(),
+            ),
+        ),
+    )
+    search_fields = ("extra_data__form_name",)
+    show_full_result_count = False
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def get_urls(self):
+        urls = super().get_urls()
+        export_view = self.admin_site.admin_view(
+            ExportSubmissionStatisticsView.as_view(
+                media=self.media,
+            )  # pyright: ignore[reportArgumentType]
+        )
+        custom_urls = [
+            path("export/", export_view, name="formstatistics_export"),
+        ]
+        return custom_urls + urls
+
+    def changelist_view(self, request, extra_context=None):
+        # we can't really pass the queryset as just an extra thing because it doesn't
+        # apply the changelist filters. So instead, we grab the context from the
+        # parent's TemplateResponse and add it.
+        response = super().changelist_view(request, extra_context=extra_context)
+        assert isinstance(response, TemplateResponse)
+        assert response.context_data is not None
+        qs = (
+            response.context_data["cl"]
+            .queryset.annotate(
+                form_id=ExpressionWrapper(
+                    F("extra_data__form_id"), output_field=IntegerField()
+                )
+            )
+            .exclude(form_id__isnull=True)
+            .values("form_id")
+            .annotate(
+                submission_count=Count("id"),
+                form_name=F("extra_data__form_name"),
+            )
+            .order_by("form_name")
+        )
+        response.context_data["aggregated_qs"] = qs
+        return response

src/openforms/forms/admin/views.py

@@ -26,7 +26,7 @@
 
 from ..forms import ExportStatisticsForm
 from ..forms.form import FormImportForm
-from ..models import Form, FormsExport, FormStatistics
+from ..models import Form, FormsExport, FormSubmissionStatistics
 from ..utils import import_form
 from .tasks import process_forms_export, process_forms_import
 
@@ -122,8 +122,8 @@ def _bulk_import_forms(self, import_file):
 class ExportSubmissionStatisticsView(
     LoginRequiredMixin, PermissionRequiredMixin, FormView
 ):
-    permission_required = "forms.view_formstatistics"
-    template_name = "admin/forms/formstatistics/export_form.html"
+    permission_required = "forms.view_formsubmissionstatistics"
+    template_name = "admin/forms/formsubmissionstatistics/export_form.html"
     form_class = ExportStatisticsForm
 
     # must be set by the ModelAdmin
@@ -160,7 +160,7 @@ def form_fields():
 
         context.update(
             {
-                "opts": FormStatistics._meta,
+                "opts": FormSubmissionStatistics._meta,
                 "media": self.media + form.media,
                 "form_fields": form_fields,
             }

src/openforms/forms/forms/form_statistics.py

@@ -4,12 +4,15 @@
 
 from django import forms
 from django.contrib.admin.widgets import AdminDateWidget
+from django.db.models import TextChoices
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 from dateutil.relativedelta import relativedelta
 from tablib import Dataset
 
+from openforms.logging import logevent
+
 from ..models import Form
 from ..statistics import export_registration_statistics
 
@@ -28,7 +31,24 @@ def get_last_of_previous_month() -> date:
     return get_last_of_previous_month.date()
 
 
+class EventChoices(TextChoices):
+    registration_success = logevent.REGISTRATION_SUCCESS_EVENT, _(
+        "Successfully registered"
+    )
+    submission_success = logevent.FORM_SUBMIT_SUCCESS_EVENT, _("Completed")
+
+
 class ExportStatisticsForm(forms.Form):
+    kind = forms.ChoiceField(
+        label=_("Kind"),
+        choices=EventChoices.choices,
+        initial=EventChoices.registration_success,
+        help_text=_(
+            "Successfully registered submissions were sent to an external system for "
+            "further processing. Completed submissions are form submissions finished "
+            "by the end-user that may or may not be registered."
+        ),
+    )
     start_date = forms.DateField(
         label=_("From"),
         required=True,
@@ -50,7 +70,7 @@ class ExportStatisticsForm(forms.Form):
     limit_to_forms = forms.ModelMultipleChoiceField(
         label=_("Forms"),
         required=False,
-        queryset=Form.objects.filter(_is_deleted=False),
+        queryset=Form.objects.filter(_is_deleted=False).order_by("name"),
         help_text=_(
             "Limit the export to the selected forms, if specified. Leave the field "
             "empty to export all forms. Hold CTRL (or COMMAND on Mac) to select "
@@ -61,8 +81,10 @@ class ExportStatisticsForm(forms.Form):
     def export(self) -> Dataset:
         start_date: date = self.cleaned_data["start_date"]
         end_date: date = self.cleaned_data["end_date"]
+        event: str = self.cleaned_data["kind"]
         return export_registration_statistics(
             start_date,
             end_date,
             self.cleaned_data["limit_to_forms"],
+            event,
         )

src/openforms/forms/migrations/0099_formsubmissionstatisticsv2_delete_formstatistics.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.17 on 2025-01-13 20:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("logging", "0003_backfill_submission_log_records"),
+        ("forms", "0098_v270_to_v300"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="FormStatistics",
+        ),
+        migrations.CreateModel(
+            name="FormSubmissionStatistics",
+            fields=[],
+            options={
+                "verbose_name": "form submission statistics",
+                "verbose_name_plural": "form submission statistics",
+                "proxy": True,
+                "indexes": [],
+                "constraints": [],
+            },
+            bases=("logging.timelinelogproxy",),
+        ),
+    ]