From 58b23647cd31e14bc1a690813c6a1bdd47488ce2 Mon Sep 17 00:00:00 2001 From: robinvandermolen Date: Mon, 11 Nov 2024 15:21:38 +0100 Subject: [PATCH 1/4] :card_file_box: [#4815] Change submission removal limit to 0 Allowing submissions to be deleted after 0 days (i.e. on the same day) --- ...configuration_submissions_removal_limit.py | 54 +++++++++++++++++ src/openforms/config/models/config.py | 8 +-- ..._all_submissions_removal_limit_and_more.py | 58 +++++++++++++++++++ src/openforms/forms/models/form.py | 8 +-- .../admin/form_design/DataRemoval.js | 8 +-- 5 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 src/openforms/config/migrations/0064_alter_globalconfiguration_submissions_removal_limit.py create mode 100644 src/openforms/forms/migrations/0105_alter_form_all_submissions_removal_limit_and_more.py diff --git a/src/openforms/config/migrations/0064_alter_globalconfiguration_submissions_removal_limit.py b/src/openforms/config/migrations/0064_alter_globalconfiguration_submissions_removal_limit.py new file mode 100644 index 0000000000..2640cc362c --- /dev/null +++ b/src/openforms/config/migrations/0064_alter_globalconfiguration_submissions_removal_limit.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.16 on 2024-11-11 14:08 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("config", "0063_merge_20240923_1612"), + ] + + operations = [ + migrations.AlterField( + model_name="globalconfiguration", + name="all_submissions_removal_limit", + field=models.PositiveIntegerField( + default=90, + help_text="Amount of days when all submissions will be permanently deleted", + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="all submissions removal limit", + ), + ), + migrations.AlterField( + model_name="globalconfiguration", + name="errored_submissions_removal_limit", + field=models.PositiveIntegerField( + default=30, + help_text="Amount of days errored submissions will remain before being removed", + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="errored submission removal limit", + ), + ), + migrations.AlterField( + model_name="globalconfiguration", + name="incomplete_submissions_removal_limit", + field=models.PositiveIntegerField( + default=7, + help_text="Amount of days incomplete submissions will remain before being removed", + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="incomplete submission removal limit", + ), + ), + migrations.AlterField( + model_name="globalconfiguration", + name="successful_submissions_removal_limit", + field=models.PositiveIntegerField( + default=7, + help_text="Amount of days successful submissions will remain before being removed", + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="successful submission removal limit", + ), + ), + ] diff --git a/src/openforms/config/models/config.py b/src/openforms/config/models/config.py index a616f8d15f..980e74e8fc 100644 --- a/src/openforms/config/models/config.py +++ b/src/openforms/config/models/config.py @@ -421,7 +421,7 @@ class GlobalConfiguration(SingletonModel): successful_submissions_removal_limit = models.PositiveIntegerField( _("successful submission removal limit"), default=7, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days successful submissions will remain before being removed" ), @@ -436,7 +436,7 @@ class GlobalConfiguration(SingletonModel): incomplete_submissions_removal_limit = models.PositiveIntegerField( _("incomplete submission removal limit"), default=7, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days incomplete submissions will remain before being removed" ), @@ -451,7 +451,7 @@ class GlobalConfiguration(SingletonModel): errored_submissions_removal_limit = models.PositiveIntegerField( _("errored submission removal limit"), default=30, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days errored submissions will remain before being removed" ), @@ -466,7 +466,7 @@ class GlobalConfiguration(SingletonModel): all_submissions_removal_limit = models.PositiveIntegerField( _("all submissions removal limit"), default=90, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_("Amount of days when all submissions will be permanently deleted"), ) diff --git a/src/openforms/forms/migrations/0105_alter_form_all_submissions_removal_limit_and_more.py b/src/openforms/forms/migrations/0105_alter_form_all_submissions_removal_limit_and_more.py new file mode 100644 index 0000000000..e3b54c84cc --- /dev/null +++ b/src/openforms/forms/migrations/0105_alter_form_all_submissions_removal_limit_and_more.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.16 on 2024-11-14 09:31 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("forms", "0104_select_datatype_string"), + ] + + operations = [ + migrations.AlterField( + model_name="form", + name="all_submissions_removal_limit", + field=models.PositiveIntegerField( + blank=True, + help_text="Amount of days when all submissions of this form will be permanently deleted. Leave blank to use value in General Configuration.", + null=True, + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="all submissions removal limit", + ), + ), + migrations.AlterField( + model_name="form", + name="errored_submissions_removal_limit", + field=models.PositiveIntegerField( + blank=True, + help_text="Amount of days errored submissions of this form will remain before being removed. Leave blank to use value in General Configuration.", + null=True, + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="errored submission removal limit", + ), + ), + migrations.AlterField( + model_name="form", + name="incomplete_submissions_removal_limit", + field=models.PositiveIntegerField( + blank=True, + help_text="Amount of days incomplete submissions of this form will remain before being removed. Leave blank to use value in General Configuration.", + null=True, + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="incomplete submission removal limit", + ), + ), + migrations.AlterField( + model_name="form", + name="successful_submissions_removal_limit", + field=models.PositiveIntegerField( + blank=True, + help_text="Amount of days successful submissions of this form will remain before being removed. Leave blank to use value in General Configuration.", + null=True, + validators=[django.core.validators.MinValueValidator(0)], + verbose_name="successful submission removal limit", + ), + ), + ] diff --git a/src/openforms/forms/models/form.py b/src/openforms/forms/models/form.py index b45ab28fdc..b08c994c73 100644 --- a/src/openforms/forms/models/form.py +++ b/src/openforms/forms/models/form.py @@ -295,7 +295,7 @@ class Form(models.Model): _("successful submission removal limit"), blank=True, null=True, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days successful submissions of this form will remain before being removed. " "Leave blank to use value in General Configuration." @@ -315,7 +315,7 @@ class Form(models.Model): _("incomplete submission removal limit"), blank=True, null=True, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days incomplete submissions of this form will remain before being removed. " "Leave blank to use value in General Configuration." @@ -335,7 +335,7 @@ class Form(models.Model): _("errored submission removal limit"), blank=True, null=True, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days errored submissions of this form will remain before being removed. " "Leave blank to use value in General Configuration." @@ -355,7 +355,7 @@ class Form(models.Model): _("all submissions removal limit"), blank=True, null=True, - validators=[MinValueValidator(1)], + validators=[MinValueValidator(0)], help_text=_( "Amount of days when all submissions of this form will be permanently deleted. " "Leave blank to use value in General Configuration." diff --git a/src/openforms/js/components/admin/form_design/DataRemoval.js b/src/openforms/js/components/admin/form_design/DataRemoval.js index 7c8811ea24..1eb1e06bbd 100644 --- a/src/openforms/js/components/admin/form_design/DataRemoval.js +++ b/src/openforms/js/components/admin/form_design/DataRemoval.js @@ -61,7 +61,7 @@ const DataRemoval = ({submissionsRemovalOptions, onChange}) => { @@ -109,7 +109,7 @@ const DataRemoval = ({submissionsRemovalOptions, onChange}) => { @@ -154,7 +154,7 @@ const DataRemoval = ({submissionsRemovalOptions, onChange}) => { /> } > - + @@ -198,7 +198,7 @@ const DataRemoval = ({submissionsRemovalOptions, onChange}) => { /> } > - + From 41fef81b3615f44c8d948e39d4966cafc818f7c2 Mon Sep 17 00:00:00 2001 From: robinvandermolen Date: Tue, 12 Nov 2024 10:27:22 +0100 Subject: [PATCH 2/4] :bento: [#4815] Rebuild openapi.yaml --- src/openapi.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/openapi.yaml b/src/openapi.yaml index f7dbb3086b..29aee529b7 100644 --- a/src/openapi.yaml +++ b/src/openapi.yaml @@ -10240,7 +10240,7 @@ components: successfulSubmissionsRemovalLimit: type: integer maximum: 2147483647 - minimum: 1 + minimum: 0 nullable: true title: Successful submission removal limit description: Amount of days successful submissions of this form will remain @@ -10257,7 +10257,7 @@ components: incompleteSubmissionsRemovalLimit: type: integer maximum: 2147483647 - minimum: 1 + minimum: 0 nullable: true title: Incomplete submission removal limit description: Amount of days incomplete submissions of this form will remain @@ -10274,7 +10274,7 @@ components: erroredSubmissionsRemovalLimit: type: integer maximum: 2147483647 - minimum: 1 + minimum: 0 nullable: true title: Errored submission removal limit description: Amount of days errored submissions of this form will remain @@ -10292,7 +10292,7 @@ components: allSubmissionsRemovalLimit: type: integer maximum: 2147483647 - minimum: 1 + minimum: 0 nullable: true description: Amount of days when all submissions of this form will be permanently deleted. Leave blank to use value in General Configuration. From 2ff331fea1775f1a70a7fd7144b3dde899c077e5 Mon Sep 17 00:00:00 2001 From: robinvandermolen Date: Wed, 13 Nov 2024 10:24:27 +0100 Subject: [PATCH 3/4] :white_check_mark: [#4815] Fixing test assertion --- src/openforms/data_removal/tests/test_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openforms/data_removal/tests/test_tasks.py b/src/openforms/data_removal/tests/test_tasks.py index 49b980d01d..263c1ac339 100644 --- a/src/openforms/data_removal/tests/test_tasks.py +++ b/src/openforms/data_removal/tests/test_tasks.py @@ -174,7 +174,7 @@ def test_incomplete_submissions_correctly_deleted(self): with self.assertRaises(ObjectDoesNotExist): pending_submission_to_be_deleted.refresh_from_db() with self.assertRaises(ObjectDoesNotExist): - pending_submission_to_be_deleted.refresh_from_db() + in_progress_submission_to_be_deleted.refresh_from_db() def test_incomplete_submissions_with_form_settings_override_global_configuration( self, From 9e77550207b21e94c222265cbc55ab0da96f5a97 Mon Sep 17 00:00:00 2001 From: robinvandermolen Date: Wed, 13 Nov 2024 11:23:15 +0100 Subject: [PATCH 4/4] :white_check_mark: [#4815] Testing submission_removal_limit 0 --- .../data_removal/tests/test_tasks.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/src/openforms/data_removal/tests/test_tasks.py b/src/openforms/data_removal/tests/test_tasks.py index 263c1ac339..0dca1d6f30 100644 --- a/src/openforms/data_removal/tests/test_tasks.py +++ b/src/openforms/data_removal/tests/test_tasks.py @@ -58,6 +58,34 @@ def test_successful_submissions_correctly_deleted(self): with self.assertRaises(ObjectDoesNotExist): submission_to_be_deleted.refresh_from_db() + def test_successful_submissions_correctly_deleted_the_same_day_when_form_removal_limit_is_0( + self, + ): + form = FormFactory.create( + successful_submissions_removal_limit=0, + incomplete_submissions_removal_limit=7, + errored_submissions_removal_limit=7, + all_submissions_removal_limit=7, + ) + + with freeze_time("2024-11-12T18:00:00+01:00"): + # Submission not connected to form + SubmissionFactory.create(registration_success=True) + + submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_success=True + ) + + self.assertEqual(Submission.objects.count(), 2) + + with freeze_time("2024-11-12T19:00:00+01:00"): + delete_submissions() + + # Only the submission connected to the form should be deleted + self.assertEqual(Submission.objects.count(), 1) + with self.assertRaises(ObjectDoesNotExist): + submission_to_be_deleted.refresh_from_db() + @tag("gh-2632") def test_delete_successful_submission_with_deleted_form_step(self): config = GlobalConfiguration.get_solo() @@ -176,6 +204,42 @@ def test_incomplete_submissions_correctly_deleted(self): with self.assertRaises(ObjectDoesNotExist): in_progress_submission_to_be_deleted.refresh_from_db() + def test_incomplete_submissions_correctly_deleted_the_same_day_when_form_removal_limit_is_0( + self, + ): + form = FormFactory.create( + successful_submissions_removal_limit=7, + incomplete_submissions_removal_limit=0, + errored_submissions_removal_limit=7, + all_submissions_removal_limit=7, + ) + + with freeze_time("2024-11-12T18:00:00+01:00"): + # Incomplete submissions not connected to the form + SubmissionFactory.create(registration_status=RegistrationStatuses.pending) + SubmissionFactory.create( + registration_status=RegistrationStatuses.in_progress + ) + + SubmissionFactory.create(form=form, registration_success=True) + pending_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.pending + ) + in_progress_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.in_progress + ) + + self.assertEqual(Submission.objects.count(), 5) + + with freeze_time("2024-11-12T19:00:00+01:00"): + delete_submissions() + + self.assertEqual(Submission.objects.count(), 3) + with self.assertRaises(ObjectDoesNotExist): + pending_submission_to_be_deleted.refresh_from_db() + with self.assertRaises(ObjectDoesNotExist): + in_progress_submission_to_be_deleted.refresh_from_db() + def test_incomplete_submissions_with_form_settings_override_global_configuration( self, ): @@ -277,6 +341,42 @@ def test_failed_submissions_correctly_deleted(self): with self.assertRaises(ObjectDoesNotExist): submission_to_be_deleted.refresh_from_db() + def test_failed_submissions_correctly_deleted_the_same_day_when_form_removal_limit_is_0( + self, + ): + form = FormFactory.create( + successful_submissions_removal_limit=7, + incomplete_submissions_removal_limit=7, + errored_submissions_removal_limit=0, + all_submissions_removal_limit=7, + ) + + with freeze_time("2024-11-12T18:00:00+01:00"): + # Failed submission not connected to the form + SubmissionFactory.create(registration_status=RegistrationStatuses.failed) + + # Successful and incomplete submissions + SubmissionFactory.create(form=form, registration_success=True) + SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.pending + ) + SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.in_progress + ) + + failed_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.failed + ) + + self.assertEqual(Submission.objects.count(), 5) + + with freeze_time("2024-11-12T19:00:00+01:00"): + delete_submissions() + + self.assertEqual(Submission.objects.count(), 4) + with self.assertRaises(ObjectDoesNotExist): + failed_submission_to_be_deleted.refresh_from_db() + def test_failed_submissions_with_form_settings_override_global_configuration(self): config = GlobalConfiguration.get_solo() form_longer_limit = FormFactory.create( @@ -339,6 +439,53 @@ def test_all_submissions_correctly_deleted(self): with self.assertRaises(ObjectDoesNotExist): old_submission.refresh_from_db() + def test_all_submissions_correctly_deleted_the_same_day_when_form_removal_limit_is_0( + self, + ): + form = FormFactory.create( + successful_submissions_removal_limit=7, + incomplete_submissions_removal_limit=7, + errored_submissions_removal_limit=7, + all_submissions_removal_limit=0, + ) + + with freeze_time("2024-11-12T18:00:00+01:00"): + # Submissions not connected to the form + SubmissionFactory.create(registration_success=True) + SubmissionFactory.create(registration_status=RegistrationStatuses.pending) + SubmissionFactory.create( + registration_status=RegistrationStatuses.in_progress + ) + SubmissionFactory.create(registration_status=RegistrationStatuses.failed) + + submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_success=True + ) + pending_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.pending + ) + in_progress_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.in_progress + ) + failed_submission_to_be_deleted = SubmissionFactory.create( + form=form, registration_status=RegistrationStatuses.failed + ) + + self.assertEqual(Submission.objects.count(), 8) + + with freeze_time("2024-11-12T19:00:00+01:00"): + delete_submissions() + + self.assertEqual(Submission.objects.count(), 4) + with self.assertRaises(ObjectDoesNotExist): + submission_to_be_deleted.refresh_from_db() + with self.assertRaises(ObjectDoesNotExist): + pending_submission_to_be_deleted.refresh_from_db() + with self.assertRaises(ObjectDoesNotExist): + in_progress_submission_to_be_deleted.refresh_from_db() + with self.assertRaises(ObjectDoesNotExist): + failed_submission_to_be_deleted.refresh_from_db() + def test_all_submissions_with_form_settings_override_global_configuration(self): config = GlobalConfiguration.get_solo() form_longer_limit = FormFactory.create(