From 128df478fff76ba88fdfcf2cd3c00094e2882954 Mon Sep 17 00:00:00 2001 From: vladsha-dev Date: Fri, 17 Jan 2025 13:04:06 +0100 Subject: [PATCH] [OSDEV-1514] Database. Upgrade the PostgreSQL version to 13 (#479) - Upgraded the PostgreSQL version from 12 to 13 for the database used in local development, DB anonymization, DB restore setup, and environments in the AWS cloud. Additionally, the postgis and pg_trgm extensions have been upgraded to versions 3.4.2 and 1.5, respectively, based on the available extension versions for PostgreSQL 13.15 in AWS RDS. For more information, see [Extensions supported for RDS for PostgreSQL 13](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html#postgresql-extensions-13x). Allowed major version upgrades and activated the `apply immediately` flag to perform the PostgreSQL major version upgrade in AWS. - Corrected spelling mistakes in the `src/anon-tools/do_dump.sh` file and in the name of the folder `database_anonymizer_sheduled_task`. Removed the unused `src/anon-tools/anon.sql` file and the redundant `src/anon-tools/initdb.sql` file. Removed commented-out code in the `src/anon-tools/Dockerfile.dump` and `deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py` files. - Introduced `rds_allow_major_version_upgrade` and `rds_apply_immediately` Terraform variables to enable or disable major version upgrades and the `apply immediately` flag, depending on the environment. --- .github/workflows/deploy_to_aws.yml | 4 +- .../environments/terraform-development.tfvars | 8 +-- .../environments/terraform-preprod.tfvars | 4 +- .../environments/terraform-production.tfvars | 6 +- .../environments/terraform-staging.tfvars | 6 +- deployment/environments/terraform-test.tfvars | 6 +- deployment/terraform/anonymize_db_job.tf | 2 +- .../docker/Dockerfile | 2 +- deployment/terraform/database.tf | 50 +++++++++-------- .../.gitignore | 0 .../docker/.dockerignore | 0 .../docker/Dockerfile | 0 .../docker/anonymize_script.sql | 0 .../docker/database_anonymizer.py | 3 - .../docker/requirements.txt | 0 .../kms.tf | 0 .../locals.tf | 0 .../main.tf | 0 .../variables.tf | 0 deployment/terraform/variables.tf | 16 +++++- doc/release/RELEASE-NOTES-TEMPLATE.md | 6 +- doc/release/RELEASE-NOTES.md | 55 ++++++++----------- src/anon-tools/Dockerfile.dump | 5 +- src/anon-tools/Dockerfile.restore | 2 +- src/anon-tools/anon.sql | 35 ------------ src/anon-tools/do_dump.sh | 2 +- src/anon-tools/initdb.sql | 20 ------- src/database/Dockerfile.local | 23 +++++--- ...atistic_and_upgrade_postgres_extensions.py | 51 +++++++++++++++++ ...tistic_and_upgrade_postgres_extensions.sql | 4 ++ 30 files changed, 164 insertions(+), 146 deletions(-) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/.gitignore (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/docker/.dockerignore (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/docker/Dockerfile (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/docker/anonymize_script.sql (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/docker/database_anonymizer.py (98%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/docker/requirements.txt (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/kms.tf (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/locals.tf (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/main.tf (100%) rename deployment/terraform/{database_anonymizer_sheduled_task => database_anonymizer_scheduled_task}/variables.tf (100%) delete mode 100644 src/anon-tools/anon.sql delete mode 100644 src/anon-tools/initdb.sql create mode 100644 src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py create mode 100644 src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql diff --git a/.github/workflows/deploy_to_aws.yml b/.github/workflows/deploy_to_aws.yml index 2def34ede..a4c791814 100644 --- a/.github/workflows/deploy_to_aws.yml +++ b/.github/workflows/deploy_to_aws.yml @@ -224,8 +224,8 @@ jobs: uses: docker/build-push-action@v2 if: ${{ steps.get_env_name.outputs.lowercase == 'production' }} with: - context: deployment/terraform/database_anonymizer_sheduled_task/docker - file: deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile + context: deployment/terraform/database_anonymizer_scheduled_task/docker + file: deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.IMAGE_NAME }}-database-anonymizer-${{ steps.get_env_name.outputs.lowercase }}:${{ env.GIT_COMMIT }} diff --git a/deployment/environments/terraform-development.tfvars b/deployment/environments/terraform-development.tfvars index 8a593143b..d25e2c638 100644 --- a/deployment/environments/terraform-development.tfvars +++ b/deployment/environments/terraform-development.tfvars @@ -14,15 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.micro" rds_database_identifier = "opensupplyhub-enc-stg" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true -snapshot_identifier = "" -rds_deletion_protection = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "1" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-preprod.tfvars b/deployment/environments/terraform-preprod.tfvars index 380714645..6b65c6bab 100644 --- a/deployment/environments/terraform-preprod.tfvars +++ b/deployment/environments/terraform-preprod.tfvars @@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-pp" rds_database_name = "opensupplyhub" diff --git a/deployment/environments/terraform-production.tfvars b/deployment/environments/terraform-production.tfvars index 933cc1ebb..c77987158 100644 --- a/deployment/environments/terraform-production.tfvars +++ b/deployment/environments/terraform-production.tfvars @@ -13,13 +13,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.m6in.4xlarge" rds_database_identifier = "opensupplyhub-enc-prd" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "10" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-staging.tfvars b/deployment/environments/terraform-staging.tfvars index fd586e28c..616369e98 100644 --- a/deployment/environments/terraform-staging.tfvars +++ b/deployment/environments/terraform-staging.tfvars @@ -12,13 +12,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "128" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.large" rds_database_identifier = "opensupplyhub-enc-stg" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true app_ecs_desired_count = "4" app_ecs_deployment_min_percent = "100" diff --git a/deployment/environments/terraform-test.tfvars b/deployment/environments/terraform-test.tfvars index b2a7c4b25..adccf0196 100644 --- a/deployment/environments/terraform-test.tfvars +++ b/deployment/environments/terraform-test.tfvars @@ -14,13 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5" bastion_instance_type = "t3.nano" rds_allocated_storage = "256" -rds_engine_version = "12" -rds_parameter_group_family = "postgres12" +rds_engine_version = "13" +rds_parameter_group_family = "postgres13" rds_instance_type = "db.t3.xlarge" rds_database_identifier = "opensupplyhub-enc-tst" rds_database_name = "opensupplyhub" rds_multi_az = false rds_storage_encrypted = true +rds_allow_major_version_upgrade = true +rds_apply_immediately = true anonymized_database_instance_type = "db.t3.2xlarge" anonymized_database_identifier = "database-anonymizer" diff --git a/deployment/terraform/anonymize_db_job.tf b/deployment/terraform/anonymize_db_job.tf index eb859113e..7bd8d7fe6 100644 --- a/deployment/terraform/anonymize_db_job.tf +++ b/deployment/terraform/anonymize_db_job.tf @@ -1,7 +1,7 @@ module "database_anonymizer" { count = var.database_anonymizer_enabled == true ? 1 : 0 - source = "./database_anonymizer_sheduled_task" + source = "./database_anonymizer_scheduled_task" rds_database_identifier = var.rds_database_identifier rds_database_name = var.rds_database_name diff --git a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile index 6e0b7e9b9..fc2f8e174 100644 --- a/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile +++ b/deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ diff --git a/deployment/terraform/database.tf b/deployment/terraform/database.tf index 4e703b814..3ef35d234 100644 --- a/deployment/terraform/database.tf +++ b/deployment/terraform/database.tf @@ -76,30 +76,32 @@ resource "aws_db_parameter_group" "default" { } module "database_enc" { - source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.0.3" - - vpc_id = module.vpc.id - allocated_storage = var.rds_allocated_storage - engine_version = var.rds_engine_version - instance_type = var.rds_instance_type - storage_type = var.rds_storage_type - database_identifier = var.rds_database_identifier - database_name = var.rds_database_name - database_username = var.rds_database_username - database_password = var.rds_database_password - backup_retention_period = var.rds_backup_retention_period - backup_window = var.rds_backup_window - maintenance_window = var.rds_maintenance_window - auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade - final_snapshot_identifier = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())]) - skip_final_snapshot = var.rds_skip_final_snapshot - copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot - multi_availability_zone = var.rds_multi_az - storage_encrypted = var.rds_storage_encrypted - subnet_group = aws_db_subnet_group.default.name - parameter_group = aws_db_parameter_group.default.name - deletion_protection = var.rds_deletion_protection - snapshot_identifier = var.snapshot_identifier + source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.1.0" + + vpc_id = module.vpc.id + allocated_storage = var.rds_allocated_storage + engine_version = var.rds_engine_version + instance_type = var.rds_instance_type + storage_type = var.rds_storage_type + database_identifier = var.rds_database_identifier + database_name = var.rds_database_name + database_username = var.rds_database_username + database_password = var.rds_database_password + backup_retention_period = var.rds_backup_retention_period + backup_window = var.rds_backup_window + maintenance_window = var.rds_maintenance_window + auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade + allow_major_version_upgrade = var.rds_allow_major_version_upgrade + apply_immediately = var.rds_apply_immediately + final_snapshot_identifier = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())]) + skip_final_snapshot = var.rds_skip_final_snapshot + copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot + multi_availability_zone = var.rds_multi_az + storage_encrypted = var.rds_storage_encrypted + subnet_group = aws_db_subnet_group.default.name + parameter_group = aws_db_parameter_group.default.name + deletion_protection = var.rds_deletion_protection + snapshot_identifier = var.snapshot_identifier alarm_cpu_threshold = var.rds_cpu_threshold_percent alarm_disk_queue_threshold = var.rds_disk_queue_threshold diff --git a/deployment/terraform/database_anonymizer_sheduled_task/.gitignore b/deployment/terraform/database_anonymizer_scheduled_task/.gitignore similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/.gitignore rename to deployment/terraform/database_anonymizer_scheduled_task/.gitignore diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/.dockerignore b/deployment/terraform/database_anonymizer_scheduled_task/docker/.dockerignore similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/.dockerignore rename to deployment/terraform/database_anonymizer_scheduled_task/docker/.dockerignore diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile b/deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile rename to deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/anonymize_script.sql b/deployment/terraform/database_anonymizer_scheduled_task/docker/anonymize_script.sql similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/anonymize_script.sql rename to deployment/terraform/database_anonymizer_scheduled_task/docker/anonymize_script.sql diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py b/deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py similarity index 98% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py rename to deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py index 6a7e9a99a..754c9ad7e 100755 --- a/deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py +++ b/deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py @@ -77,9 +77,6 @@ ) db = pg8000.native.Connection(**connection_information) -# cur = db.cursor() -# cur.execute(open("anonymize_script.sql", "r").read()) -# cur.commit() db.run(open("anonymize_script.sql", "r").read()) print('Database anonymized successfully!') diff --git a/deployment/terraform/database_anonymizer_sheduled_task/docker/requirements.txt b/deployment/terraform/database_anonymizer_scheduled_task/docker/requirements.txt similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/docker/requirements.txt rename to deployment/terraform/database_anonymizer_scheduled_task/docker/requirements.txt diff --git a/deployment/terraform/database_anonymizer_sheduled_task/kms.tf b/deployment/terraform/database_anonymizer_scheduled_task/kms.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/kms.tf rename to deployment/terraform/database_anonymizer_scheduled_task/kms.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/locals.tf b/deployment/terraform/database_anonymizer_scheduled_task/locals.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/locals.tf rename to deployment/terraform/database_anonymizer_scheduled_task/locals.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/main.tf b/deployment/terraform/database_anonymizer_scheduled_task/main.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/main.tf rename to deployment/terraform/database_anonymizer_scheduled_task/main.tf diff --git a/deployment/terraform/database_anonymizer_sheduled_task/variables.tf b/deployment/terraform/database_anonymizer_scheduled_task/variables.tf similarity index 100% rename from deployment/terraform/database_anonymizer_sheduled_task/variables.tf rename to deployment/terraform/database_anonymizer_scheduled_task/variables.tf diff --git a/deployment/terraform/variables.tf b/deployment/terraform/variables.tf index e275140bd..a6eae78a3 100644 --- a/deployment/terraform/variables.tf +++ b/deployment/terraform/variables.tf @@ -70,11 +70,11 @@ variable "rds_allocated_storage" { } variable "rds_engine_version" { - default = "12.4" + default = "13" } variable "rds_parameter_group_family" { - default = "postgres12" + default = "postgres13" } variable "rds_instance_type" { @@ -114,6 +114,18 @@ variable "rds_auto_minor_version_upgrade" { default = true } +variable "rds_allow_major_version_upgrade" { + default = false + type = bool + description = "Indicates that major PostgreSQL engine version upgrades are allowed." +} + +variable "rds_apply_immediately" { + default = false + type = bool + description = "Specifies whether any database modifications are applied immediately, or during the next maintenance window." +} + variable "rds_final_snapshot_identifier" { default = "osh-rds-snapshot" } diff --git a/doc/release/RELEASE-NOTES-TEMPLATE.md b/doc/release/RELEASE-NOTES-TEMPLATE.md index 3cd139754..5ddb5b5d9 100644 --- a/doc/release/RELEASE-NOTES-TEMPLATE.md +++ b/doc/release/RELEASE-NOTES-TEMPLATE.md @@ -7,11 +7,13 @@ Use the format below to document information about the new release. * Release date: *Provide release date* ### Database changes +* *Describe high-level database changes.* + #### Migrations: * *Describe migrations here.* -#### Scheme changes -* *Describe scheme changes here.* +#### Schema changes +* *Describe schema changes here.* ### Code/API changes * *Describe code/API changes here.* diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md index c49c9e034..72f0f012d 100644 --- a/doc/release/RELEASE-NOTES.md +++ b/doc/release/RELEASE-NOTES.md @@ -10,13 +10,18 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Release date: January 25, 2025 ### Database changes +* [OSDEV-1514](https://opensupplyhub.atlassian.net/browse/OSDEV-1514) - Upgraded the PostgreSQL version from 12 to 13 for the database used in local development, DB anonymization, DB restore setup, and environments in the AWS cloud. Additionally, the postgis and pg_trgm extensions have been upgraded to versions 3.4.2 and 1.5, respectively, based on the available extension versions for PostgreSQL 13.15 in AWS RDS. For more information, see [Extensions supported for RDS for PostgreSQL 13](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html#postgresql-extensions-13x). Allowed major version upgrades and activated the `apply immediately` flag to perform the PostgreSQL major version upgrade in AWS. + #### Migrations: +* 0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py - This migration refreshes the `pg_statistic` table after the upgrade to PostgreSQL 13 and upgrades the postgis and pg_trgm extensions to versions 3.4.2 and 1.5, respectively. -#### Scheme changes +#### Schema changes ### Code/API changes +[OSDEV-1514](https://opensupplyhub.atlassian.net/browse/OSDEV-1514) - Corrected spelling mistakes in the `src/anon-tools/do_dump.sh` file and in the name of the folder `database_anonymizer_sheduled_task`. Removed the unused `src/anon-tools/anon.sql` file and the redundant `src/anon-tools/initdb.sql` file. Removed commented-out code in the `src/anon-tools/Dockerfile.dump` and `deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py` files. ### Architecture/Environment changes +[OSDEV-1514](https://opensupplyhub.atlassian.net/browse/OSDEV-1514) - Introduced `rds_allow_major_version_upgrade` and `rds_apply_immediately` Terraform variables to enable or disable major version upgrades and the `apply immediately` flag, depending on the environment. ### Bugfix @@ -43,6 +48,10 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html * Ensure that the following commands are included in the `post_deployment` command: * `migrate` * `reindex_database` +* This release will upgrade PostgreSQL from version 12 to version 13. + * The upgrade will be performed automatically by Terrafrom and AWS, but some steps need to be completed **before** and **after** the upgrade. Please refer to [the Confluence article](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/620134402/PostgreSQL+database+upgrade+from+version+12+to+version+13) for detailed instructions. + * Steps to be completed before the upgrade are marked with the statement: "**This should be done before deploying the upgraded database.**". Post-upgrade tasks can be found under the [After the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/620134402/PostgreSQL+database+upgrade+from+version+12+to+version+13#After-the-PostgreSQL-major-version-upgrade) section. + * In case of an unsuccessful release along with the database upgrade, follow the instructions under the [Guide for rolling back the PostgreSQL major version upgrade](https://opensupplyhub.atlassian.net/wiki/spaces/SD/pages/620134402/PostgreSQL+database+upgrade+from+version+12+to+version+13#Guide-for-rolling-back-the-PostgreSQL-major-version-upgrade) section. ## Release 1.27.0 @@ -54,7 +63,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html ### Database changes #### Migrations: -#### Scheme changes +#### Schema changes ### Code/API changes * [OSDEV-1409](https://opensupplyhub.atlassian.net/browse/OSDEV-1409) - Introduced a new PATCH `/api/v1/moderation-events/{moderation_id}/production-locations/{os_id}/` endpoint. This endpoint allows the creation of a new contribution for an existing production location based on the provided moderation event. @@ -102,7 +111,7 @@ The page also features `Go Back` and `Submit` buttons for navigation and form su #### Migrations: * 0162_update_moderationevent_table_fields.py - This migration updates the ModerationEvent table and its constraints. -#### Scheme changes +#### Schema changes * [OSDEV-1158](https://opensupplyhub.atlassian.net/browse/OSDEV-1158) - The following updates to the ModerationEvent table have been made: 1. Set `uuid` as the primary key. 2. Make `geocode_result` field optional. It can be blank if lat and lng @@ -182,7 +191,7 @@ This issue has been fixed by adding additional requests to delete the appropriat * 0160_allow_null_parsing_errors_in_facilitylist.py - This migration allows empty parsing_errors in api_facilitylist. * 0161_create_disable_list_uploading_switch.py - This migration creates disable_list_uploading switch in the Django admin panel and record in the waffle_switch table. -#### Scheme changes +#### Schema changes * [OSDEV-1346](https://opensupplyhub.atlassian.net/browse/OSDEV-1346) - Alter status options for api_moderationevent table. * [OSDEV-1411](https://opensupplyhub.atlassian.net/browse/OSDEV-1411) - Allows empty parsing_errors in api_facilitylist. @@ -265,7 +274,7 @@ to modify moderation event `status`. #### Migrations: * 0158_create_moderation_events_table.py - This migration creates api_moderationevent table for Moderation Queue. -#### Scheme changes +#### Schema changes * [OSDEV-1229](https://opensupplyhub.atlassian.net/browse/OSDEV-1229) - Created Moderation Events Postgres table to track moderation events in the database. ### Code/API changes @@ -292,7 +301,7 @@ to modify moderation event `status`. * 0156_introduce_list_level_parsing_errors - This migration introduces the parsing_errors field for the FacilityList model to collect list-level and internal errors logged during the background parsing of the list. * 0157_delete_endpoint_switcher_for_list_uploads - This migration deletes the `use_old_upload_list_endpoint` switcher that was necessary to toggle between the old and new list upload endpoints. -#### Scheme changes +#### Schema changes * [OSDEV-1039](https://opensupplyhub.atlassian.net/browse/OSDEV-1039) - Since the `use_old_upload_list_endpoint` switcher is no longer necessary for the list upload, it has been deleted from the DB. Additionally, the `parsing_errors` field has been added to the FacilityList model. ### Code/API changes @@ -382,7 +391,7 @@ Note: This instruction updates item 3 of the ['Release to Production and Sandbox #### Migrations: * 0155_remove_verification_method_column_from_facility_claim - This migration replaces the old `index_approved_claim` function with a new one that does not index the `verification_method` and `phone_number` fields. Additionally, it removes the `verification_method` and `phone_number` fields from the FacilityClaim model and the respective history table. -#### Scheme changes +#### Schema changes * [OSDEV-1092](https://opensupplyhub.atlassian.net/browse/OSDEV-1092) - Since the `verification_method` and `phone_number` fields are no longer necessary for the claim form and aren't used anywhere in the codebase, they have been deleted from the FacilityClaim model and the respective history table. ### Code/API changes @@ -469,7 +478,7 @@ Note: This instruction updates item 3 of the ['Release to Production and Sandbox * 0153_add_sector_group_table - creates the `SectorGroup` model and populates it with the sector groups names. * 0154_associate_sectors_with_groups - associates sectors with sector groups. -#### Scheme changes +#### Schema changes * [OSDEV-1142](https://opensupplyhub.atlassian.net/browse/OSDEV-1142) - Technical Debt. Remove unused `api_tilecache` and `api_dynamicsetting` tables. Migration has been created, removed related data in the code base. * [OSDEV-360](https://opensupplyhub.atlassian.net/browse/OSDEV-360) - The following changes have been implemented: * A new table, `api_sectorgroup`, has been introduced and populated with sector group names. @@ -511,15 +520,6 @@ Note: This instruction updates item 3 of the ['Release to Production and Sandbox #### Migrations: * 0151_replace_index_number_of_workers - replace function `index_number_of_workers` to use one source of truth for both`number_of_workers` & `extended_fields`. -#### Scheme changes -* *Describe scheme changes here.* - -### Code/API changes -* *Describe code/API changes here.* - -### Architecture/Environment changes -* *Describe architecture/environment changes here.* - ### Bugfix * [OSDEV-1145](https://opensupplyhub.atlassian.net/browse/OSDEV-1145) - Error message appearing as red dot with no context. Error display has been fixed. Simplified displaying logic of errors. Changed error property type. * [OSDEV-576](https://opensupplyhub.atlassian.net/browse/OSDEV-576) - Implemented one source of truth to Search query source & Production Location Details page source for field `number_of_workers`. @@ -543,13 +543,6 @@ Note: This instruction updates item 3 of the ['Release to Production and Sandbox * Product name: Open Supply Hub * Release date: July 13, 2024 -### Database changes -#### Migrations: -* *Describe migrations here.* - -#### Scheme changes -* *Describe scheme changes here.* - ### Code/API changes * [OSDEV-1100](https://opensupplyhub.atlassian.net/browse/OSDEV-1100) - Replaced all mentions of "facility" and "facilities" with the new production location naming in the Logstash app. Renamed `location` field in the production locations index to `coordinates`. * [OSDEV-705](https://opensupplyhub.atlassian.net/browse/OSDEV-705) - Created an additional `RowCoordinatesSerializer` in the ContriCleaner to handle coordinate values ("lat" and "lng"). Moved the conversion of "lat" and "lng" into float point numbers from `FacilityListViewSet` to this serializer. @@ -635,7 +628,7 @@ drop_calc_column_func. * 0148_remove_facility_workers_count_field_from_facilityclaim - removes the facility_workers_count field from the FacilityClaim model. * 0149_rename_facility_workers_count_new_to_facility_workers_count - renames the facility_workers_count_new field to facility_workers_count. -#### Scheme changes +#### Schema changes * [OSDEV-1084](https://opensupplyhub.atlassian.net/browse/OSDEV-1084) - To enable adding a range for the number of workers during the claiming process, the type of the `facility_workers_count` field in the `FacilityClaim` table was changed from `IntegerField` to `CharField`. ### Architecture/Environment changes @@ -735,7 +728,7 @@ and receives information about merges that have occurred for the contributors wi * 0143_create_facility_claim_attachment_table.py - create api_facilityclaimattachments table to store claimant attachments per facility claim * 0144_remove_unnecessary_columns_from_facility_claim.py - This migration replaces the old `index_approved_claim` function with a similar one that does not index the `preferred_contact_method` field. Additionally, the migration removes `email` and `preferred_contact_method` from the `FacilityClaim` model and the respective history table. -#### Scheme changes +#### Schema changes * [OSDEV-931](https://opensupplyhub.atlassian.net/browse/OSDEV-931) - Since `email` and `preferred_contact_method` are no longer necessary for the claim form, they have been removed from the `FacilityClaim` model and the respective history table. Additionally, the old `index_approved_claim` function has been replaced with a similar one that does not index the `preferred_contact_method` field. ### Code/API changes @@ -847,7 +840,7 @@ database: * 0141_delete_contributor_webhooks.py - deletes `ContributorWebhook` model * 0142_introduce_temporary_endpoint_switcher_for_list_uploads.py - This migration introduces a temporary API endpoint switcher for list uploads. -#### Scheme changes +#### Schema changes * [OSDEV-893](https://opensupplyhub.atlassian.net/browse/OSDEV-893) - Introduce a temporary API endpoint switcher for list uploads to enable switching to the old list upload API endpoint if the new endpoint affects production uptime. ### Code/API changes @@ -904,7 +897,7 @@ Updated existing users api_apilimit records renewal_period value. * 0139_remove_ppe_switch.py - This migration removes the ppe switch. * 0140_remove_indexing_ppe_fields.py - This migration updates indexing functions to not index PPE fields. -#### Scheme changes +#### Schema changes * [OSDEV-835](https://opensupplyhub.atlassian.net/browse/OSDEV-835) - Since the FacilityIndex model is primarily used to store cached facility data and display it publicly via the `/facilities/{id}` API endpoint, only public data can be shown. Therefore, caching emails to the FacilityIndex model was removed from the PostgreSQL indexing functions. All instances where emails are publicly displayed have been removed. The only remaining field is `ppe_contact_email`, but all functionality and code related to PPE will be deleted in this [OSDEV-562](https://opensupplyhub.atlassian.net/browse/OSDEV-562) ticket. * [OSDEV-562](https://opensupplyhub.atlassian.net/browse/OSDEV-562) - Remove PPE fields (ppe_product_types, ppe_contact_email, ppe_contact_phone, ppe_website, ppe) from the `api_facility`, `api_facilityindex`, `api_facilitylistitem`, `api_facilitylistitemtemp`, `api_historicalfacility`. Remove this fields from indexing processes. @@ -1017,7 +1010,7 @@ Updated existing users api_apilimit records renewal_period value. * 0132_add_moderation_mode_field - This migration adds the field `is_moderation_mode` to table `api_user`. * 0133_introduce_tile_caching - This migration creates the TileCache table and the DynamicSetting table. This migration is reversible. -#### Scheme changes +#### Schema changes * [OSDEV-622](https://opensupplyhub.atlassian.net/browse/OSDEV-622) - Separate data-gathering functions were created for the `api_facilityindexnew` table columns to collect data independently of the main procedure. The `index_facilities` and `index_facilities_by` procedures were updated to use new separate functions for collecting data for the `api_facilityindexnew` table columns that require long SQL queries. * [OSDEV-595](https://opensupplyhub.atlassian.net/browse/OSDEV-595) - Rename FacilityIndexNew to FacilityIndex * [OSDEV-623](https://opensupplyhub.atlassian.net/browse/OSDEV-623), [OSDEV-624](https://opensupplyhub.atlassian.net/browse/OSDEV-624), [OSDEV-638](https://opensupplyhub.atlassian.net/browse/OSDEV-638) - New SQL triggers have been introduced to handle changes in the `api_contributor`, `api_extendedfield`, `api_facility`, `api_facilityclaim`, `api_facilitylistitem`, `api_facilitymatch`, `api_source`, and `api_facilitylist` tables at the database level. This change is essential for the future functionality of DedupeHub, which will communicate directly with the database. All the Django signals have been removed. Additionally, reindexing of the necessary columns of the index table has been transferred to these triggers, eliminating the need for the large SQL procedure previously used in conjunction with Django signals. @@ -1068,7 +1061,7 @@ Fix issue with exceeding API requests. [OSDEV-557](https://opensupplyhub.atlassi #### Migrations: - 0130_facility_index_gin_index - implement indexes for fields on "api_facilityindexnew" table related to tile generation -#### Scheme changes +#### Schema changes * indexing fields in api_facilityindexnew * contrib_types * contributors_id @@ -1098,7 +1091,7 @@ Fix issue with exceeding API requests. [OSDEV-557](https://opensupplyhub.atlassi - 0128_custom_text_implementation - creates custom_text SQL functions and updated index_facilities and index_facilities_by to use it - 0129_delete_facility_index - removes api_facilityindex table -#### Scheme changes +#### Schema changes * introduce fields to api_facility_list_items * raw_json:JSON * raw_header:Text diff --git a/src/anon-tools/Dockerfile.dump b/src/anon-tools/Dockerfile.dump index 52478c73e..8a0d946d4 100644 --- a/src/anon-tools/Dockerfile.dump +++ b/src/anon-tools/Dockerfile.dump @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ @@ -31,9 +31,8 @@ RUN chmod 644 ~/.ssh/known_hosts COPY ./do_dump.sh ./do_dump.sh -COPY ./initdb.sql /docker-entrypoint-initdb.d VOLUME /keys -# ENTRYPOINT [ "docker-entrypoint.sh" ] + CMD ["sh", "do_dump.sh"] diff --git a/src/anon-tools/Dockerfile.restore b/src/anon-tools/Dockerfile.restore index a8fb26843..7236a5f5c 100644 --- a/src/anon-tools/Dockerfile.restore +++ b/src/anon-tools/Dockerfile.restore @@ -1,4 +1,4 @@ -FROM postgis/postgis:12-3.4-alpine +FROM postgis/postgis:13-3.4-alpine WORKDIR /opt/ diff --git a/src/anon-tools/anon.sql b/src/anon-tools/anon.sql deleted file mode 100644 index d71a213d3..000000000 --- a/src/anon-tools/anon.sql +++ /dev/null @@ -1,35 +0,0 @@ --- DECLARE --- current_table text; --- column_exists_email boolean; --- column_exists_username boolean; --- column_exists_password boolean; --- column_exists_phone_number boolean; --- BEGIN --- FOR current_table IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') --- LOOP --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''email'')' INTO column_exists_email; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''username'')' INTO column_exists_username; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''password'')' INTO column_exists_password; --- EXECUTE 'SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = ''' || current_table || ''' AND column_name = ''phone_number'')' INTO column_exists_phone_number; - --- IF column_exists_email THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- email = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) || ''@'' || substring(email from position(''@'' in email) + 1) ELSE email END'; - --- IF column_exists_username THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- username = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN substr(md5(random()::text), 1, 20) ELSE username END'; --- END IF; - --- IF column_exists_password THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- password = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) ELSE password END'; --- END IF; - --- IF column_exists_phone_number THEN --- EXECUTE 'UPDATE ' || current_table || ' SET --- phone_number = CASE WHEN email NOT LIKE ''%@speedandfunction.com'' AND email NOT LIKE ''%@opensupplyhub.org'' THEN md5(random()::text) ELSE phone_number END'; --- END IF; --- END IF; --- END LOOP; --- END; diff --git a/src/anon-tools/do_dump.sh b/src/anon-tools/do_dump.sh index bc5bd0ab7..90720462b 100755 --- a/src/anon-tools/do_dump.sh +++ b/src/anon-tools/do_dump.sh @@ -69,7 +69,7 @@ pg_dump --clean --no-owner --no-privileges -Fc -d anondb -U anondb -f /dumps/os ls -la /dumps -echo "Finshed anonymization" +echo "Finished anonymization" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_TEST \ AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_TEST \ diff --git a/src/anon-tools/initdb.sql b/src/anon-tools/initdb.sql deleted file mode 100644 index 4b2e2f148..000000000 --- a/src/anon-tools/initdb.sql +++ /dev/null @@ -1,20 +0,0 @@ --- CREATE EXTENSION plpgsql; -CREATE EXTENSION btree_gin; -CREATE EXTENSION pg_trgm; -CREATE EXTENSION pgcrypto; -CREATE EXTENSION unaccent; --- CREATE EXTENSION postgis; - --- CREATE EXTENSION postgis; --- -- enabling raster support --- CREATE EXTENSION postgis_raster; - --- -- enabling advanced 3d support --- CREATE EXTENSION postgis_sfcgal; --- -- enabling SQL/MM Net Topology --- CREATE EXTENSION postgis_topology; - --- -- using US census data for geocoding and standardization --- CREATE EXTENSION address_standardizer; --- CREATE EXTENSION fuzzystrmatch; --- CREATE EXTENSION postgis_tiger_geocoder; diff --git a/src/database/Dockerfile.local b/src/database/Dockerfile.local index 241989ddd..dca21dceb 100644 --- a/src/database/Dockerfile.local +++ b/src/database/Dockerfile.local @@ -1,4 +1,11 @@ -FROM postgres:12.17 +# The latest supported version of PostgreSQL in AWS RDS is 13.15. +# However, the official PostgreSQL Docker registry provides only +# the latest minor version for PostgreSQL 13, and it is not +# possible to specify a specific minor version like 13.15. Therefore, +# it was decided to provide only the major version and let Docker +# determine which minor version to install. This should not cause +# incompatibilities, even if PostgreSQL 13.15 is used in AWS. +FROM postgres:13 ENV POSTGRES_USER=opensupplyhub \ POSTGRES_PASSWORD=opensupplyhub \ @@ -6,7 +13,7 @@ ENV POSTGRES_USER=opensupplyhub \ # Build and install the PostGIS package from scratch to ensure it matches the -# LLVM version used by the Docker container based on postgres:12.17 image. +# LLVM version used by the Docker container based on postgres:13 image. # See PostGIS installation details at https://postgis.net/docs/manual-3.4/postgis_installation.html # Install the requirements for building and using PostGIS. @@ -21,15 +28,15 @@ RUN apt-get update && apt-get install -y \ libprotobuf-c-dev \ protobuf-c-compiler \ pkg-config \ - postgresql-server-dev-12 + postgresql-server-dev-13 # Download the PostGIS source archive, build and install the PostGIS package. -RUN wget http://download.osgeo.org/postgis/source/postgis-3.4.0.tar.gz && \ - tar -xvzf postgis-3.4.0.tar.gz && \ - cd /postgis-3.4.0 && \ +RUN wget https://download.osgeo.org/postgis/source/postgis-3.4.2.tar.gz && \ + tar -xvzf postgis-3.4.2.tar.gz && \ + cd /postgis-3.4.2 && \ ./configure && \ make && \ make install && \ cd .. && \ - rm -rf postgis-3.4.0 && \ - rm postgis-3.4.0.tar.gz + rm -rf postgis-3.4.2 && \ + rm postgis-3.4.2.tar.gz diff --git a/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py b/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py new file mode 100644 index 000000000..5db520f23 --- /dev/null +++ b/src/django/api/migrations/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.py @@ -0,0 +1,51 @@ +# Generated by Django 3.2.17 on 2025-01-14 11:17 + +from django.db.migrations import Migration, RunPython +from django.db import connection +from api.migrations._migration_helper import MigrationHelper + +helper = MigrationHelper(connection) + + +def refresh_pg_statistic_and_perform_upgrading_pg_extensions( + apps, schema_editor): + helper.run_sql_files([ + '0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql' + ]) + + +class Migration(Migration): + ''' + This migration refreshes the pg_statistic table and upgrades the + PostgreSQL database extensions. + + Since this migration will be executed after the PostgreSQL major version + upgrade, the ANALYZE operation should be run to refresh the pg_statistic + table. Optimizer statistics aren't transferred during a major version + upgrade, so it is necessary to regenerate all statistics to avoid + performance issues. + + Currently, the database uses the following extensions: + 1. postgis + 2. unaccent + 3. pg_trgm + 4. plpgsql + 5. btree_gin + 6. pgcrypto + + Based on the available extension versions for PostgreSQL 13.15 in AWS RDS, + which will be used across all AWS environments after the database upgrade + to Postgres 13, it was found that the `postgis` extension can be upgraded + to version 3.4.2 in Production and Staging. Additionally, the `pg_trgm` + extension can be upgraded to version 1.5 in Development, Test, Production, + and Staging. If the specified versions are already installed in the + database, there will be no issues. + ''' + + dependencies = [ + ('api', '0162_update_moderationevent_table_fields'), + ] + + operations = [ + RunPython(refresh_pg_statistic_and_perform_upgrading_pg_extensions) + ] diff --git a/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql b/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql new file mode 100644 index 000000000..7c4045c6f --- /dev/null +++ b/src/django/sqls/0163_refresh_pg_statistic_and_upgrade_postgres_extensions.sql @@ -0,0 +1,4 @@ +ANALYZE VERBOSE; + +ALTER EXTENSION postgis UPDATE TO '3.4.2'; +ALTER EXTENSION pg_trgm UPDATE TO '1.5';