[OSDEV-1514] Database. Upgrade the PostgreSQL version to 13 (#479)

- Upgraded the PostgreSQL version from 12 to 13 for the database used in
local development, DB anonymization, DB restore setup, and environments
in the AWS cloud. Additionally, the postgis and pg_trgm extensions have
been upgraded to versions 3.4.2 and 1.5, respectively, based on the
available extension versions for PostgreSQL 13.15 in AWS RDS. For more
information, see [Extensions supported for RDS for PostgreSQL
13](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html#postgresql-extensions-13x).
Allowed major version upgrades and activated the `apply immediately`
flag to perform the PostgreSQL major version upgrade in AWS.
- Corrected spelling mistakes in the `src/anon-tools/do_dump.sh` file
and in the name of the folder `database_anonymizer_sheduled_task`.
Removed the unused `src/anon-tools/anon.sql` file and the redundant
`src/anon-tools/initdb.sql` file. Removed commented-out code in the
`src/anon-tools/Dockerfile.dump` and
`deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py`
files.
- Introduced `rds_allow_major_version_upgrade` and
`rds_apply_immediately` Terraform variables to enable or disable major
version upgrades and the `apply immediately` flag, depending on the
environment.

.github/workflows/deploy_to_aws.yml

@@ -224,8 +224,8 @@ jobs:
         uses: docker/build-push-action@v2
         if: ${{ steps.get_env_name.outputs.lowercase == 'production' }}
         with:
-          context: deployment/terraform/database_anonymizer_sheduled_task/docker
-          file: deployment/terraform/database_anonymizer_sheduled_task/docker/Dockerfile
+          context: deployment/terraform/database_anonymizer_scheduled_task/docker
+          file: deployment/terraform/database_anonymizer_scheduled_task/docker/Dockerfile
           push: true
           tags: ${{ vars.ECR_REGISTRY }}/${{ vars.IMAGE_NAME }}-database-anonymizer-${{ steps.get_env_name.outputs.lowercase }}:${{ env.GIT_COMMIT }}
 

deployment/environments/terraform-development.tfvars

@@ -14,15 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.micro"
 rds_database_identifier = "opensupplyhub-enc-stg"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
-snapshot_identifier   = ""
-rds_deletion_protection = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "1"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-preprod.tfvars

@@ -14,8 +14,8 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-pp"
 rds_database_name = "opensupplyhub"

deployment/environments/terraform-production.tfvars

@@ -13,13 +13,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.m6in.4xlarge"
 rds_database_identifier = "opensupplyhub-enc-prd"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "10"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-staging.tfvars

@@ -12,13 +12,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "128"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.large"
 rds_database_identifier = "opensupplyhub-enc-stg"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 app_ecs_desired_count = "4"
 app_ecs_deployment_min_percent = "100"

deployment/environments/terraform-test.tfvars

@@ -14,13 +14,15 @@ bastion_ami = "ami-0bb3fad3c0286ebd5"
 bastion_instance_type = "t3.nano"
 
 rds_allocated_storage = "256"
-rds_engine_version = "12"
-rds_parameter_group_family = "postgres12"
+rds_engine_version = "13"
+rds_parameter_group_family = "postgres13"
 rds_instance_type = "db.t3.xlarge"
 rds_database_identifier = "opensupplyhub-enc-tst"
 rds_database_name = "opensupplyhub"
 rds_multi_az = false
 rds_storage_encrypted = true
+rds_allow_major_version_upgrade = true
+rds_apply_immediately = true
 
 anonymized_database_instance_type = "db.t3.2xlarge"
 anonymized_database_identifier = "database-anonymizer"

deployment/terraform/anonymize_db_job.tf

@@ -1,7 +1,7 @@
 module "database_anonymizer" {
   count = var.database_anonymizer_enabled == true ? 1 : 0
 
-  source = "./database_anonymizer_sheduled_task"
+  source = "./database_anonymizer_scheduled_task"
 
   rds_database_identifier       = var.rds_database_identifier
   rds_database_name             = var.rds_database_name

deployment/terraform/anonymized_database_dump_scheduled_task/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgis/postgis:12-3.4-alpine
+FROM postgis/postgis:13-3.4-alpine
 
 WORKDIR /opt/
 

deployment/terraform/database.tf

@@ -76,30 +76,32 @@ resource "aws_db_parameter_group" "default" {
 }
 
 module "database_enc" {
-  source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.0.3"
-
-  vpc_id                     = module.vpc.id
-  allocated_storage          = var.rds_allocated_storage
-  engine_version             = var.rds_engine_version
-  instance_type              = var.rds_instance_type
-  storage_type               = var.rds_storage_type
-  database_identifier        = var.rds_database_identifier
-  database_name              = var.rds_database_name
-  database_username          = var.rds_database_username
-  database_password          = var.rds_database_password
-  backup_retention_period    = var.rds_backup_retention_period
-  backup_window              = var.rds_backup_window
-  maintenance_window         = var.rds_maintenance_window
-  auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade
-  final_snapshot_identifier  = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())])
-  skip_final_snapshot        = var.rds_skip_final_snapshot
-  copy_tags_to_snapshot      = var.rds_copy_tags_to_snapshot
-  multi_availability_zone    = var.rds_multi_az
-  storage_encrypted          = var.rds_storage_encrypted
-  subnet_group               = aws_db_subnet_group.default.name
-  parameter_group            = aws_db_parameter_group.default.name
-  deletion_protection        = var.rds_deletion_protection
-  snapshot_identifier        = var.snapshot_identifier
+  source = "github.com/opensupplyhub/terraform-aws-postgresql-rds?ref=3.1.0"
+
+  vpc_id                      = module.vpc.id
+  allocated_storage           = var.rds_allocated_storage
+  engine_version              = var.rds_engine_version
+  instance_type               = var.rds_instance_type
+  storage_type                = var.rds_storage_type
+  database_identifier         = var.rds_database_identifier
+  database_name               = var.rds_database_name
+  database_username           = var.rds_database_username
+  database_password           = var.rds_database_password
+  backup_retention_period     = var.rds_backup_retention_period
+  backup_window               = var.rds_backup_window
+  maintenance_window          = var.rds_maintenance_window
+  auto_minor_version_upgrade  = var.rds_auto_minor_version_upgrade
+  allow_major_version_upgrade = var.rds_allow_major_version_upgrade
+  apply_immediately           = var.rds_apply_immediately
+  final_snapshot_identifier   = join("-", [var.rds_final_snapshot_identifier, formatdate("YYYYMMDDhhmmss", timestamp())])
+  skip_final_snapshot         = var.rds_skip_final_snapshot
+  copy_tags_to_snapshot       = var.rds_copy_tags_to_snapshot
+  multi_availability_zone     = var.rds_multi_az
+  storage_encrypted           = var.rds_storage_encrypted
+  subnet_group                = aws_db_subnet_group.default.name
+  parameter_group             = aws_db_parameter_group.default.name
+  deletion_protection         = var.rds_deletion_protection
+  snapshot_identifier         = var.snapshot_identifier
 
   alarm_cpu_threshold                = var.rds_cpu_threshold_percent
   alarm_disk_queue_threshold         = var.rds_disk_queue_threshold

deployment/terraform/database_anonymizer_sheduled_task/docker/database_anonymizer.py → deployment/terraform/database_anonymizer_scheduled_task/docker/database_anonymizer.py

@@ -77,9 +77,6 @@
 )
 
 db = pg8000.native.Connection(**connection_information)
-# cur = db.cursor()
-# cur.execute(open("anonymize_script.sql", "r").read())
-# cur.commit()
 db.run(open("anonymize_script.sql", "r").read())
 print('Database anonymized successfully!')
 

deployment/terraform/variables.tf

@@ -70,11 +70,11 @@ variable "rds_allocated_storage" {
 }
 
 variable "rds_engine_version" {
-  default = "12.4"
+  default = "13"
 }
 
 variable "rds_parameter_group_family" {
-  default = "postgres12"
+  default = "postgres13"
 }
 
 variable "rds_instance_type" {
@@ -114,6 +114,18 @@ variable "rds_auto_minor_version_upgrade" {
   default = true
 }
 
+variable "rds_allow_major_version_upgrade" {
+  default     = false
+  type        = bool
+  description = "Indicates that major PostgreSQL engine version upgrades are allowed."
+}
+
+variable "rds_apply_immediately" {
+  default     = false
+  type        = bool
+  description = "Specifies whether any database modifications are applied immediately, or during the next maintenance window."
+}
+
 variable "rds_final_snapshot_identifier" {
   default = "osh-rds-snapshot"
 }

doc/release/RELEASE-NOTES-TEMPLATE.md

@@ -7,11 +7,13 @@ Use the format below to document information about the new release.
 * Release date: *Provide release date*
 
 ### Database changes
+* *Describe high-level database changes.*
+
 #### Migrations:
 * *Describe migrations here.*
 
-#### Scheme changes
-* *Describe scheme changes here.*
+#### Schema changes
+* *Describe schema changes here.*
 
 ### Code/API changes
 * *Describe code/API changes here.*