From ab3c2474d2f9b65b3e8f61cc5831d15322f19469 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Wed, 14 Feb 2024 15:02:35 -0500 Subject: [PATCH 1/4] Add change log for Repeat Records Couch to SQL migration --- changelog/0077-sql-repeatrecord-pr1.yml | 103 +++++++++++++++++ .../changelog/0077-sql-repeatrecord-pr1.md | 109 ++++++++++++++++++ docs/source/changelog/index.md | 5 + 3 files changed, 217 insertions(+) create mode 100644 changelog/0077-sql-repeatrecord-pr1.yml create mode 100644 docs/source/changelog/0077-sql-repeatrecord-pr1.md diff --git a/changelog/0077-sql-repeatrecord-pr1.yml b/changelog/0077-sql-repeatrecord-pr1.yml new file mode 100644 index 0000000000..f65a6faff2 --- /dev/null +++ b/changelog/0077-sql-repeatrecord-pr1.yml @@ -0,0 +1,103 @@ +title: 'SQL Repat Record Migration' +key: sql-repeatrecord-pr1 +date: 2024-02-13 +optional_per_env: yes +# (optional) Min version of HQ that MUST be deployed before this change can be rolled out (commit hash) +min_commcare_version: TODO +# (optional) Max version of HQ that can be deployed before this change MUST be rolled out (commit hash) +max_commcare_version: +context: | + Prepare for and migrate Repeat Records from Couch to SQL. + +details: | + This is an optional procedure that should be done if you use data forwarding + and want to ensure a future deploy is not blocked on needing to perform manual + migration steps. + +update_steps: | + Run the following command to determine how many repeat records you have to + migrate (in all commands below, replace `` with your environment name): + + ```sh + cchq django-manage shell -c ' + from corehq.motech.repeaters.management.commands.populate_repeatrecords import count_docs; + print("Records to migrate:", count_docs())' + ``` + + The output should include something like the following: + + ``` + Records to migrate: 125011 + ``` + + A future deploy will be blocked, requiring manual migration steps, if the + number of records to migrate is more than 1000. These steps can be skipped + and the migration will be performed automatically during a future deploy if + the number is less than 1000 (as counted at deployment time). It is safe and + recommended to perform these steps even if the number is less than 1000. + + Before proceeding, **make a full backup of your CommCare HQ data**. [Backup + procedures](https://commcare-cloud.readthedocs.io/en/latest/backups_dr/2-backups-guide.html) + are outside the scope of this document. + + Next, estimate the amount of free space that will be required for new data in + PostgreSQL (main or repeaters database, depending on configuration): + + ```sh + cchq django-manage shell -c ' + from corehq.motech.repeaters.models import RepeatRecord; + size = RepeatRecord.get_db().info()["sizes"]["active"]; + mult = int((int.bit_length(size) - 1) / 10); + unit = " kMGT"[mult].strip(); + print(f"Couch DB file size: {size / (1024 ** mult):.2f} {unit}B")' + ``` + + The output will include something like: + + ``` + Couch DB file size: 8.33 GB + ``` + + Ensure there is at least 1.5 to 2 times that amount of free space available on + the machine hosting the main PostgreSQL database in your environment. That is, + unless you have configured CommCare HQ to store repeaters in a separate + database, in which case the extra space should be available to the repeaters + database. + + Run the following commands to perform the migration. + + ```sh + # Create a private release in which to run management commands. + # Make a note of the release directory path. + cchq deploy commcare --private --keep-days=30 + + # Start a screen session + cchq ssh django_manage -t screen -S couch2sql + + # If the connection drops, reconnect with + cchq ssh django_manage -t screen -x couch2sql + + # In the screen session + sudo -iu cchq + mkdir ~/couch2sql + cd # created above + + # Run the migration command + ./manage.py populate_repeatrecords --log-path ~/couch2sql + # Re-run this command until it completes successfully (exit code 0). + + # After successful completion, run again with --fixup-diffs=... if directed + # to do so by command output. + ``` + + DO NOT delete repeat records from Couch DB yet, they are still needed! A + future change log entry will instruct when it is safe to to that. + + If you do not want to migrate historical repeat records from Couch you can + make it appear as if the migration has completed without actually migrating + anything. Use the following command to do that: + + ```sh + # WARNING do not do this unless you want to discard repeat records in Couch + cchq django-manage populate_repeatrecords --override-is-migration-completed yes + ``` diff --git a/docs/source/changelog/0077-sql-repeatrecord-pr1.md b/docs/source/changelog/0077-sql-repeatrecord-pr1.md new file mode 100644 index 0000000000..6ad0df9923 --- /dev/null +++ b/docs/source/changelog/0077-sql-repeatrecord-pr1.md @@ -0,0 +1,109 @@ + + +# 77. SQL Repat Record Migration + +**Date:** 2024-02-13 + +**Optional per env:** _only required on some environments_ + + +## CommCare Version Dependency +The following version of CommCare must be deployed before rolling out this change: +[TODO](https://github.com/dimagi/commcare-hq/commit/TODO) + + +## Change Context +Prepare for and migrate Repeat Records from Couch to SQL. + +## Details +This is an optional procedure that should be done if you use data forwarding +and want to ensure a future deploy is not blocked on needing to perform manual +migration steps. + +## Steps to update +Run the following command to determine how many repeat records you have to +migrate (in all commands below, replace `` with your environment name): + +```sh +cchq django-manage shell -c ' +from corehq.motech.repeaters.management.commands.populate_repeatrecords import count_docs; +print("Records to migrate:", count_docs())' +``` + +The output should include something like the following: + +``` +Records to migrate: 125011 +``` + +A future deploy will be blocked, requiring manual migration steps, if the +number of records to migrate is more than 1000. These steps can be skipped +and the migration will be performed automatically during a future deploy if +the number is less than 1000 (as counted at deployment time). It is safe and +recommended to perform these steps even if the number is less than 1000. + +Before proceeding, **make a full backup of your CommCare HQ data**. [Backup +procedures](https://commcare-cloud.readthedocs.io/en/latest/backups_dr/2-backups-guide.html) +are outside the scope of this document. + +Next, estimate the amount of free space that will be required for new data in +PostgreSQL (main or repeaters database, depending on configuration): + +```sh +cchq django-manage shell -c ' +from corehq.motech.repeaters.models import RepeatRecord; +size = RepeatRecord.get_db().info()["sizes"]["active"]; +mult = int((int.bit_length(size) - 1) / 10); +unit = " kMGT"[mult].strip(); +print(f"Couch DB file size: {size / (1024 ** mult):.2f} {unit}B")' +``` + +The output will include something like: + +``` +Couch DB file size: 8.33 GB +``` + +Ensure there is at least 1.5 to 2 times that amount of free space available on +the machine hosting the main PostgreSQL database in your environment. That is, +unless you have configured CommCare HQ to store repeaters in a separate +database, in which case the extra space should be available to the repeaters +database. + +Run the following commands to perform the migration. + +```sh +# Create a private release in which to run management commands. +# Make a note of the release directory path. +cchq deploy commcare --private --keep-days=30 + +# Start a screen session +cchq ssh django_manage -t screen -S couch2sql + +# If the connection drops, reconnect with +cchq ssh django_manage -t screen -x couch2sql + +# In the screen session +sudo -iu cchq +mkdir ~/couch2sql +cd # created above + +# Run the migration command +./manage.py populate_repeatrecords --log-path ~/couch2sql +# Re-run this command until it completes successfully (exit code 0). + +# After successful completion, run again with --fixup-diffs=... if directed +# to do so by command output. +``` + +DO NOT delete repeat records from Couch DB yet, they are still needed! A +future change log entry will instruct when it is safe to to that. + +If you do not want to migrate historical repeat records from Couch you can +make it appear as if the migration has completed without actually migrating +anything. Use the following command to do that: + +```sh +# WARNING do not do this unless you want to discard repeat records in Couch +cchq django-manage populate_repeatrecords --override-is-migration-completed yes +``` diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md index d9751da51d..4d7d81e932 100644 --- a/docs/source/changelog/index.md +++ b/docs/source/changelog/index.md @@ -7,6 +7,11 @@ need to be applied on your environment to keep it up to date. ### Changelog +#### **2024-02-13** [SQL Repat Record Migration](0077-sql-repeatrecord-pr1.md) +Prepare for and migrate Repeat Records from Couch to SQL. + + +--- #### **2024-01-14** [Elasticsearch upgrade from 2.4.6 to 5.6.16](0076-upgrade-to-es-5.md) Upgrade to Elasticsearch 5. From 195ffe377b0e1af21c791205a1b29978eabd312d Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 18 Mar 2024 16:48:19 -0400 Subject: [PATCH 2/4] Fix typos and improve wording --- changelog/0077-sql-repeatrecord-pr1.yml | 17 +++++++++-------- .../changelog/0077-sql-repeatrecord-pr1.md | 17 +++++++++-------- docs/source/changelog/index.md | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/changelog/0077-sql-repeatrecord-pr1.yml b/changelog/0077-sql-repeatrecord-pr1.yml index f65a6faff2..20330284f1 100644 --- a/changelog/0077-sql-repeatrecord-pr1.yml +++ b/changelog/0077-sql-repeatrecord-pr1.yml @@ -1,4 +1,4 @@ -title: 'SQL Repat Record Migration' +title: 'SQL Repeat Record Migration' key: sql-repeatrecord-pr1 date: 2024-02-13 optional_per_env: yes @@ -30,11 +30,12 @@ update_steps: | Records to migrate: 125011 ``` - A future deploy will be blocked, requiring manual migration steps, if the - number of records to migrate is more than 1000. These steps can be skipped - and the migration will be performed automatically during a future deploy if - the number is less than 1000 (as counted at deployment time). It is safe and - recommended to perform these steps even if the number is less than 1000. + If the number of records to migrate is more than 1000 and the steps below are + not done, a future deploy will be blocked, at which point this procedure will + be required to proceed with the deploy. These steps can be skipped and the + migration will be performed automatically during a future deploy if the number + is less than 1000 (as counted at deployment time). It is safe and recommended + to perform these steps even if the number is less than 1000. Before proceeding, **make a full backup of your CommCare HQ data**. [Backup procedures](https://commcare-cloud.readthedocs.io/en/latest/backups_dr/2-backups-guide.html) @@ -48,7 +49,7 @@ update_steps: | from corehq.motech.repeaters.models import RepeatRecord; size = RepeatRecord.get_db().info()["sizes"]["active"]; mult = int((int.bit_length(size) - 1) / 10); - unit = " kMGT"[mult].strip(); + unit = " KMGT"[mult].strip(); print(f"Couch DB file size: {size / (1024 ** mult):.2f} {unit}B")' ``` @@ -91,7 +92,7 @@ update_steps: | ``` DO NOT delete repeat records from Couch DB yet, they are still needed! A - future change log entry will instruct when it is safe to to that. + future change log entry will instruct when it is safe to do that. If you do not want to migrate historical repeat records from Couch you can make it appear as if the migration has completed without actually migrating diff --git a/docs/source/changelog/0077-sql-repeatrecord-pr1.md b/docs/source/changelog/0077-sql-repeatrecord-pr1.md index 6ad0df9923..19d42d9128 100644 --- a/docs/source/changelog/0077-sql-repeatrecord-pr1.md +++ b/docs/source/changelog/0077-sql-repeatrecord-pr1.md @@ -1,6 +1,6 @@ -# 77. SQL Repat Record Migration +# 77. SQL Repeat Record Migration **Date:** 2024-02-13 @@ -36,11 +36,12 @@ The output should include something like the following: Records to migrate: 125011 ``` -A future deploy will be blocked, requiring manual migration steps, if the -number of records to migrate is more than 1000. These steps can be skipped -and the migration will be performed automatically during a future deploy if -the number is less than 1000 (as counted at deployment time). It is safe and -recommended to perform these steps even if the number is less than 1000. +If the number of records to migrate is more than 1000 and the steps below are +not done, a future deploy will be blocked, at which point this procedure will +be required to proceed with the deploy. These steps can be skipped and the +migration will be performed automatically during a future deploy if the number +is less than 1000 (as counted at deployment time). It is safe and recommended +to perform these steps even if the number is less than 1000. Before proceeding, **make a full backup of your CommCare HQ data**. [Backup procedures](https://commcare-cloud.readthedocs.io/en/latest/backups_dr/2-backups-guide.html) @@ -54,7 +55,7 @@ cchq django-manage shell -c ' from corehq.motech.repeaters.models import RepeatRecord; size = RepeatRecord.get_db().info()["sizes"]["active"]; mult = int((int.bit_length(size) - 1) / 10); -unit = " kMGT"[mult].strip(); +unit = " KMGT"[mult].strip(); print(f"Couch DB file size: {size / (1024 ** mult):.2f} {unit}B")' ``` @@ -97,7 +98,7 @@ cd # created above ``` DO NOT delete repeat records from Couch DB yet, they are still needed! A -future change log entry will instruct when it is safe to to that. +future change log entry will instruct when it is safe to do that. If you do not want to migrate historical repeat records from Couch you can make it appear as if the migration has completed without actually migrating diff --git a/docs/source/changelog/index.md b/docs/source/changelog/index.md index 4d7d81e932..65b63bb1cf 100644 --- a/docs/source/changelog/index.md +++ b/docs/source/changelog/index.md @@ -7,7 +7,7 @@ need to be applied on your environment to keep it up to date. ### Changelog -#### **2024-02-13** [SQL Repat Record Migration](0077-sql-repeatrecord-pr1.md) +#### **2024-02-13** [SQL Repeat Record Migration](0077-sql-repeatrecord-pr1.md) Prepare for and migrate Repeat Records from Couch to SQL. From 97c95bce979c5a08782e2bc11a3cd8f8fb5c007d Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Tue, 19 Mar 2024 10:36:12 -0400 Subject: [PATCH 3/4] Add min commcare version --- changelog/0077-sql-repeatrecord-pr1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/0077-sql-repeatrecord-pr1.yml b/changelog/0077-sql-repeatrecord-pr1.yml index 20330284f1..4881efe6df 100644 --- a/changelog/0077-sql-repeatrecord-pr1.yml +++ b/changelog/0077-sql-repeatrecord-pr1.yml @@ -3,7 +3,7 @@ key: sql-repeatrecord-pr1 date: 2024-02-13 optional_per_env: yes # (optional) Min version of HQ that MUST be deployed before this change can be rolled out (commit hash) -min_commcare_version: TODO +min_commcare_version: 090a403508e60dc36fc3a61f2b975fb53adce616 # (optional) Max version of HQ that can be deployed before this change MUST be rolled out (commit hash) max_commcare_version: context: | From daca9a58afff8a7f30957f65968b851010f3bcaa Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Tue, 19 Mar 2024 10:38:36 -0400 Subject: [PATCH 4/4] make docs --- docs/source/changelog/0077-sql-repeatrecord-pr1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/changelog/0077-sql-repeatrecord-pr1.md b/docs/source/changelog/0077-sql-repeatrecord-pr1.md index 19d42d9128..d19569ed92 100644 --- a/docs/source/changelog/0077-sql-repeatrecord-pr1.md +++ b/docs/source/changelog/0077-sql-repeatrecord-pr1.md @@ -9,7 +9,7 @@ ## CommCare Version Dependency The following version of CommCare must be deployed before rolling out this change: -[TODO](https://github.com/dimagi/commcare-hq/commit/TODO) +[090a4035](https://github.com/dimagi/commcare-hq/commit/090a403508e60dc36fc3a61f2b975fb53adce616) ## Change Context