From f272861821cc67b9f7a106bb063263b56df6d00f Mon Sep 17 00:00:00 2001 From: NedPK Date: Mon, 2 Sep 2024 19:37:44 +0300 Subject: [PATCH 1/2] Data Migration Process Suggestion --- .../Migrate-your-database-to-SkySQL.md | 68 +++++++++++ .../SkySQL-custom-migration.md | 114 ++++++++++++++++++ .../SkySQL-managed-migration.md | 12 ++ 3 files changed, 194 insertions(+) create mode 100644 docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md create mode 100644 docs/Data loading, Migration/SkySQL-custom-migration.md create mode 100644 docs/Data loading, Migration/SkySQL-managed-migration.md diff --git a/docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md b/docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md new file mode 100644 index 00000000..2f360f45 --- /dev/null +++ b/docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md @@ -0,0 +1,68 @@ +# Migrating your existing Production DB + +SkySQL provides a range of options to suit different migration scenarios. + + +Below are the most common scenarios for database migration to SkySQL. + +--- +## Prerequisites + +1. An active SkySQL account. +2. An existing source database with the IP added to your SkySQL allowlist. + +
+Considerations +
+ +Ensure that your SkySQL servce deploymned configuration is compatible with your existing source database one, including: +
+--- +## SkyDBA Assisted Migration + + - Existing customers can submit a [support case](https://support.skysql.com) to request assistance with a migration. + - New customers can [contact us](mailto:support@skysql.com) to begin the migration planning process. + +Our [SkyDBA team](https://skysqlinc.github.io/skysql-docs/FractionalDBA/) can help design a migration plan to suit your needs. + +
+SkyDBA Assisted Migration Approach +
+ We use a multi-step process to assist customers with migrations: +
+--- +## Self-Service Migration to SkySQL + +SkySQL provides two diffeent options for self-service migration + +### Option 1 SkySQL Managed Migration +SkySQL Managed Migration is a fully managed service that handles the entire migration process, including data migration, schema migration, and user migration. + +- [Sky SQL Managed Migration Tutorial](./SkySQL-managed-migration.md) + +### Option 2: SkySQL Custom Migration + +For most small, mid-size and large migrations SkySQL Managed Migration is the quickest and safest option. However, for large migrations or migrations with specific requirements, you and your team may require more flexibility and control over the migration process. In these cases, you can opt for a custom migration where you and your team can design a custoemmigration plan considering the steps suggested below. + +- [Sky SQL Custom Migration Tutorial](./SkySQL-custom-migration.md) + +SkyDBA could be engaged to assist with the custom migration plan design and execution. + diff --git a/docs/Data loading, Migration/SkySQL-custom-migration.md b/docs/Data loading, Migration/SkySQL-custom-migration.md new file mode 100644 index 00000000..52563353 --- /dev/null +++ b/docs/Data loading, Migration/SkySQL-custom-migration.md @@ -0,0 +1,114 @@ + +//TODO Add a section on physical backup and restore + +## Live Replication for Minimal Downtime + +To minimize downtime during migration, set up live binary-loggd based replication from your source database to the SkySQL database. +Click [here](<./Replicating data from external DB.md>) for a detailed walk through of the steps involved. + + + + + + + +Follow these steps: +# Replicating data from an External DB + + + + + +2. **Create the Users and Grants Separately**: To avoid conflicts with the existing SkySQL users, use `SELECT CONCAT` on your source database to create users and grants in separate files. Note that you may need to create the schema and table grants separately as well. + + ```bash + mysql -u [username] -p -h [hostname] --silent --skip-column-names -e "SELECT CONCAT('CREATE USER \'', user, '\'@\'', host, '\' IDENTIFIED BY PASSWORD \'', authentication_string, '\';') FROM mysql.user;" > users.sql + + mysql -h [hostname] -u [username] -p --silent --skip-column-names -e "SELECT CONCAT('GRANT ', privilege_type, ' ON ', table_schema, '.* TO \'', grantee, '\';') FROM information_schema.schema_privileges;" > grants.sql + + mysql -h [hostname] -u [username] -p --silent --skip-column-names -e "SELECT CONCAT('GRANT ', privilege_type, ' ON ', table_schema, '.', table_name, ' TO \'', grantee, '\';') FROM information_schema.table_privileges;" >> grants.sql + ``` + +3. **Import the Dumps into SkySQL**: Import the logical dumps (SQL files) into your SkySQL database, ensuring to load the user and grant dumps after the main dump. + + ```bash + mariadb -u [SkySQL username] -p -h [SkySQL hostname] --port 3306 --ssl-verify-server-cert < dump.sql + mariadb -u [SkySQL username] -p -h [SkySQL hostname] --port 3306 --ssl-verify-server-cert < users.sql + mariadb -u [SkySQL username] -p -h [SkySQL hostname] --port 3306 --ssl-verify-server-cert < grants.sql + ``` + +If you encounter an error while importing your users, you may need to uninstall the `simple_password_check` plugin on your SkySQL instance. + + ```sql + UNINSTALL PLUGIN simple_password_check; + ``` + +4. **Start Replication**: Turn on replication using SkySQL stored procedures. There are procedures allowing you to set and start replication. See our [documentation](<../Reference Guide/Sky Stored Procedures.md>) for details. The `dump.sql` file you created in step 1 will contain the GTID and binary log information needed for the `change_external_primary` procedure. + + ```sql + CALL sky.change_external_primary( + host VARCHAR(255), + port INT, + logfile TEXT, + logpos LONG , + use_ssl_encryption BOOLEAN + ); + + CALL sky.replication_grants(); + CALL sky.start_replication(); + ``` + +### Performance Optimization During Migration + +- **Disable Foreign Key Checks**: Temporarily disable foreign key checks during import to speed up the process. + + ```sql + SET foreign_key_checks = 0; + ``` + +- **Disable Binary Logging**: If binary logging is not required during the import process, and you are using a standalone instance, it can potentially be disabled to improve performance. SkyDBA Services can assist with this as part of a detailed migration plan. + +### Data Integrity and Validation + +- **Consistency Checks**: Perform consistency checks on the source database before migration. Use a [supported SQL client](<../../Connecting to Sky DBs/>) to connect to your SkySQL instance and run the following. + + ```sql + CHECK TABLE [table_name] FOR UPGRADE; + ``` + +- **Post-Import Validation**: Validate the data integrity and consistency after the import. + + ```sql + CHECKSUM TABLE [table_name]; + ``` + +### Advanced Migration Techniques + +- **Adjust Buffer Sizes**: Temporarily increase buffer sizes to optimize the import performance. This can be done via the Configuration Manager in the portal. + + ```sql + innodb_buffer_pool_size = 2G + innodb_log_file_size = 512M + ``` + +- **Parallel Dump and Import**: Use tools that support parallel processing for dumping and importing data. + + ```bash + mysqlpump -u [username] -p --default-parallelism=4 --add-drop-database \ + --databases [database_name] > dump.sql + ``` + +- **Incremental Backups**: For large datasets, incremental backups can be used to minimize the amount of data to be transferred. SkyDBA Services can assist you with setting these up as part of a custom migration plan. + +### Monitoring and Logging + +- **Enable Detailed Logging**: Enable detailed logging while testing the migration process to monitor and troubleshoot effectively. The slow_log can be enabled in the SkySQL configuration manager. + +- **Resource Monitoring**: Use monitoring tools to track resource usage (CPU, memory, I/O) during the migration to ensure system stability. See our [monitoring documentation](<../Portal features/Service Monitoring Panels.md>) for details. + +### Additional Resources + +- [Backup with mariadb-dump](https://mariadb.com/kb/en/mariadb-dump/) +- [MariaDB Backup Documentation](https://mariadb.com/kb/en/mariadb-backup-overview/) +- [Advanced Backup Techniques](https://mariadb.com/kb/en/backup-and-restore-overview/) +- [Migrate RDS MySQL to SkySQL using the AWS Data Migration Service (DMS)](<./migrate-rds-mysql-to-skysql-using-amazon-data-migration-service_whitepaper_1109.pdf>) diff --git a/docs/Data loading, Migration/SkySQL-managed-migration.md b/docs/Data loading, Migration/SkySQL-managed-migration.md new file mode 100644 index 00000000..7ec01352 --- /dev/null +++ b/docs/Data loading, Migration/SkySQL-managed-migration.md @@ -0,0 +1,12 @@ +//TODO +//Steps to follow + +1. Dump using logical or physical backup +2. Upload the dump to S3/GCS bucket under your control +3. Call the migration API - ref - +[Sky SQL Managed Migration Tutorial](../Backup and Restore/Restore From Your Own Bucket.md) + +4. To minimize downtime during migration, set up live binary-loggd based replication from your source database to the SkySQL database. +Click [here](<./Replicating data from external DB.md>) for a detailed walk through of the steps involved. + +Verify \ No newline at end of file From 383e8adf130f9481cc5e3e0c5d869e7162e68549 Mon Sep 17 00:00:00 2001 From: NedPK Date: Mon, 2 Sep 2024 19:37:55 +0300 Subject: [PATCH 2/2] Proposal for migration documentation pocess update --- ...L.md => (WIP)Migrate-your-database-to-SkySQL.md} | 13 +++++++------ .../SkySQL-managed-migration.md | 6 +++--- mkdocs.yml | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) rename docs/Data loading, Migration/{Migrate-your-database-to-SkySQL.md => (WIP)Migrate-your-database-to-SkySQL.md} (86%) diff --git a/docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md b/docs/Data loading, Migration/(WIP)Migrate-your-database-to-SkySQL.md similarity index 86% rename from docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md rename to docs/Data loading, Migration/(WIP)Migrate-your-database-to-SkySQL.md index 2f360f45..44889544 100644 --- a/docs/Data loading, Migration/Migrate-your-database-to-SkySQL.md +++ b/docs/Data loading, Migration/(WIP)Migrate-your-database-to-SkySQL.md @@ -1,4 +1,4 @@ -# Migrating your existing Production DB +# (WIP) Migrate your database to SkySQL SkySQL provides a range of options to suit different migration scenarios.