From 662e1166be0b9c8816a1a03c2475b0179f11400a Mon Sep 17 00:00:00 2001 From: Diego Cabrejas Date: Sat, 28 Sep 2019 23:28:54 +0100 Subject: [PATCH 1/3] Add import from remote command --- src/Console/ImportFromRemoteCommand.php | 71 +++++++ src/Console/UploadDbCommand.php | 61 ------ src/Console/UploadToRemoteCommand.php | 64 +++++++ src/Model/Config.php | 31 ++- src/Model/Db/DbCleaner.php | 37 ++++ src/Model/Db/DbCompresser.php | 52 +++++ src/Model/Db/DbDownloader.php | 72 +++++++ src/Model/Db/DbDumper.php | 109 +++++++++++ src/Model/Db/DbImporter.php | 49 +++++ src/Model/{ => Db}/DbTables.php | 3 +- src/Model/Db/DbUploader.php | 45 +++++ src/Model/DbDumper.php | 180 ------------------ src/Model/DbFacade.php | 116 +++++++++++ src/Model/ProjectMeta.php | 67 +++++++ .../{DbUploader.php => S3/ClientProvider.php} | 34 ++-- src/etc/di.xml | 3 +- 16 files changed, 724 insertions(+), 270 deletions(-) create mode 100644 src/Console/ImportFromRemoteCommand.php delete mode 100644 src/Console/UploadDbCommand.php create mode 100644 src/Console/UploadToRemoteCommand.php create mode 100644 src/Model/Db/DbCleaner.php create mode 100644 src/Model/Db/DbCompresser.php create mode 100644 src/Model/Db/DbDownloader.php create mode 100644 src/Model/Db/DbDumper.php create mode 100644 src/Model/Db/DbImporter.php rename src/Model/{ => Db}/DbTables.php (98%) create mode 100644 src/Model/Db/DbUploader.php delete mode 100644 src/Model/DbDumper.php create mode 100644 src/Model/DbFacade.php create mode 100644 src/Model/ProjectMeta.php rename src/Model/{DbUploader.php => S3/ClientProvider.php} (56%) diff --git a/src/Console/ImportFromRemoteCommand.php b/src/Console/ImportFromRemoteCommand.php new file mode 100644 index 0000000..0baa3cc --- /dev/null +++ b/src/Console/ImportFromRemoteCommand.php @@ -0,0 +1,71 @@ +dbFacade = $dbFacade; + } + + /** + * @return void + */ + protected function configure() + { + $this->setName('wearejh:stripped-db-provider:import-from-remote'); + $this->setDescription("Import DB from JH's Cloud Storage"); + $this->addArgument( + self::ARGUMENT_PROJECT_NAME, + InputArgument::REQUIRED, + 'Source Project Name to import from eg. prod-stroustrup-workshop.' + ); + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + try { + $sourceProjectMeta = new ProjectMeta($input->getArgument(self::ARGUMENT_PROJECT_NAME)); + $output->writeln('Downloading Database From Cloud Storage...'); + $this->dbFacade->downloadDatabaseDump($sourceProjectMeta); + $output->writeln(sprintf( + "Dump downloaded at %s", + $sourceProjectMeta->getLocalAbsoluteCompressedFileDumpPath() + )); + $output->writeln('Uncompressing Database ...'); + $this->dbFacade->uncompressDatabaseDump($sourceProjectMeta); + $output->writeln('Importing Database ...'); + $this->dbFacade->importDatabaseDump($sourceProjectMeta); + $output->writeln("Database successfully imported."); + } catch (\Exception $e) { + $output->writeln("{$e->getMessage()}"); + } finally { + if (isset($sourceProjectMeta)) { + $this->dbFacade->cleanUpLocalDumpFiles($sourceProjectMeta); + } + } + } +} diff --git a/src/Console/UploadDbCommand.php b/src/Console/UploadDbCommand.php deleted file mode 100644 index 4068c0e..0000000 --- a/src/Console/UploadDbCommand.php +++ /dev/null @@ -1,61 +0,0 @@ -dbDumper = $dbDumper; - $this->dbUploader = $dbUploader; - } - - /** - * @return void - */ - protected function configure() - { - $this->setName('wearejh:db:backup-stripped-db'); - $this->setDescription("Upload a stripped DB dump to JH's Cloud Storage"); - } - - /** - * {@inheritdoc} - */ - protected function execute(InputInterface $input, OutputInterface $output) - { - try { - $output->writeln('Dumping Database...'); - $this->dbDumper->dumpDb(); - $output->writeln("Dump created at {$this->dbDumper->getAbsoluteDumpPath()}"); - $output->writeln('Uploading Dump to Cloud Storage...'); - $this->dbUploader->uploadDBDump($this->dbDumper->getAbsoluteDumpPath()); - $output->writeln("Dump successfully updated."); - } catch (\Exception $e) { - $output->writeln("{$e->getMessage()}"); - } finally { - $output->writeln('Cleaning up files...'); - $this->dbDumper->cleanUp(); - $output->writeln("Done."); - } - } -} diff --git a/src/Console/UploadToRemoteCommand.php b/src/Console/UploadToRemoteCommand.php new file mode 100644 index 0000000..20f3efb --- /dev/null +++ b/src/Console/UploadToRemoteCommand.php @@ -0,0 +1,64 @@ +dbFacade = $dbFacade; + $this->config = $config; + } + + /** + * @return void + */ + protected function configure() + { + $this->setName('wearejh:stripped-db-provider:upload-to-remote'); + $this->setDescription("Upload a stripped DB dump to JH's Cloud Storage"); + } + + /** + * {@inheritdoc} + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + try { + $projectMeta = $this->config->getProjectMeta(); + $output->writeln('Dumping Database...'); + $this->dbFacade->dumpDatabase($projectMeta); + $output->writeln("Dump created at {$projectMeta->getLocalAbsoluteFileDumpPath()}"); + $output->writeln('Compressing Dump...'); + $this->dbFacade->compressDatabaseDump($projectMeta); + $output->writeln('Uploading Dump to Cloud Storage...'); + $this->dbFacade->uploadDatabaseDump($projectMeta); + $output->writeln("Dump successfully uploaded at {$projectMeta->getRemoteDumpObjectKey()}."); + } catch (\Exception $e) { + $output->writeln("{$e->getMessage()}"); + } finally { + if (isset($projectMeta)) { + $this->dbFacade->cleanUpLocalDumpFiles($projectMeta); + } + } + } +} diff --git a/src/Model/Config.php b/src/Model/Config.php index dcc3771..66c7333 100644 --- a/src/Model/Config.php +++ b/src/Model/Config.php @@ -5,6 +5,8 @@ namespace Jh\StrippedDbProvider\Model; use Magento\Framework\App\Config\ScopeConfigInterface; +use Magento\Framework\App\DeploymentConfig; +use Magento\Framework\Config\ConfigOptionsListConstants; class Config { @@ -32,9 +34,17 @@ class Config */ private $config; - public function __construct(ScopeConfigInterface $config) - { + /** + * @var DeploymentConfig + */ + private $deploymentConfig; + + public function __construct( + ScopeConfigInterface $config, + DeploymentConfig $deploymentConfig + ) { $this->config = $config; + $this->deploymentConfig = $deploymentConfig; } public function isEnabled(): bool @@ -42,9 +52,11 @@ public function isEnabled(): bool return (bool) $this->config->isSetFlag(self::XML_PATH_ENABLED); } - public function getProjectName(): string + public function getProjectMeta(): ProjectMeta { - return (string) $this->config->getValue(self::XML_PATH_PROJECT_NAME); + return new ProjectMeta( + (string) $this->config->getValue(self::XML_PATH_PROJECT_NAME) + ); } public function getBucketRegion(): string @@ -66,4 +78,15 @@ public function getSecretAccessKey(): string { return (string) $this->config->getValue(self::XML_PATH_SECRET_ACCESS_KEY); } + + public function getLocalDbConfigData(string $key): ?string + { + return $this->deploymentConfig->get( + sprintf( + "%s/%s", + ConfigOptionsListConstants::CONFIG_PATH_DB_CONNECTION_DEFAULT, + $key + ) + ); + } } diff --git a/src/Model/Db/DbCleaner.php b/src/Model/Db/DbCleaner.php new file mode 100644 index 0000000..3160a68 --- /dev/null +++ b/src/Model/Db/DbCleaner.php @@ -0,0 +1,37 @@ +shell = $shell; + } + + /** + * Attempt to silently remove the database dumps + * + * @param ProjectMeta $projectMeta + * @return string + */ + public function cleanUp(ProjectMeta $projectMeta) + { + try { + $this->shell->execute("rm %s", [$projectMeta->getLocalAbsoluteFileDumpPath()]); + $this->shell->execute("rm %s", [$projectMeta->getLocalAbsoluteCompressedFileDumpPath()]); + } catch (\Exception $e) { + //empty + } + } +} diff --git a/src/Model/Db/DbCompresser.php b/src/Model/Db/DbCompresser.php new file mode 100644 index 0000000..4c9dede --- /dev/null +++ b/src/Model/Db/DbCompresser.php @@ -0,0 +1,52 @@ +shell = $shell; + } + + /** + * @param ProjectMeta $projectMeta + * @throws LocalizedException + */ + public function compressDump(ProjectMeta $projectMeta) + { + $this->shell->execute( + sprintf( + "gzip -c %s > %s", + $projectMeta->getLocalAbsoluteFileDumpPath(), + $projectMeta->getLocalAbsoluteCompressedFileDumpPath() + ) + ); + } + + /** + * @param ProjectMeta $projectMeta + * @throws LocalizedException + */ + public function uncompressDump(ProjectMeta $projectMeta) + { + $this->shell->execute( + sprintf( + "gunzip -c %s > %s", + $projectMeta->getLocalAbsoluteCompressedFileDumpPath(), + $projectMeta->getLocalAbsoluteFileDumpPath() + ) + ); + } +} diff --git a/src/Model/Db/DbDownloader.php b/src/Model/Db/DbDownloader.php new file mode 100644 index 0000000..91b319c --- /dev/null +++ b/src/Model/Db/DbDownloader.php @@ -0,0 +1,72 @@ +config = $config; + $this->clientProvider = $clientProvider; + } + + /** + * @param ProjectMeta $projectMeta + */ + public function downloadDBDump(ProjectMeta $projectMeta) + { + $client = $this->clientProvider->getClient(); + $bucketName = $this->config->getBucketName(); + $objectKey = $projectMeta->getRemoteDumpObjectKey(); + + if (!$client->doesObjectExist($bucketName, $objectKey)) { + throw new \RuntimeException( + sprintf( + "Object '%s' does not exist in bucket '%s'", + $objectKey, + $bucketName + ) + ); + } + + $result = $client->getObject([ + 'Bucket' => $bucketName, + 'Key' => $objectKey + ]); + + if (!$result->get('Body') instanceof Stream) { + throw new \RuntimeException("Wrong response received from the remote"); + } + + /** + * @var $stream Stream + */ + $stream = $result->get('Body'); + $localStorageFilePath = $projectMeta->getLocalAbsoluteCompressedFileDumpPath(); + $success = file_put_contents( + $localStorageFilePath, + $stream->getContents() + ); + + if (!$success) { + throw new \RuntimeException("Could not write DB to " . $localStorageFilePath); + } + } +} diff --git a/src/Model/Db/DbDumper.php b/src/Model/Db/DbDumper.php new file mode 100644 index 0000000..ce44b19 --- /dev/null +++ b/src/Model/Db/DbDumper.php @@ -0,0 +1,109 @@ +shell = $shell; + $this->config = $config; + $this->dbTables = $dbTables; + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function dumpDb(ProjectMeta $projectMeta): void + { + $this->shell->execute($this->buildStructureDumpForStrippedTablesCmd($projectMeta)); + $this->shell->execute($this->buildDataDumpExcludingStrippedTablesCmd($projectMeta)); + } + + /** + * @return string + */ + private function buildDataDumpExcludingStrippedTablesCmd(ProjectMeta $projectMeta): string + { + $dumpCmd = "mysqldump --single-transaction --quick -h%s -u%s --password=%s %s %s | %s | %s >> %s"; + return sprintf( + $dumpCmd, + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_USER), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME), + $this->buildListOfStructureOnlyTables(true), + $this->rmPasswordWarning, + $this->rmDefinerCommand, + $projectMeta->getLocalAbsoluteFileDumpPath() + ); + } + + /** + * @return string + */ + private function buildStructureDumpForStrippedTablesCmd(ProjectMeta $projectMeta): string + { + $dumpCmd = "mysqldump --single-transaction --quick --no-data -h%s -u%s --password=%s %s %s | %s | %s > %s"; + return sprintf( + $dumpCmd, + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_USER), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME), + $this->buildListOfStructureOnlyTables(false), + $this->rmPasswordWarning, + $this->rmDefinerCommand, + $projectMeta->getLocalAbsoluteFileDumpPath() + ); + } + + /** + * @param bool $ignoreTableFlag + * @return string + */ + private function buildListOfStructureOnlyTables(bool $ignoreTableFlag = false): string + { + $tableNameList = $this->dbTables->getStructureOnlyTables(); + if ($ignoreTableFlag) { + $dbName = $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME); + $tableNameList = array_map(function ($tableName) use ($dbName) { + return sprintf( + "--ignore-table=%s.%s", + $dbName, + $tableName + ); + }, $tableNameList); + } + + return implode(" ", $tableNameList); + } +} diff --git a/src/Model/Db/DbImporter.php b/src/Model/Db/DbImporter.php new file mode 100644 index 0000000..633de07 --- /dev/null +++ b/src/Model/Db/DbImporter.php @@ -0,0 +1,49 @@ +shell = $shell; + $this->config = $config; + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function importDatabase(ProjectMeta $projectMeta): void + { + $cmd = sprintf( + "mysql -h%s -u%s --password=%s %s < %s", + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_USER), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), + $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME), + $projectMeta->getLocalAbsoluteFileDumpPath() + ); + + $this->shell->execute($cmd); + } +} diff --git a/src/Model/DbTables.php b/src/Model/Db/DbTables.php similarity index 98% rename from src/Model/DbTables.php rename to src/Model/Db/DbTables.php index 3c98d52..bb6b06e 100644 --- a/src/Model/DbTables.php +++ b/src/Model/Db/DbTables.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Jh\StrippedDbProvider\Model; +namespace Jh\StrippedDbProvider\Model\Db; use Magento\Framework\App\DeploymentConfig; +use Jh\StrippedDbProvider\Model\Config; class DbTables { diff --git a/src/Model/Db/DbUploader.php b/src/Model/Db/DbUploader.php new file mode 100644 index 0000000..30f256e --- /dev/null +++ b/src/Model/Db/DbUploader.php @@ -0,0 +1,45 @@ +config = $config; + $this->clientProvider = $clientProvider; + } + + /** + * @param ProjectMeta $projectMeta + * @return \Aws\ResultInterface + */ + public function uploadDBDump(ProjectMeta $projectMeta) + { + $client = $this->clientProvider->getClient(); + + $uploader = new MultipartUploader($client, $projectMeta->getLocalAbsoluteCompressedFileDumpPath(), [ + 'bucket' => $this->config->getBucketName(), + 'key' => $projectMeta->getRemoteDumpObjectKey() + ]); + + return $uploader->upload(); + } +} diff --git a/src/Model/DbDumper.php b/src/Model/DbDumper.php deleted file mode 100644 index a1bb2f7..0000000 --- a/src/Model/DbDumper.php +++ /dev/null @@ -1,180 +0,0 @@ -shell = $shell; - $this->config = $config; - $this->phpFinder = $phpFinderFactory->create(); - $this->deploymentConfig = $deploymentConfig; - $this->dbTables = $dbTables; - } - - /** - * @throws LocalizedException - */ - public function dumpDb(): void - { - $this->shell->execute($this->buildStructureDumpForStrippedTablesCmd()); - $this->shell->execute($this->buildDataDumpExcludingStrippedTablesCmd()); - $this->shell->execute($this->buildCompressDumpCmd()); - } - - /** - * @return string - */ - private function buildDataDumpExcludingStrippedTablesCmd(): string - { - $dumpCmd = "mysqldump --single-transaction --quick -h%s -u%s --password=%s %s %s | %s | %s >> %s"; - return sprintf( - $dumpCmd, - $this->getDbConfigData(ConfigOptionsListConstants::KEY_HOST), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_USER), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_NAME), - $this->buildListOfStructureOnlyTables(true), - $this->rmPasswordWarning, - $this->rmDefinerCommand, - $this->getAbsoluteDumpPath(false) - ); - } - - /** - * @return string - */ - private function buildStructureDumpForStrippedTablesCmd(): string - { - $dumpCmd = "mysqldump --single-transaction --quick --no-data -h%s -u%s --password=%s %s %s | %s | %s > %s"; - return sprintf( - $dumpCmd, - $this->getDbConfigData(ConfigOptionsListConstants::KEY_HOST), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_USER), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), - $this->getDbConfigData(ConfigOptionsListConstants::KEY_NAME), - $this->buildListOfStructureOnlyTables(false), - $this->rmPasswordWarning, - $this->rmDefinerCommand, - $this->getAbsoluteDumpPath(false) - ); - } - - /** - * @return string - */ - private function buildCompressDumpCmd(): string - { - return sprintf( - "gzip -c %s > %s", - $this->getAbsoluteDumpPath(false), - $this->getAbsoluteDumpPath(true) - ); - } - - /** - * @param bool $ignoreTableFlag - * @return string - */ - private function buildListOfStructureOnlyTables(bool $ignoreTableFlag = false): string - { - $tableNameList = $this->dbTables->getStructureOnlyTables(); - if ($ignoreTableFlag) { - $dbName = $this->getDbConfigData(ConfigOptionsListConstants::KEY_NAME); - $tableNameList = array_map(function ($tableName) use ($dbName) { - return sprintf( - "--ignore-table=%s.%s", - $dbName, - $tableName - ); - }, $tableNameList); - } - - return implode(" ", $tableNameList); - } - - /** - * Attempt to silently remove the database dumps - * @return string - */ - public function cleanUp() - { - try { - $this->shell->execute("rm %s", [$this->getAbsoluteDumpPath()]); - $this->shell->execute("rm %s", [$this->getAbsoluteDumpPath(false)]); - } catch (\Exception $e) { - //empty - } - } - - /** - * @param bool $compressed - * @return string - */ - public function getAbsoluteDumpPath(bool $compressed = true): string - { - $fileName = sprintf( - "%s.sql%s", - $this->config->getProjectName(), - ($compressed ? '.gz' : '') - ); - return BP . '/var/' . $fileName; - } - - /** - * @param string $key - * @return string|null - */ - private function getDbConfigData(string $key): ?string - { - return $this->deploymentConfig->get( - sprintf( - "%s/%s", - ConfigOptionsListConstants::CONFIG_PATH_DB_CONNECTION_DEFAULT, - $key - ) - ); - } -} diff --git a/src/Model/DbFacade.php b/src/Model/DbFacade.php new file mode 100644 index 0000000..f7d4813 --- /dev/null +++ b/src/Model/DbFacade.php @@ -0,0 +1,116 @@ +dumper = $dumper; + $this->compresser = $compresser; + $this->downloader = $downloader; + $this->uploader = $uploader; + $this->cleaner = $cleaner; + $this->importer = $importer; + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function dumpDatabase(ProjectMeta $projectMeta) + { + $this->dumper->dumpDb($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + */ + public function downloadDatabaseDump(ProjectMeta $projectMeta) + { + $this->downloader->downloadDBDump($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function compressDatabaseDump(ProjectMeta $projectMeta) + { + $this->compresser->compressDump($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function uncompressDatabaseDump(ProjectMeta $projectMeta) + { + $this->compresser->uncompressDump($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + * @throws \Exception + */ + public function importDatabaseDump(ProjectMeta $projectMeta) + { + $this->importer->importDatabase($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + */ + public function cleanUpLocalDumpFiles(ProjectMeta $projectMeta) + { + $this->cleaner->cleanUp($projectMeta); + } + + /** + * @param ProjectMeta $projectMeta + */ + public function uploadDatabaseDump(ProjectMeta $projectMeta) + { + $this->uploader->uploadDBDump($projectMeta); + } +} diff --git a/src/Model/ProjectMeta.php b/src/Model/ProjectMeta.php new file mode 100644 index 0000000..c4ca9e0 --- /dev/null +++ b/src/Model/ProjectMeta.php @@ -0,0 +1,67 @@ +validateName($name); + $this->name = $name; + } + + /** + * @param string $name + * @throws \RuntimeException + */ + private function validateName(string $name) + { + $ext = pathinfo($name, PATHINFO_EXTENSION); + + if (strlen($ext) > 0) { + throw new \RuntimeException("Project name '$name' must not contain any dots"); + } + } + + public function getDumpFileName(): string + { + return $this->name . '.sql'; + } + + public function getCompressedDumpFileName(): string + { + return $this->getDumpFileName() . '.gz'; + } + + public function getLocalAbsoluteFileDumpPath(): string + { + return $this->localStoragePath . $this->getDumpFileName(); + } + + public function getLocalAbsoluteCompressedFileDumpPath(): string + { + return $this->localStoragePath . $this->getCompressedDumpFileName(); + } + + public function getRemoteDumpObjectKey(): string + { + return $this->remoteStoragePath . $this->getCompressedDumpFileName(); + } +} diff --git a/src/Model/DbUploader.php b/src/Model/S3/ClientProvider.php similarity index 56% rename from src/Model/DbUploader.php rename to src/Model/S3/ClientProvider.php index 51dffff..921c3f0 100644 --- a/src/Model/DbUploader.php +++ b/src/Model/S3/ClientProvider.php @@ -2,23 +2,23 @@ declare(strict_types=1); -namespace Jh\StrippedDbProvider\Model; +namespace Jh\StrippedDbProvider\Model\S3; -use Aws\S3\MultipartUploader; use Aws\S3\S3Client; use Aws\Credentials\Credentials; +use Jh\StrippedDbProvider\Model\Config; -class DbUploader +class ClientProvider { /** - * @var S3Client + * @var Config */ - private $client; + private $config; /** - * @var Config + * @var S3Client */ - private $config; + private $client; public function __construct(Config $config) { @@ -26,23 +26,9 @@ public function __construct(Config $config) } /** - * @param string $dumpAbsolutePath - * @return \Aws\ResultInterface - * @throws \RuntimeException + * @return S3Client */ - public function uploadDBDump(string $dumpAbsolutePath) - { - $client = $this->getS3Client(); - - $uploader = new MultipartUploader($client, $dumpAbsolutePath, [ - 'bucket' => $this->config->getBucketName(), - 'key' => 'stripped-db-backups/' . basename($dumpAbsolutePath) - ]); - - return $uploader->upload(); - } - - private function getS3Client(): S3Client + public function getClient(): S3Client { if (is_null($this->client)) { $credentials = new Credentials( @@ -59,4 +45,6 @@ private function getS3Client(): S3Client return $this->client; } + + } diff --git a/src/etc/di.xml b/src/etc/di.xml index f0f4f5f..cb7659d 100644 --- a/src/etc/di.xml +++ b/src/etc/di.xml @@ -3,7 +3,8 @@ - Jh\StrippedDbProvider\Console\UploadDbCommand + Jh\StrippedDbProvider\Console\UploadToRemoteCommand + Jh\StrippedDbProvider\Console\ImportFromRemoteCommand From 52cc3c16116ee557d3336d401924b3ade05ebe5e Mon Sep 17 00:00:00 2001 From: Diego Cabrejas Date: Mon, 6 Jul 2020 19:42:03 +0100 Subject: [PATCH 2/3] Change cron to use the new facade. Refactor to remove dependency on mysqldump. --- .gitignore | 4 ++- composer.json | 13 +++++--- src/Cron/UploadDbCron.php | 33 +++++++++----------- src/Model/Db/DbDumper.php | 66 ++++++++------------------------------- 4 files changed, 39 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 723ef36..688d850 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.idea \ No newline at end of file +.idea +composer.lock +vendor \ No newline at end of file diff --git a/composer.json b/composer.json index e7f0204..a247a91 100644 --- a/composer.json +++ b/composer.json @@ -2,11 +2,16 @@ "name": "wearejh/stripped-db-provider", "description": "Stripped DB Provider for Magento 2", "type": "magento2-module", + "repositories": [ + { + "type": "vcs", + "url": "git@github.com:dcabrejas/mysqldump-php.git" + } + ], "require": { "php": "^7.1", - "magento/framework": "*", - "magento/module-backend": "*", - "aws/aws-sdk-php": "^3.112.0" + "aws/aws-sdk-php": "^3.112.0", + "ifsnop/mysqldump-php": "dev-allow-select-empty-set@dev" }, "autoload": { "files": [ @@ -16,4 +21,4 @@ "Jh\\StrippedDbProvider\\": "src/" } } -} \ No newline at end of file +} diff --git a/src/Cron/UploadDbCron.php b/src/Cron/UploadDbCron.php index 3e251a5..ecdb7ce 100644 --- a/src/Cron/UploadDbCron.php +++ b/src/Cron/UploadDbCron.php @@ -5,8 +5,7 @@ namespace Jh\StrippedDbProvider\Cron; use Jh\StrippedDbProvider\Model\Config; -use Jh\StrippedDbProvider\Model\DbDumper; -use Jh\StrippedDbProvider\Model\DbUploader; +use Jh\StrippedDbProvider\Model\DbFacade; use Psr\Log\LoggerInterface; class UploadDbCron @@ -17,31 +16,23 @@ class UploadDbCron private $config; /** - * @var DbDumper - */ - private $dbDumper; - - /** - * @var DbUploader + * @var LoggerInterface */ - private $dbUploader; + private $logger; /** - * @var LoggerInterface + * @var DbFacade */ - private $logger; + private $dbFacade; public function __construct( Config $config, - DbDumper $dbDumper, - DbUploader $dbUploader, + DbFacade $dbFacade, LoggerInterface $logger ) { - $this->config = $config; - $this->dbDumper = $dbDumper; - $this->dbUploader = $dbUploader; $this->logger = $logger; + $this->dbFacade = $dbFacade; } /** @@ -54,12 +45,16 @@ public function execute() } try { - $this->dbDumper->dumpDb(); - $this->dbUploader->uploadDBDump($this->dbDumper->getAbsoluteDumpPath()); + $projectMeta = $this->config->getProjectMeta(); + $this->dbFacade->dumpDatabase($projectMeta); + $this->dbFacade->compressDatabaseDump($projectMeta); + $this->dbFacade->uploadDatabaseDump($projectMeta); } catch (\Exception $e) { $this->logger->critical($e); } finally { - $this->dbDumper->cleanUp(); + if (isset($projectMeta)) { + $this->dbFacade->cleanUpLocalDumpFiles($projectMeta); + } } } } diff --git a/src/Model/Db/DbDumper.php b/src/Model/Db/DbDumper.php index ce44b19..52b0c83 100644 --- a/src/Model/Db/DbDumper.php +++ b/src/Model/Db/DbDumper.php @@ -4,6 +4,7 @@ namespace Jh\StrippedDbProvider\Model\Db; +use Ifsnop\Mysqldump\Mysqldump; use Magento\Framework\Shell; use Magento\Framework\Config\ConfigOptionsListConstants; use Jh\StrippedDbProvider\Model\Config; @@ -11,8 +12,6 @@ class DbDumper { - private $rmDefinerCommand = "LANG=C LC_CTYPE=C LC_ALL=C sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/'"; - private $rmPasswordWarning = "grep -v 'Warning: Using a password'"; /** * @var Shell */ @@ -44,66 +43,27 @@ public function __construct( */ public function dumpDb(ProjectMeta $projectMeta): void { - $this->shell->execute($this->buildStructureDumpForStrippedTablesCmd($projectMeta)); - $this->shell->execute($this->buildDataDumpExcludingStrippedTablesCmd($projectMeta)); - } - - /** - * @return string - */ - private function buildDataDumpExcludingStrippedTablesCmd(ProjectMeta $projectMeta): string - { - $dumpCmd = "mysqldump --single-transaction --quick -h%s -u%s --password=%s %s %s | %s | %s >> %s"; - return sprintf( - $dumpCmd, - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST), + $hostName = $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST); + $dbName = $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME); + $dumper = new Mysqldump( + "mysql:host={$hostName};dbname={$dbName}", $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_USER), $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME), - $this->buildListOfStructureOnlyTables(true), - $this->rmPasswordWarning, - $this->rmDefinerCommand, - $projectMeta->getLocalAbsoluteFileDumpPath() + ['skip-definer' => true] ); - } - /** - * @return string - */ - private function buildStructureDumpForStrippedTablesCmd(ProjectMeta $projectMeta): string - { - $dumpCmd = "mysqldump --single-transaction --quick --no-data -h%s -u%s --password=%s %s %s | %s | %s > %s"; - return sprintf( - $dumpCmd, - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_HOST), - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_USER), - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_PASSWORD), - $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME), - $this->buildListOfStructureOnlyTables(false), - $this->rmPasswordWarning, - $this->rmDefinerCommand, - $projectMeta->getLocalAbsoluteFileDumpPath() - ); + $dumper->setTableLimits($this->getTableLimits()); + $dumper->start($projectMeta->getLocalAbsoluteFileDumpPath()); } - /** - * @param bool $ignoreTableFlag - * @return string - */ - private function buildListOfStructureOnlyTables(bool $ignoreTableFlag = false): string + private function getTableLimits(): array { $tableNameList = $this->dbTables->getStructureOnlyTables(); - if ($ignoreTableFlag) { - $dbName = $this->config->getLocalDbConfigData(ConfigOptionsListConstants::KEY_NAME); - $tableNameList = array_map(function ($tableName) use ($dbName) { - return sprintf( - "--ignore-table=%s.%s", - $dbName, - $tableName - ); - }, $tableNameList); + $tableLimits = []; + foreach ($tableNameList as $tableName) { + $tableLimits[$tableName] = 0; } - return implode(" ", $tableNameList); + return $tableLimits; } } From defaf18e2d25b109604847d67d0f408eb79abdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20S=C5=82awik?= Date: Mon, 8 Nov 2021 09:08:49 +0100 Subject: [PATCH 3/3] Use stable version of mysqldump-php library --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index a247a91..b46f89f 100644 --- a/composer.json +++ b/composer.json @@ -5,13 +5,13 @@ "repositories": [ { "type": "vcs", - "url": "git@github.com:dcabrejas/mysqldump-php.git" + "url": "git@github.com:maciejslawik/mysqldump-php.git" } ], "require": { "php": "^7.1", "aws/aws-sdk-php": "^3.112.0", - "ifsnop/mysqldump-php": "dev-allow-select-empty-set@dev" + "ifsnop/mysqldump-php": ">=3.0" }, "autoload": { "files": [