Skip to content

Commit

Permalink
refactor: enhance spaces and namings
Browse files Browse the repository at this point in the history
  • Loading branch information
sh1nj1 committed Aug 17, 2024
1 parent e9e078a commit 17a5a69
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 77 deletions.
2 changes: 1 addition & 1 deletion Gemfile.lock_for_bundler2
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
mysql_db_tool (0.2.2)
mysql_db_tool (0.3.0)

GEM
remote: https://rubygems.org/
Expand Down
50 changes: 25 additions & 25 deletions lib/mysql_db_tool/backup.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ class Backup

def initialize(options = {})
@options = options
tableConfig = MySQLDBTool::Config::ConfigLoader.load(options[:env])
@data_tables = tableConfig[:data_tables]
@ignore_tables = tableConfig[:ignore_tables]
@db_info = tableConfig[:db_info]
config = MySQLDBTool::Config::ConfigLoader.load(options[:env])
@data_tables = config[:data_tables]
@ignore_tables = config[:ignore_tables]
@db_info = config[:db_info]

@db_info[:database] = @options[:database] if @options[:database]
end

Expand All @@ -21,51 +21,51 @@ def perform
verify_tools_exist

id=@options[:id] || "0"
isGzip=@options[:gzip]
limitDays=@options[:limit_days] || 3
gzipSuffix = @options[:gzip_suffix] || '.gz'
use_gzip=@options[:gzip]
limit_days=@options[:limit_days] || 3
gzip_suffix = @options[:gzip_suffix] || '.gz'

Array(@db_info[:database]).flat_map.each_with_index do |database, index |

defaultOptions="--column-statistics=0 #{mysqlDefaultOptions(@db_info, database)}"
backupDir=backupDirName(id, "#{index}_#{database}")
default_options="--column-statistics=0 #{mysql_default_options(@db_info, database)}"
backup_dir=backup_dir_name(id, "#{index}_#{database}")

commands = []
if File.exist? backupDir
puts "[skip - directory exists] #{backupDir}"
if File.exist? backup_dir
puts "[skip - directory exists] #{backup_dir}"
return commands
end
commands.push "mkdir -p #{backupDir}"
commands.push "mkdir -p #{backup_dir}"

backupFile="#{backupDir}/#{DateTime.now.strftime("%Y-%m-%d")}_#{id}"
whereDate=(Date.today - limitDays).strftime("%Y-%m-%d 00:00:00")
backup_file="#{backup_dir}/#{DateTime.now.strftime("%Y-%m-%d")}_#{id}"
where_date=(Date.today - limit_days).strftime("%Y-%m-%d 00:00:00")
options=ENV['DUMP_OPTIONS'] || "--single-transaction --skip-lock-tables"

puts "backupFile=#{backupFile}"
puts "backupFile=#{backup_file}"

ignoreTablesOption = @ignore_tables.map { |e| "--ignore-table=#{e.include?('.') ? e : "#{database}.#{e}"}" }.join(' ')
ignore_tables_option = @ignore_tables.map { |e| "--ignore-table=#{e.include?('.') ? e : "#{database}.#{e}"}" }.join(' ')

commands.push gzipCommand("mysqldump --no-data #{ignoreTablesOption} #{defaultOptions}", isGzip, "#{backupFile}-schema.sql", gzipSuffix)
commands.push gzip_command("mysqldump --no-data #{ignore_tables_option} #{default_options}", use_gzip, "#{backup_file}-schema.sql", gzip_suffix)

backupTables = []
backup_tables = []

@data_tables.each {|table|
where = table[:where].empty? ? "" : "--where=\"#{table[:where]} >= '#{whereDate}'\""
where = table[:where].empty? ? "" : "--where=\"#{table[:where]} >= '#{where_date}'\""
if where.empty?
backupTables.push(table[:name])
backup_tables.push(table[:name])
next
else
commands.push(gzipCommand("mysqldump --no-create-info #{options} #{where} #{defaultOptions} #{table[:name]}", isGzip, "#{backupFile}-#{table[:name]}.sql", gzipSuffix))
commands.push(gzip_command("mysqldump --no-create-info #{options} #{where} #{default_options} #{table[:name]}", use_gzip, "#{backup_file}-#{table[:name]}.sql", gzip_suffix))
end
}

commands.push(gzipCommand("mysqldump --no-create-info #{ignoreTablesOption} #{options} #{defaultOptions} #{backupTables.join(' ')}", isGzip, "#{backupFile}-all-other-tables.sql", gzipSuffix))
commands.push(gzip_command("mysqldump --no-create-info #{ignore_tables_option} #{options} #{default_options} #{backup_tables.join(' ')}", use_gzip, "#{backup_file}-all-other-tables.sql", gzip_suffix))
commands
end
end

def gzipCommand(command, isGzip, file, gzipSuffix = '.gz')
"#{command} #{isGzip ? '| gzip ' : ''} > #{file}#{isGzip ? gzipSuffix : ''}"
def gzip_command(command, use_gzip, file, gzip_suffix = '.gz')
"#{command} #{use_gzip ? '| gzip ' : ''} > #{file}#{use_gzip ? gzip_suffix : ''}"
end

end
Expand Down
12 changes: 6 additions & 6 deletions lib/mysql_db_tool/db_backup_base.rb
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@


def run(isRun, command)
if not isRun
def run(is_run, command)
if not is_run
puts "[dryRun] #{command}"
else
puts "Running: [#{command}]"
puts `#{command}`
end
end

def backupDirName(id, dbName = "")
"backup-#{id}#{dbName.empty? ? '' : "/#{dbName}"}"
def backup_dir_name(id, db_name = "")
"backup-#{id}#{db_name.empty? ? '' : "/#{db_name}"}"
end

def mysqlDefaultOptions(db_info, database)
def mysql_default_options(db_info, database)
" --ssl-mode=disabled -h #{db_info[:host]} -u #{db_info[:user]} #{db_info[:password].to_s.empty? ? '' : " -p'#{db_info[:password]}'"} #{db_info[:port].to_s.empty? ? '' : " -P'#{db_info[:port]}'"} #{database} "
end

Expand All @@ -22,7 +22,7 @@ def verify_tools_exist
missing_tools = []

tools.each do |tool|
if not system("which #{tool} > /dev/null 2>&1")
unless system("which #{tool} > /dev/null 2>&1")
missing_tools << tool
puts "'#{tool}' is not available"
end
Expand Down
57 changes: 27 additions & 30 deletions lib/mysql_db_tool/restore.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ class Restore

def initialize(options = {})
@options = options
tableConfig = MySQLDBTool::Config::ConfigLoader.load(options[:env])
@data_tables = tableConfig[:data_tables]
@ignore_tables = tableConfig[:ignore_tables]
@db_info = tableConfig[:db_info]
config = MySQLDBTool::Config::ConfigLoader.load(options[:env])
@data_tables = config[:data_tables]
@ignore_tables = config[:ignore_tables]
@db_info = config[:db_info]

@db_info[:database] = @options[:database] if @options[:database]
end
Expand All @@ -25,66 +25,63 @@ def perform
end

id=@options[:id] || "0"
isRun=@options[:run]
isDropAllTables=@options[:drop_all_tables]
is_drop_all_tables=@options[:drop_all_tables]

puts "ARGV=#{ARGV}, env=#{env}, id=#{id}, run=#{isRun} isDropAllTables=#{isDropAllTables}"
backup_dir=backup_dir_name(id)

backupDir=backupDirName(id)

puts "backupDir=#{backupDir}"
puts "backupDir=#{backup_dir}"
databases = Array(@db_info[:database])

databaseMap = {}
sameDb = true
database_map = {}
same_db = true

Dir.entries(backupDir).reject {|f| File.directory? f}.sort.each do |f|
Dir.entries(backup_dir).reject {|f| File.directory? f}.sort.each do |f|

index, origin_database = split_integer_and_string(f)
database = get_element_or_last(databases, index)
sameDb = sameDb && (database == origin_database)
databaseMap["`#{origin_database}`\\."] = "`#{database}`."
same_db = same_db && (database == origin_database)
database_map["`#{origin_database}`\\."] = "`#{database}`."
end

gsubstring = sameDb ? "" : databaseMap.map { |k,v| ".gsub(/#{k}/, \"#{v}\")" }.join("")
replace_db_names_command = same_db ? "" : database_map.map { |k,v| ".gsub(/#{k}/, \"#{v}\")" }.join("")

Dir.entries(backupDir).reject {|f| File.directory? f}.sort.flat_map do |f|
Dir.entries(backup_dir).reject {|f| File.directory? f}.sort.flat_map do |f|

commands = []

index, origin_database = split_integer_and_string(f)
database = get_element_or_last(databases, index)

defaultOptions=mysqlDefaultOptions(@db_info, database)
backupDir=backupDirName(id, f)
default_options=mysql_default_options(@db_info, database)
backup_dir=backup_dir_name(id, f)

commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_tables.sql')} | mysql #{defaultOptions}") if isDropAllTables
commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_views.sql')} | mysql #{defaultOptions}") if isDropAllTables
commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_tables.sql')} | mysql #{default_options}") if is_drop_all_tables
commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_views.sql')} | mysql #{default_options}") if is_drop_all_tables

Dir.entries(backupDir).reject {|f| File.directory? f} .select {|f| f.include?("-schema.sql")} .each {|f|
restore_each(commands, backupDir+"/"+f, defaultOptions, gsubstring)
Dir.entries(backup_dir).reject {|f| File.directory? f}.select {|f| f.include?("-schema.sql")}.each {|f|
restore_each(commands, backup_dir+"/"+f, default_options, replace_db_names_command)
}
Dir.entries(backupDir).reject {|f| File.directory? f} .reject {|f| f.include?("-schema.sql")} .each {|f|
restore_each(commands, backupDir+"/"+f, defaultOptions, gsubstring)
Dir.entries(backup_dir).reject {|f| File.directory? f}.reject {|f| f.include?("-schema.sql")}.each {|f|
restore_each(commands, backup_dir+"/"+f, default_options, replace_db_names_command)
}
commands
end
end

private

def restore_each(commands, file, defaultOptions, gsubstring)
def restore_each(commands, file, default_options, replace_db_names_command)
command = ""
replacing = " | ruby -pe '$_=$_#{gsubstring}'" unless gsubstring.empty?
replacing = " | ruby -pe '$_=$_#{replace_db_names_command}'" unless replace_db_names_command.empty?
if file.end_with? ".sql"
command = "cat #{file} #{replacing} | mysql #{defaultOptions}"
command = "cat #{file} #{replacing} | mysql #{default_options}"
elsif gzip_file?(file)
command = "zcat #{file} #{replacing} | mysql #{defaultOptions}"
command = "zcat #{file} #{replacing} | mysql #{default_options}"
else
puts "not supported file #{file}"
end

commands.push(command) if not command.empty?
commands.push(command) unless command.empty?
end

def split_integer_and_string(input)
Expand Down
14 changes: 7 additions & 7 deletions spec/mysql_db_tool/backup_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
RSpec.describe MySQLDBTool::Backup do

before do
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return ({
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return({
db_info: { user: 'test-user', password: '', host: 'my-host', database: ['test_db-abc'] },
data_tables: [],
ignore_tables: []
Expand All @@ -25,7 +25,7 @@
it 'backs up each database' do

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"mkdir -p backup-42/0_test_db-abc",
"mysqldump --no-data --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc > backup-42/0_test_db-abc/2024-07-17_42-schema.sql",
"mysqldump --no-create-info --single-transaction --skip-lock-tables --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc > backup-42/0_test_db-abc/2024-07-17_42-all-other-tables.sql"
Expand All @@ -41,7 +41,7 @@
it 'backs up with gzip' do

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"mkdir -p backup-42/0_test_db-abc",
"mysqldump --no-data --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc | gzip > backup-42/0_test_db-abc/2024-07-17_42-schema.sql.gz",
"mysqldump --no-create-info --single-transaction --skip-lock-tables --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc | gzip > backup-42/0_test_db-abc/2024-07-17_42-all-other-tables.sql.gz"
Expand All @@ -57,7 +57,7 @@
it 'backs up with gzip and set gzip suffix' do

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"mkdir -p backup-42/0_test_db-abc",
"mysqldump --no-data --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc | gzip > backup-42/0_test_db-abc/2024-07-17_42-schema.sql.Z",
"mysqldump --no-create-info --single-transaction --skip-lock-tables --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc | gzip > backup-42/0_test_db-abc/2024-07-17_42-all-other-tables.sql.Z"
Expand All @@ -72,14 +72,14 @@

it 'backs up with ignore table' do

allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return ({
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return({
db_info: { user: 'test-user', password: '', host: 'my-host', database: ['test_db-abc'] },
data_tables: [],
ignore_tables: ['ignore_table1', 'ignore_table2']
})

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"mkdir -p backup-42/0_test_db-abc",
"mysqldump --no-data --ignore-table=test_db-abc.ignore_table1 --ignore-table=test_db-abc.ignore_table2 --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc > backup-42/0_test_db-abc/2024-07-17_42-schema.sql",
"mysqldump --no-create-info --ignore-table=test_db-abc.ignore_table1 --ignore-table=test_db-abc.ignore_table2 --single-transaction --skip-lock-tables --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db-abc > backup-42/0_test_db-abc/2024-07-17_42-all-other-tables.sql"
Expand All @@ -95,7 +95,7 @@
it 'backs up with database option' do

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"mkdir -p backup-42/0_test_db2",
"mysqldump --no-data --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db2 > backup-42/0_test_db2/2024-07-17_42-schema.sql",
"mysqldump --no-create-info --single-transaction --skip-lock-tables --column-statistics=0 --ssl-mode=disabled -h my-host -u test-user test_db2 > backup-42/0_test_db2/2024-07-17_42-all-other-tables.sql"
Expand Down
14 changes: 6 additions & 8 deletions spec/mysql_db_tool/restore_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
let(:instance) { described_class.new(options) }

it 'restores one db' do
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return ({
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return({
db_info: { user: 'test-user', password: '', host: 'my-host', database: ['test_db-abc'] },
data_tables: [],
ignore_tables: []
Expand All @@ -24,12 +24,12 @@
allow(Dir).to receive(:entries).with('backup-0/0-my-db').and_return(['.', '..', 'a.sql'])

commands = instance.perform
expect(commands).to eq ([
expect(commands).to eq([
"cat backup-0/0-my-db/a.sql | ruby -pe '$_=$_.gsub(/``\\./, \"`test_db-abc`.\")' | mysql --ssl-mode=disabled -h my-host -u test-user test_db-abc "
])
end
it 'restores each database' do
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return ({
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return({
db_info: { user: 'test-user', password: '', host: 'my-host', database: ['db-1', 'db-2'] },
data_tables: [],
ignore_tables: []
Expand All @@ -39,8 +39,7 @@
allow(Dir).to receive(:entries).with('backup-0/1_db-2').and_return(['.', '..', 'b.sql'])

commands = instance.perform
puts "commands=#{commands}"
expect(commands).to eq ([
expect(commands).to eq([
"cat backup-0/0_db-1/a.sql | mysql --ssl-mode=disabled -h my-host -u test-user db-1 ",
"cat backup-0/1_db-2/b.sql | mysql --ssl-mode=disabled -h my-host -u test-user db-2 "
])
Expand All @@ -53,7 +52,7 @@

it 'restores' do

allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return ({
allow(MySQLDBTool::Config::ConfigLoader).to receive(:load).and_return({
db_info: { user: 'test-user', password: '', host: 'my-host', database: ['db-1', 'db-2'] },
data_tables: [],
ignore_tables: []
Expand All @@ -63,8 +62,7 @@
allow(Dir).to receive(:entries).with('backup-0/1_db-2').and_return(['.', '..', 'b.sql'])

commands = instance.perform
puts "commands=#{commands}"
expect(commands).to eq ([
expect(commands).to eq([
"cat backup-0/0_db-1/a.sql | ruby -pe '$_=$_.gsub(/`db-1`\\./, \"`rdb-1`.\").gsub(/`db-2`\\./, \"`rdb-2`.\")' | mysql --ssl-mode=disabled -h my-host -u test-user rdb-1 ",
"cat backup-0/1_db-2/b.sql | ruby -pe '$_=$_.gsub(/`db-1`\\./, \"`rdb-1`.\").gsub(/`db-2`\\./, \"`rdb-2`.\")' | mysql --ssl-mode=disabled -h my-host -u test-user rdb-2 "
])
Expand Down

0 comments on commit 17a5a69

Please sign in to comment.