From 236226aef5854264ef9aafd49714ecf41411b946 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 20 Sep 2023 17:49:07 -0400 Subject: [PATCH] Remove VimStrings from MiqQueue Events used to have vmware specific objects in the queue. These leaked through for a customer and it blew up. This has not been in the queue for a number of years, but if someone delays upgrading and then doesn't empty the queue before upgrading... this will happen. The solution is to remove entries --- ...5_remove_vim_strings_from_notifications.rb | 24 +-------- ...3_remove_vim_strings_from_miq_provision.rb | 24 +-------- ...move_vim_strings_from_custom_attributes.rb | 24 +-------- ...im_string_from_miq_request_task_options.rb | 24 +-------- ...43746_remove_vim_strings_from_miq_queue.rb | 16 ++++++ lib/migration_helper.rb | 20 +++++++ ..._remove_vim_strings_from_miq_queue_spec.rb | 54 +++++++++++++++++++ 7 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb create mode 100644 spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb diff --git a/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb b/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb index de216a6d0..d3d17228a 100644 --- a/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb +++ b/db/migrate/20200427122455_remove_vim_strings_from_notifications.rb @@ -7,30 +7,10 @@ class Notification < ActiveRecord::Base end def up - say_with_time("Removing VimStrings from Notifications") do - base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:VimString%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:VimString', '!ruby/string:String') end def down - say_with_time("Restoring VimStrings in Notifications") do - base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:String%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:String', '!ruby/string:VimString') end end diff --git a/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb b/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb index ec8998d55..eb9a20364 100644 --- a/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb +++ b/db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb @@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base end def up - say_with_time("Removing VimStrings from MiqRequestTask") do - base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:VimString%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:VimString', '!ruby/string:String', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:VimString', '!ruby/string:String') end def down - say_with_time("Restoring VimStrings in MiqRequestTask") do - base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:String%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:String', '!ruby/string:VimString', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:String', '!ruby/string:VimString') end end diff --git a/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb b/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb index 2c7f12350..6a41d4dc7 100644 --- a/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb +++ b/db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb @@ -7,30 +7,10 @@ class CustomAttribute < ActiveRecord::Base end def up - say_with_time("Removing VimStrings from CustomAttribute") do - base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:VimString%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:VimString', '!ruby/string:String', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:VimString', '!ruby/string:String') end def down - say_with_time("Restoring VimStrings from CustomAttribute") do - base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:String%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:String', '!ruby/string:VimString', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:String', '!ruby/string:VimString') end end diff --git a/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb b/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb index cf45175ef..5ae7b9d52 100644 --- a/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb +++ b/db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb @@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base end def up - say_with_time("Removing VimStrings from MiqRequestTask") do - base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:VimString%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:VimString', '!ruby/string:String') end def down - say_with_time("Restoring VimStrings from MiqRequestTask") do - base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:String%") - say_batch_started(base_relation.size) - - loop do - count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')") - break if count == 0 - - say_batch_processed(count) - end - end + regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:String', '!ruby/string:VimString') end end diff --git a/db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb b/db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb new file mode 100644 index 000000000..28b34785d --- /dev/null +++ b/db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb @@ -0,0 +1,16 @@ +class RemoveVimStringsFromMiqQueue < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + include MigrationHelper + + class MiqQueue < ActiveRecord::Base + end + + def up + regex_replace_column_value(MiqQueue, 'args', '!ruby/string:VimString', '!ruby/string:String') + regex_replace_column_value(MiqQueue, 'args', '!ruby/hash-with-ivars:VimHash', '!ruby/hash-with-ivars:Hash') + end + + # this is very old, not rolling back + def down + end +end diff --git a/lib/migration_helper.rb b/lib/migration_helper.rb index 395f0d0a3..d9e9dfb6a 100644 --- a/lib/migration_helper.rb +++ b/lib/migration_helper.rb @@ -265,4 +265,24 @@ def previously_migrated_as?(bad_date) "DELETE FROM schema_migrations WHERE version = #{connection.quote(bad_date)}" ) > 0 end + + # sometimes a string in a column needs to be renamed to another value + # this uses update_all and regex REPLACE to do it database side + # + # @param model model with the values + # @param column column with the values + # @param old_value previous string that wants to be changed + # @param new_value new string that the value will be changed to + def regex_replace_column_value(model, column, old_value, new_value, batch_size: 50_000) + say_with_time("Removing #{old_value} from #{model.name}.#{column}") do + base_relation = model.where("#{column} LIKE ?", "%#{old_value}%") + say_batch_started(base_relation.size) + loop do + count = base_relation.limit(batch_size).update_all("#{column} = REGEXP_REPLACE(#{column}, '#{old_value}', '#{new_value}', 'g')") + break if count < batch_size + + say_batch_processed(count) + end + end + end end diff --git a/spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb b/spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb new file mode 100644 index 000000000..5d9e90a67 --- /dev/null +++ b/spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb @@ -0,0 +1,54 @@ +require_migration + +# This is mostly necessary for data migrations, so feel free to delete this +# file if you do no need it. +describe RemoveVimStringsFromMiqQueue do + let(:miq_queue_stub) { migration_stub(:MiqQueue) } + + migration_context :up do + it "converts VimStrings to Strings" do + queue_args = <<~CONTEXT + --- + - :event_type: DrsMigrateVM_Task + :chain_id: &1 !ruby/string:VimString + str: '12345678' + xsiType: :SOAP::SOAPInt + vimType: + :is_task: true + :source: VC + :message: &3 !ruby/string:VimString + str: 'Task: Migrate virtual machine' + xsiType: :SOAP::SOAPString + vimType: + :timestamp: &2 !ruby/string:VimString + str: '2022-10-06T06:40:59.901255Z' + xsiType: :SOAP::SOAPDateTime + vimType: + :full_data: !ruby/hash-with-ivars:VimHash + elements: + key: !ruby/string:VimString + str: '12345678' + xsiType: :SOAP::SOAPInt + vimType: + chainId: *1 + createdTime: *2 + userName: !ruby/string:VimString + str: '' + xsiType: :SOAP::SOAPString + vimType: + CONTEXT + + queue_item = miq_queue_stub.create!( + :args => queue_args + ) + + migrate + queue_item.reload + + args = YAML.safe_load(queue_item.args, :permitted_classes => [Symbol, Date, String, Hash], :aliases => true) + arg = args.first + expect(arg.class).to eq(Hash) + expect(arg[:chain_id].class).to eq(String) + end + end +end