Merge pull request #706 from kbrock/vim_string_migrations

Remove VimStrings from MiqQueue

db/migrate/20200427122455_remove_vim_strings_from_notifications.rb

@@ -7,30 +7,10 @@ class Notification < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from Notifications") do
-      base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings in Notifications") do
-      base_relation = Notification.in_my_region.where("options LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(Notification.in_my_region, 'options', '!ruby/string:String', '!ruby/string:VimString')
   end
 end

db/migrate/20200629194033_remove_vim_strings_from_miq_provision.rb

@@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings in MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("phase_context LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("phase_context = REGEXP_REPLACE(phase_context, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask.in_my_region, 'phase_context', '!ruby/string:String', '!ruby/string:VimString')
   end
 end

db/migrate/20221111201223_remove_vim_strings_from_custom_attributes.rb

@@ -7,30 +7,10 @@ class CustomAttribute < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from CustomAttribute") do
-      base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings from CustomAttribute") do
-      base_relation = CustomAttribute.in_my_region.where("serialized_value LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("serialized_value = REGEXP_REPLACE(serialized_value, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(CustomAttribute.in_my_region, 'serialized_value', '!ruby/string:String', '!ruby/string:VimString')
   end
 end

db/migrate/20221114165219_remove_vim_string_from_miq_request_task_options.rb

@@ -9,30 +9,10 @@ class MiqRequestTask < ActiveRecord::Base
   end
 
   def up
-    say_with_time("Removing VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:VimString%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:VimString', '!ruby/string:String', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:VimString', '!ruby/string:String')
   end
 
   def down
-    say_with_time("Restoring VimStrings from MiqRequestTask") do
-      base_relation = MiqRequestTask.in_my_region.where("options LIKE ?", "%ruby/string:String%")
-      say_batch_started(base_relation.size)
-
-      loop do
-        count = base_relation.limit(50_000).update_all("options = REGEXP_REPLACE(options, '!ruby/string:String', '!ruby/string:VimString', 'g')")
-        break if count == 0
-
-        say_batch_processed(count)
-      end
-    end
+    regex_replace_column_value(MiqRequestTask, 'options', '!ruby/string:String', '!ruby/string:VimString')
   end
 end

db/migrate/20230919143746_remove_vim_strings_from_miq_queue.rb

@@ -0,0 +1,16 @@
+class RemoveVimStringsFromMiqQueue < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+  include MigrationHelper
+
+  class MiqQueue < ActiveRecord::Base
+  end
+
+  def up
+    regex_replace_column_value(MiqQueue, 'args', '!ruby/string:VimString', '!ruby/string:String')
+    regex_replace_column_value(MiqQueue, 'args', '!ruby/hash-with-ivars:VimHash', '!ruby/hash-with-ivars:Hash')
+  end
+
+  # this is very old, not rolling back
+  def down
+  end
+end

lib/migration_helper.rb

@@ -265,4 +265,24 @@ def previously_migrated_as?(bad_date)
       "DELETE FROM schema_migrations WHERE version = #{connection.quote(bad_date)}"
     ) > 0
   end
+
+  # sometimes a string in a column needs to be renamed to another value
+  # this uses update_all and regex REPLACE to do it database side
+  #
+  # @param model     model with the values
+  # @param column    column with the values
+  # @param old_value previous string that wants to be changed
+  # @param new_value new string that the value will be changed to
+  def regex_replace_column_value(model, column, old_value, new_value, batch_size: 50_000)
+    say_with_time("Removing #{old_value} from #{model.name}.#{column}") do
+      base_relation = model.where("#{column} LIKE ?", "%#{old_value}%")
+      say_batch_started(base_relation.size)
+      loop do
+        count = base_relation.limit(batch_size).update_all("#{column} = REGEXP_REPLACE(#{column}, '#{old_value}', '#{new_value}', 'g')")
+        break if count < batch_size
+
+        say_batch_processed(count)
+      end
+    end
+  end
 end

spec/migrations/20230919143746_remove_vim_strings_from_miq_queue_spec.rb

@@ -0,0 +1,54 @@
+require_migration
+
+# This is mostly necessary for data migrations, so feel free to delete this
+# file if you do no need it.
+describe RemoveVimStringsFromMiqQueue do
+  let(:miq_queue_stub) { migration_stub(:MiqQueue) }
+
+  migration_context :up do
+    it "converts VimStrings to Strings" do
+      queue_args = <<~CONTEXT
+        ---
+        - :event_type: DrsMigrateVM_Task
+          :chain_id: &1 !ruby/string:VimString
+            str: '12345678'
+            xsiType: :SOAP::SOAPInt
+            vimType:
+          :is_task: true
+          :source: VC
+          :message: &3 !ruby/string:VimString
+            str: 'Task: Migrate virtual machine'
+            xsiType: :SOAP::SOAPString
+            vimType:
+          :timestamp: &2 !ruby/string:VimString
+            str: '2022-10-06T06:40:59.901255Z'
+            xsiType: :SOAP::SOAPDateTime
+            vimType:
+          :full_data: !ruby/hash-with-ivars:VimHash
+            elements:
+              key: !ruby/string:VimString
+                str: '12345678'
+                xsiType: :SOAP::SOAPInt
+                vimType:
+              chainId: *1
+              createdTime: *2
+              userName: !ruby/string:VimString
+                str: ''
+                xsiType: :SOAP::SOAPString
+                vimType:
+      CONTEXT
+
+      queue_item = miq_queue_stub.create!(
+        :args => queue_args
+      )
+
+      migrate
+      queue_item.reload
+
+      args = YAML.safe_load(queue_item.args, :permitted_classes => [Symbol, Date, String, Hash], :aliases => true)
+      arg = args.first
+      expect(arg.class).to eq(Hash)
+      expect(arg[:chain_id].class).to eq(String)
+    end
+  end
+end