From 4afc32211c3621ef3ceced738c4d338f81dc1e5f Mon Sep 17 00:00:00 2001
From: Alex Gaziev <alex.gaziev@gmail.com>
Date: Sun, 27 Mar 2022 16:20:36 +0530
Subject: [PATCH 1/4] Some new features without readme for now. For discussion

---
 lib/evil_seed/configuration.rb      | 13 ++++++-
 lib/evil_seed/configuration/root.rb | 31 ++++++++++++++--
 lib/evil_seed/record_dumper.rb      | 11 ++++--
 lib/evil_seed/relation_dumper.rb    | 57 ++++++++++++++++++++++-------
 lib/evil_seed/root_dumper.rb        |  5 ++-
 5 files changed, 94 insertions(+), 23 deletions(-)

diff --git a/lib/evil_seed/configuration.rb b/lib/evil_seed/configuration.rb
index 0979964..4dc73d8 100644
--- a/lib/evil_seed/configuration.rb
+++ b/lib/evil_seed/configuration.rb
@@ -7,10 +7,15 @@
 module EvilSeed
   # This module holds configuration for creating dump: which models and their constraints
   class Configuration
-    attr_accessor :record_dumper_class
+    attr_accessor :record_dumper_class, :verbose, :verbose_sql, :unscoped, :dont_nullify, :skip_columns
 
     def initialize
       @record_dumper_class = RecordDumper
+      @verbose = false
+      @verbose_sql = false
+      @unscoped = true
+      @dont_nullify = false
+      @skip_columns = {}
     end
 
     def roots
@@ -18,7 +23,7 @@ def roots
     end
 
     def root(model, *constraints)
-      new_root = Root.new(model, *constraints)
+      new_root = Root.new(model, dont_nullify, *constraints)
       yield new_root if block_given?
       roots << new_root
     end
@@ -38,5 +43,9 @@ def anonymize(model_class, &block)
     def customizers
       @customizers ||= Hash.new { |h, k| h[k] = [] }
     end
+
+    def add_skip_columns(table_name, column_names)
+      @skip_columns[table_name] = column_names
+    end
   end
 end
diff --git a/lib/evil_seed/configuration/root.rb b/lib/evil_seed/configuration/root.rb
index 9ea3ada..12a518a 100644
--- a/lib/evil_seed/configuration/root.rb
+++ b/lib/evil_seed/configuration/root.rb
@@ -5,16 +5,19 @@ class Configuration
     # Configuration for dumping some root model and its associations
     class Root
       attr_reader :model, :constraints
-      attr_reader :total_limit, :association_limits
-      attr_reader :exclusions
+      attr_reader :total_limit, :association_limits, :deep_limit, :dont_nullify
+      attr_reader :exclusions, :inclusions
 
       # @param model       [String]       Name of the model class to dump
       # @param constraints [String, Hash] Everything you can feed into +where+ to limit number of records
-      def initialize(model, *constraints)
+      def initialize(model, dont_nullify, *constraints)
         @model = model
         @constraints = constraints
         @exclusions = []
+        @inclusions = []
         @association_limits = {}
+        @deep_limit = nil
+        @dont_nullify = dont_nullify
       end
 
       # Exclude some of associations from the dump
@@ -23,6 +26,12 @@ def exclude(*association_patterns)
         @exclusions += association_patterns
       end
 
+      # Exclude some of associations from the dump
+      # @param association_patterns Array<String, Regex> Patterns to exclude associated models from dump
+      def include(*association_patterns)
+        @inclusions += association_patterns
+      end
+
       # Limit number of records in all (if pattern is not provided) or given  associations to include into dump
       # @param limit               [Integer]       Maximum number of records in associations to include into dump
       # @param association_pattern [String, Regex] Pattern to limit number of records for certain associated models
@@ -34,8 +43,22 @@ def limit_associations_size(limit, association_pattern = nil)
         end
       end
 
+      # Limit deepenes of associations to include into dump
+      # @param limit               [Integer]       Maximum level to recursively dive into associations
+      def limit_deep(limit)
+        @deep_limit = limit
+      end
+
+      def do_not_nullify(nullify_flag)
+        @dont_nullify = nullify_flag
+      end
+
       def excluded?(association_path)
-        exclusions.any? { |exclusion| exclusion.match(association_path) }
+        exclusions.any? { |exclusion| association_path.match(exclusion) } #.match(association_path) }
+      end
+
+      def included?(association_path)
+        inclusions.any? { |inclusion| association_path.match(inclusion) } #.match(association_path) }
       end
     end
   end
diff --git a/lib/evil_seed/record_dumper.rb b/lib/evil_seed/record_dumper.rb
index 1d1f7d2..a443b35 100644
--- a/lib/evil_seed/record_dumper.rb
+++ b/lib/evil_seed/record_dumper.rb
@@ -53,13 +53,17 @@ def transform_and_anonymize(attributes)
     def insert_statement
       connection = model_class.connection
       table_name = connection.quote_table_name(model_class.table_name)
-      columns    = model_class.column_names.map { |c| connection.quote_column_name(c) }.join(', ')
-      "INSERT INTO #{table_name} (#{columns}) VALUES\n"
+      columns    = model_class.column_names.select { |c| !configuration.skip_columns[model_class.table_name]&.include?(c) }
+      columns    = columns.map { |c| connection.quote_column_name(c) }
+      "INSERT INTO #{table_name} (#{columns.join(', ')}) VALUES\n"
     end
 
     def write!(attributes)
+      puts("-- #{relation_dumper.association_path}\n") if configuration.verbose_sql
       @output.write("-- #{relation_dumper.association_path}\n") && @header_written = true unless @header_written
+      puts(@tuples_written.zero? ? insert_statement : ",\n") if configuration.verbose_sql
       @output.write(@tuples_written.zero? ? insert_statement : ",\n")
+      puts("  (#{prepare(attributes).join(', ')})") if configuration.verbose_sql
       @output.write("  (#{prepare(attributes).join(', ')})")
       @tuples_written += 1
       @output.write(";\n") && @tuples_written = 0 if @tuples_written == MAX_TUPLES_PER_INSERT_STMT
@@ -73,9 +77,10 @@ def finalize!
 
     def prepare(attributes)
       attributes.map do |key, value|
+        next if configuration.skip_columns[model_class.table_name]&.include?(key)
         type = model_class.attribute_types[key]
         model_class.connection.quote(type.serialize(value))
-      end
+      end.flatten.compact
     end
   end
 end
diff --git a/lib/evil_seed/relation_dumper.rb b/lib/evil_seed/relation_dumper.rb
index 4dec3cc..759db05 100644
--- a/lib/evil_seed/relation_dumper.rb
+++ b/lib/evil_seed/relation_dumper.rb
@@ -25,13 +25,15 @@ class RelationDumper
 
     attr_reader :relation, :root_dumper, :model_class, :association_path, :search_key, :identifiers, :nullify_columns,
                 :belongs_to_reflections, :has_many_reflections, :foreign_keys, :loaded_ids, :to_load_map,
-                :record_dumper, :inverse_reflection, :table_names, :options
+                :record_dumper, :inverse_reflection, :table_names, :options,
+                :current_deep, :verbose
 
-    delegate :root, :configuration, :total_limit, :loaded_map, to: :root_dumper
+    delegate :root, :configuration, :dont_nullify, :total_limit, :deep_limit, :loaded_map, to: :root_dumper
 
     def initialize(relation, root_dumper, association_path, **options)
       @relation               = relation
       @root_dumper            = root_dumper
+      @verbose                = configuration.verbose
       @identifiers            = options[:identifiers]
       @to_load_map            = Hash.new { |h, k| h[k] = [] }
       @foreign_keys           = Hash.new { |h, k| h[k] = [] }
@@ -46,31 +48,45 @@ def initialize(relation, root_dumper, association_path, **options)
       @belongs_to_reflections = setup_belongs_to_reflections
       @has_many_reflections   = setup_has_many_reflections
       @options                = options
+      @current_deep           = association_path.split('.').size
+      @dont_nullify           = dont_nullify
+
+      puts("- #{association_path}") if verbose
     end
 
     # Generate dump and write it into +io+
     # @return [Array<IO>] List of dump IOs for separate tables in order of dependencies (belongs_to are first)
     def call
       dump!
-      belongs_to_dumps = dump_belongs_to_associations!
-      has_many_dumps   = dump_has_many_associations!
-      [belongs_to_dumps, record_dumper.result, has_many_dumps].flatten.compact
+      if deep_limit and current_deep > deep_limit
+        [record_dumper.result].flatten.compact
+      else
+        belongs_to_dumps = dump_belongs_to_associations!
+        has_many_dumps   = dump_has_many_associations!
+        [belongs_to_dumps, record_dumper.result, has_many_dumps].flatten.compact
+      end
     end
 
     private
 
     def dump!
       if identifiers.present?
+        puts("  # #{search_key} => #{identifiers}") if verbose
         # Don't use AR::Base#find_each as we will get error on Oracle if we will have more than 1000 ids in IN statement
         identifiers.in_groups_of(MAX_IDENTIFIERS_IN_IN_STMT).each do |ids|
-          fetch_attributes(relation.where(search_key => ids.compact)).each do |attributes|
+          attrs = fetch_attributes(relation.where(search_key => ids.compact))
+          puts(" -- dumped #{attrs.size}") if verbose
+          attrs.each do |attributes|
             next unless check_limits!
             dump_record!(attributes)
           end
         end
       else
+        puts("  # #{relation.count}") if verbose
         relation.in_batches do |relation|
-          fetch_attributes(relation).each do |attributes|
+          attrs = fetch_attributes(relation)
+          puts(" -- dumped #{attrs.size}") if verbose
+          attrs.each do |attributes|
             next unless check_limits!
             dump_record!(attributes)
           end
@@ -79,8 +95,10 @@ def dump!
     end
 
     def dump_record!(attributes)
-      nullify_columns.each do |nullify_column|
-        attributes[nullify_column] = nil
+      unless dont_nullify
+        nullify_columns.each do |nullify_column|
+          attributes[nullify_column] = nil
+        end
       end
       return unless record_dumper.call(attributes)
       foreign_keys.each do |reflection_name, fk_column|
@@ -135,7 +153,11 @@ def check_limits!
     end
 
     def build_relation(reflection)
-      relation = reflection.klass.all
+      if configuration.unscoped
+        relation = reflection.klass.unscoped
+      else
+        relation = reflection.klass.all
+      end
       relation = relation.instance_eval(&reflection.scope) if reflection.scope
       relation = relation.where(reflection.type => model_class.to_s) if reflection.options[:as] # polymorphic
       relation
@@ -144,9 +166,13 @@ def build_relation(reflection)
     def setup_belongs_to_reflections
       model_class.reflect_on_all_associations(:belongs_to).reject do |reflection|
         next false if reflection.options[:polymorphic] # TODO: Add support for polymorphic belongs_to
+        included = root.included?("#{association_path}.#{reflection.name}")
         excluded = root.excluded?("#{association_path}.#{reflection.name}") || reflection.name == inverse_reflection
-        if excluded
-          nullify_columns << reflection.foreign_key if model_class.column_names.include?(reflection.foreign_key)
+        if excluded and not included
+          if model_class.column_names.include?(reflection.foreign_key)
+            puts(" -- excluded #{reflection.foreign_key}") if verbose
+            nullify_columns << reflection.foreign_key
+          end
         else
           foreign_keys[reflection.name] = reflection.foreign_key
           table_names[reflection.name]  = reflection.table_name
@@ -157,10 +183,15 @@ def setup_belongs_to_reflections
 
     # This method returns only direct has_one and has_many reflections. For HABTM it returns intermediate has_many
     def setup_has_many_reflections
+      puts(" -- reflections #{model_class._reflections.keys}") if verbose
       model_class._reflections.select do |_reflection_name, reflection|
         next false if model_class.primary_key.nil?
+
         next false if reflection.is_a?(ActiveRecord::Reflection::ThroughReflection)
-        %i[has_one has_many].include?(reflection.macro) && !root.excluded?("#{association_path}.#{reflection.name}")
+
+        included = root.included?("#{association_path}.#{reflection.name}")
+        excluded = root.excluded?("#{association_path}.#{reflection.name}") || reflection.name == inverse_reflection
+        %i[has_one has_many].include?(reflection.macro) && !(excluded and not included)
       end.map(&:second)
     end
   end
diff --git a/lib/evil_seed/root_dumper.rb b/lib/evil_seed/root_dumper.rb
index 1f104c9..1679fe2 100644
--- a/lib/evil_seed/root_dumper.rb
+++ b/lib/evil_seed/root_dumper.rb
@@ -5,7 +5,7 @@
 module EvilSeed
   # This module collects dumps generation for root and all it's dependencies
   class RootDumper
-    attr_reader :root, :dumper, :model_class, :total_limit, :association_limits
+    attr_reader :root, :dumper, :model_class, :total_limit, :deep_limit, :dont_nullify, :association_limits
 
     delegate :loaded_map, :configuration, to: :dumper
 
@@ -14,6 +14,8 @@ def initialize(root, dumper)
       @dumper = dumper
       @to_load_map = {}
       @total_limit = root.total_limit
+      @deep_limit = root.deep_limit
+      @dont_nullify = root.dont_nullify
       @association_limits = root.association_limits.dup
 
       @model_class = root.model.constantize
@@ -24,6 +26,7 @@ def initialize(root, dumper)
     def call
       association_path = model_class.model_name.singular
       relation = model_class.all
+      relation = relation.unscoped if configuration.unscoped
       relation = relation.where(*root.constraints) if root.constraints.any? # without arguments returns not a relation
       RelationDumper.new(relation, self, association_path).call
     end

From fad4ff78dbd9ea22c1e3a8db939d35e2203f5ca8 Mon Sep 17 00:00:00 2001
From: Polina Gurtovaya <zloymult@gmail.com>
Date: Wed, 30 Mar 2022 17:43:02 +0300
Subject: [PATCH 2/4] Add included for relations

---
 lib/evil_seed/relation_dumper.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/evil_seed/relation_dumper.rb b/lib/evil_seed/relation_dumper.rb
index 759db05..0ed0102 100644
--- a/lib/evil_seed/relation_dumper.rb
+++ b/lib/evil_seed/relation_dumper.rb
@@ -177,7 +177,7 @@ def setup_belongs_to_reflections
           foreign_keys[reflection.name] = reflection.foreign_key
           table_names[reflection.name]  = reflection.table_name
         end
-        excluded
+        excluded and not included
       end
     end
 

From 6bc83be5a5a9f8e077c7d541308428818bdcea28 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <envek@envek.name>
Date: Tue, 18 Jun 2024 22:30:20 +0900
Subject: [PATCH 3/4] Add tests, examples to README, switch unscoped default
 for compatibility

---
 README.md                           | 19 +++++++++++++++++++
 lib/evil_seed/configuration.rb      |  2 +-
 lib/evil_seed/configuration/root.rb |  2 +-
 test/db/models.rb                   |  2 ++
 test/db/schema.rb                   |  1 +
 test/db/seeds.rb                    |  6 ++++++
 test/evil_seed/dumper_test.rb       | 18 ++++++++++++++++++
 7 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b3f0158..3331bf6 100644
--- a/README.md
+++ b/README.md
@@ -60,6 +60,11 @@ EvilSeed.configure do |config|
 
     # Or for certain association only
     root.limit_associations_size(10, 'forum.questions')
+
+    # Limit the depth of associations to be dumped from the root level
+    # All traverses through has_many, belongs_to, etc are counted
+    # So forum.subforums.subforums.questions.answers will be 5 levels deep
+    root.limit_deep(10)
   end
 
   # Everything you can pass to +where+ method will work as constraints:
@@ -95,6 +100,20 @@ EvilSeed.configure do |config|
   # This will remove the columns even if the model is not a root node and is
   # dumped via an association.
   config.ignore_columns("Profile", :name)
+
+  # Disable foreign key nullification for records that are not included in the dump
+  # By default, EvilSeed will nullify foreign keys for records that are not included in the dump
+  config.dont_nullify = true
+
+  # Unscope relations to include soft-deleted records etc
+  # This is useful when you want to include all records, including those that are hidden by default
+  # By default, EvilSeed will abide default scope of models
+  config.unscoped = true
+
+  # Verbose mode will print out the progress of the dump to the console along with writing the file
+  # By default, verbose mode is off
+  config.verbose = true
+  config.verbose_sql = true
 end
 ```
 
diff --git a/lib/evil_seed/configuration.rb b/lib/evil_seed/configuration.rb
index df032ca..b69a954 100644
--- a/lib/evil_seed/configuration.rb
+++ b/lib/evil_seed/configuration.rb
@@ -13,7 +13,7 @@ def initialize
       @record_dumper_class = RecordDumper
       @verbose = false
       @verbose_sql = false
-      @unscoped = true
+      @unscoped = false
       @dont_nullify = false
       @ignored_columns = Hash.new { |h, k| h[k] = [] }
     end
diff --git a/lib/evil_seed/configuration/root.rb b/lib/evil_seed/configuration/root.rb
index 12a518a..933314a 100644
--- a/lib/evil_seed/configuration/root.rb
+++ b/lib/evil_seed/configuration/root.rb
@@ -26,7 +26,7 @@ def exclude(*association_patterns)
         @exclusions += association_patterns
       end
 
-      # Exclude some of associations from the dump
+      # Include some excluded associations back to the dump
       # @param association_patterns Array<String, Regex> Patterns to exclude associated models from dump
       def include(*association_patterns)
         @inclusions += association_patterns
diff --git a/test/db/models.rb b/test/db/models.rb
index 5fa4217..a71ffc6 100644
--- a/test/db/models.rb
+++ b/test/db/models.rb
@@ -49,6 +49,8 @@ class Answer < ActiveRecord::Base
   has_many :votes, as: :votable
 
   has_many :voters, through: :votes, source: :user
+
+  default_scope { where(deleted_at: nil) }
 end
 
 class TrackingPixel < ActiveRecord::Base
diff --git a/test/db/schema.rb b/test/db/schema.rb
index 30c20de..308bc29 100644
--- a/test/db/schema.rb
+++ b/test/db/schema.rb
@@ -40,6 +40,7 @@ def create_schema!
       t.boolean    :best, default: false
       t.text       :text
       t.references :author
+      t.datetime   :deleted_at
       t.timestamps null: false
     end
     add_foreign_key :answers, :users, column: :author_id, on_delete: :nullify
diff --git a/test/db/seeds.rb b/test/db/seeds.rb
index 2a2f77c..25b8a64 100644
--- a/test/db/seeds.rb
+++ b/test/db/seeds.rb
@@ -43,6 +43,12 @@
   author: User.find_by!(login: 'bob'),
 )
 
+answer = question.answers.create!(
+  text:   'Oops, I was wrong',
+  author: User.find_by!(login: 'eva'),
+  deleted_at: Time.current,
+)
+
 answer.votes.create!(user: User.find_by!(login: 'eva'))
 
 question_attrs = %w[first second third fourth fifth].map { |name| { name: name, forum: forums.first } }
diff --git a/test/evil_seed/dumper_test.rb b/test/evil_seed/dumper_test.rb
index e086c77..dfbc7b9 100644
--- a/test/evil_seed/dumper_test.rb
+++ b/test/evil_seed/dumper_test.rb
@@ -42,6 +42,7 @@ def test_it_dumps_tree_structures_with_foreign_keys
       assert_match(/'fourth'/, result)
       assert_match(/'fifth'/, result)
       assert result.index(/'One'/) < result.index(/'Descendant forum'/)
+      refute_match(/'Oops, I was wrong'/, result)
     end
 
     def test_limits_being_applied
@@ -59,5 +60,22 @@ def test_limits_being_applied
       assert_match(/'fourth'/, result)
       refute_match(/'fifth'/, result)
     end
+
+    def test_it_applies_unscoping_and_inclusions
+      configuration = EvilSeed::Configuration.new
+      configuration.root('Forum', name: 'Descendant forum') do |root|
+        root.include(/forum(\.parent(\.questions(\.answers)?)?)?\z/)
+        root.exclude(/.\..+/)
+      end
+      configuration.unscoped = true
+
+      io = StringIO.new
+      EvilSeed::Dumper.new(configuration).call(io)
+      result = io.string
+      File.write(File.join('tmp', "#{__method__}.sql"), result)
+      assert io.closed?
+      assert_match(/'Descendant forum'/, result)
+      assert_match(/'Oops, I was wrong'/, result)
+    end
   end
 end

From c663a973115c129bbef4a36f4bf01e2095df6a56 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <envek@envek.name>
Date: Tue, 18 Jun 2024 23:04:13 +0900
Subject: [PATCH 4/4] Add inclusion example to the readme

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3331bf6..836ef78 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,10 @@ EvilSeed.configure do |config|
     #
     # Association path is a dot-delimited string of association chain starting from model itself:
     # example: "forum.users.questions"
-    root.exclude(/\btracking_pixels\b/, 'forum.popular_questions')
+    root.exclude(/\btracking_pixels\b/, 'forum.popular_questions', /\Aforum\.parent\b/)
+
+    # Include back only certain associations
+    root.include(/\Aforum(\.parent(\.questions(\.answers)?)?)?\z/)
 
     # It's possible to limit the number of included into dump has_many and has_one records for every association
     # Note that belongs_to records for all not excluded associations are always dumped to keep referential integrity.