refactored the code

mlibrary · May 14, 2024 · 9e951f5 · 9e951f5
1 parent a000dd0
commit 9e951f5
Show file tree

Hide file tree

Showing 7 changed files with 157 additions and 61 deletions.
diff --git a/lib/config.rb b/lib/config.rb
@@ -207,6 +207,7 @@ def get_subset_by_key_stem(stem)
     :push_gateway_url,
     keyword_init: true
   )
+
   Config = Struct.new(
     "Config",
     :settings,
@@ -333,9 +334,5 @@ def self.from_env
     def self.log_level_from_env
       CheckableData.new(ENV.to_hash).get_value(key: "SETTINGS_LOG_LEVEL", checks: [LOG_LEVEL_CHECK]).to_sym
     end
-
-    def self.push_gateway_from_env
-      CheckableData.new(ENV.to_hash).get_value(key: "PROMETHEUS_PUSH_GATEWAY")
-    end
   end
 end
diff --git a/lib/metrics_collector.rb b/lib/metrics_collector.rb
@@ -20,10 +20,11 @@ def self.time_processing
   end
 
   class MetricsProvider
-    def initialize(start_time:, end_time:, status_event_repo:)
+    def initialize(start_time:, end_time:, status_event_repo:, push_gateway_url:)
       @start_time = start_time
       @end_time = end_time
       @status_event_repo = status_event_repo
+      @push_gateway_url = push_gateway_url
     end
 
     def get_latest_bag_events_by_time
@@ -61,20 +62,24 @@ def set_failed_count(events_by_time)
       dark_blue_failed_count.set(get_failure_count(events_by_time))
     end
 
-    # def set_failed_bag_id(events_by_time)
-    #   dark_blue_failed_bag_ids = registry.counter(
-    #     :dark_blue_failed_bag_ids,
-    #     docstring: "Failed bag transfer")
-    #   get_failed_ids = get_failed_bag_ids(events_by_time)
-    #   get_failed_ids.each do |e|
-    #     dark_blue_failed_bag_ids.increment({failed_id: e.bag_identifier},0)
-    #   end
-    # end
+    def set_failed_bag_id(events_by_time)
+      dark_blue_failed_bag_ids = registry.counter(
+        :dark_blue_failed_bag_ids,
+        docstring: "Failed bag transfer",
+        labels: [:failed_id]
+      )
+      get_failed_ids = get_failed_bag_ids(events_by_time)
+      get_failed_ids.each do |e|
+        dark_blue_failed_bag_ids.increment(labels: {failed_id: e.bag_identifier})
+      end
+    end
 
     def set_last_successful_run
       dark_blue_last_successful_run = registry.gauge(:dark_blue_last_successful_run,
         docstring: "Timestamp of the last successful run of the cron job")
       return unless dark_blue_last_successful_run
+      # converting starttime to milliseconds to support converting epoch time to datetime
+      # https://github.com/grafana/grafana/issues/6297
       time_in_milli_sec = (@start_time * 1000)
       dark_blue_last_successful_run.set(time_in_milli_sec)
     end
@@ -92,7 +97,7 @@ def set_all_metrics
       latest_events = get_latest_bag_events_by_time
       set_success_count(latest_events)
       set_failed_count(latest_events)
-      # set_failed_bag_id(latest_events)
+      set_failed_bag_id(latest_events)
       push_metrics
     end
 
@@ -105,7 +110,7 @@ def registry
     def gateway
       @gateway ||= Prometheus::Client::Push.new(
         job: "DarkBlueMetric",
-        gateway: S.config.metrics.push_gateway_url
+        gateway: @push_gateway_url
       )
     end
 

diff --git a/lib/status_event_repository.rb b/lib/status_event_repository.rb
@@ -150,8 +150,11 @@ def get_all_for_bag_identifier(identifier)
         .map { |se| convert_to_struct(se) }
     end
 
+    # https://sequel.jeremyevans.net/rdoc/classes/Sequel/SQL/Window.html
     def get_latest_event_for_bags(start_time:)
-      latest_events = base_query.where { timestamp >= start_time }.select_append { row_number.function.over(partition: :bag_id, order: Sequel.desc(:timestamp)).as(:rn) }.from_self.where(rn: 1)
+      latest_events = base_query.where { timestamp >= start_time }
+        .select_append { row_number.function.over(partition: :bag_id, order: Sequel.desc(:timestamp)).as(:rn) }
+        .from_self.where(rn: 1)
       latest_events.all.map { |se| convert_to_struct(se) }
     end
 

diff --git a/run_dark_blue.rb b/run_dark_blue.rb
@@ -36,7 +36,7 @@ def initialize(config)
         type: config.aptrust.remote.type,
         settings: config.aptrust.remote.settings
       ),
-      status_event_repo: StatusEventRepository::StatusEventRepositoryFactory.for(use_db: DB),
+      status_event_repo: S.status_event_repo,
       bag_repo: BagRepository::BagRepositoryFactory.for(use_db: DB)
     )
     @arch_configs = config.dark_blue.archivematicas
@@ -210,6 +210,6 @@ def self.parse(options)
     dark_blue_job.process
   end
 }
-@status_event_repo = StatusEventRepository::StatusEventRepositoryFactory.for(use_db: DB)
-metrics = DarkBlueMetrics::MetricsProvider.new(start_time: start_time, end_time: end_time, status_event_repo: @status_event_repo)
+metrics = DarkBlueMetrics::MetricsProvider.new(start_time: start_time, end_time: end_time,
+  status_event_repo: S.status_event_repo, push_gateway_url: config.metrics.push_gateway_url)
 metrics.set_all_metrics
diff --git a/services.rb b/services.rb
@@ -44,3 +44,8 @@ def self.included(klass)
     password: db_config.password,
     fractional_seconds: true)
 end
+
+S.register(:status_event_repo) do
+  db = S.config.database && S.dbconnect
+  StatusEventRepository::StatusEventRepositoryFactory.for(use_db: db)
+end
diff --git a/test/test_metrics_collector.rb b/test/test_metrics_collector.rb
@@ -16,51 +16,13 @@ def setup
     @start_time = @time_stamp.to_i
     @end_time = @start_time + 5
     @status_event_repo = StatusEventRepository::StatusEventInMemoryRepository.new
+    @push_gateway_url = "http://fake.pushgateway"
 
-    @bag_identifier_one = "repository.context-0001"
-    @bag_identifier_two = "repository.context-0002"
-    @deposited_at = Time.utc(2024, 3, 18)
-
-    @status_event_repo.create(
-      bag_identifier: @bag_identifier_one,
-      status: BagStatus::DEPOSITED,
-      timestamp: @deposited_at
-    )
-
-    @status_event_repo.create(
-      bag_identifier: @bag_identifier_two,
-      status: BagStatus::FAILED,
-      timestamp: @deposited_at
-    )
-
-    @metrics = DarkBlueMetrics::MetricsProvider.new(start_time: @start_time, end_time: @end_time, status_event_repo: @status_event_repo)
-
-    t = @metrics.get_latest_bag_events_by_time
-    p t
+    @metrics = DarkBlueMetrics::MetricsProvider.new(start_time: @start_time, end_time: @end_time, status_event_repo: @status_event_repo, push_gateway_url: @push_gateway_url)
     @registry = Prometheus::Client::Registry.new
     @gauge_mock = Minitest::Mock.new
   end
 
-  def mixin_repo
-    @status_event_repo
-  end
-
-  def mixin_bag_repo
-    @bag_repo
-  end
-
-  def mixin_bag_identifier
-    @bag_identifier
-  end
-
-  def mixin_package_repo
-    @package_repo
-  end
-
-  def mixin_package_identifier
-    @package_identifier
-  end
-
   def test_initialize
     assert_equal @start_time, @metrics.instance_variable_get(:@start_time)
     assert_equal @end_time, @metrics.instance_variable_get(:@end_time)
@@ -85,6 +47,23 @@ def test_set_processing_duration
   end
 
   def test_set_success_count
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
+
     expected = 1
     @registry.stub(:gauge, @gauge_mock) do
       events_by_time = @metrics.get_latest_bag_events_by_time
@@ -95,6 +74,22 @@ def test_set_success_count
   end
 
   def test_set_failed_count
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
     expected = 1
     @registry.stub(:gauge, @gauge_mock) do
       events_by_time = @metrics.get_latest_bag_events_by_time
@@ -105,25 +100,116 @@ def test_set_failed_count
   end
 
   def test_get_latest_bag_events_by_time
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
     actual_result = @metrics.get_latest_bag_events_by_time
     assert_equal 2, actual_result.length
   end
 
   def test_get_success_count
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
     events_by_time = @metrics.get_latest_bag_events_by_time
     actual_result = @metrics.get_success_count(events_by_time)
     assert_equal 1, actual_result
   end
 
   def test_get_failure_count
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
     events_by_time = @metrics.get_latest_bag_events_by_time
     actual_result = @metrics.get_failure_count(events_by_time)
     assert_equal 1, actual_result
   end
 
   def test_get_failed_bag_ids
+    @status_event_repo.status_events.clear
+    @bag_identifier_one = "repository.context-0001"
+    @bag_identifier_two = "repository.context-0002"
+    @deposited_at = Time.utc(2024, 3, 18)
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_one,
+      status: BagStatus::DEPOSITED,
+      timestamp: @deposited_at
+    )
+
+    @status_event_repo.create(
+      bag_identifier: @bag_identifier_two,
+      status: BagStatus::FAILED,
+      timestamp: @deposited_at
+    )
     events_by_time = @metrics.get_latest_bag_events_by_time
     actual_result = @metrics.get_failed_bag_ids(events_by_time)
     assert_equal "repository.context-0002", actual_result[0].bag_identifier
   end
+
+  def test_get_latest_bag_events_by_time_empty_array
+    @status_event_repo.status_events.clear
+    actual_result = @metrics.get_latest_bag_events_by_time
+    assert_equal [], actual_result
+  end
+
+  def test_get_success_count_nil
+    @status_event_repo.status_events.clear
+    events_by_time = @metrics.get_latest_bag_events_by_time
+    actual_result = @metrics.get_success_count(events_by_time)
+    assert_equal 0, actual_result
+  end
+
+  def test_get_failure_count_nil
+    @status_event_repo.status_events.clear
+    events_by_time = @metrics.get_latest_bag_events_by_time
+    actual_result = @metrics.get_failure_count(events_by_time)
+    assert_equal 0, actual_result
+  end
+
+  def test_get_failed_bag_ids_nil
+    @status_event_repo.status_events.clear
+    events_by_time = @metrics.get_latest_bag_events_by_time
+    actual_result = @metrics.get_failed_bag_ids(events_by_time)
+    assert_equal [], actual_result
+  end
 end
diff --git a/verify_aptrust.rb b/verify_aptrust.rb
@@ -21,7 +21,7 @@ def initialize(config)
       api_key: config.aptrust.api.api_key
     )
     @bag_repo = BagRepository::BagRepositoryFactory.for(use_db: DB)
-    @status_event_repo = StatusEventRepository::StatusEventRepositoryFactory.for(use_db: DB)
+    @status_event_repo = S.status_event_repo
     @verifier = APTrust::APTrustVerifier.new(
       aptrust_api: aptrust_api, status_event_repo: @status_event_repo
     )