From ad29f1159a2b8b8eeee91427f7437fb06437b3c9 Mon Sep 17 00:00:00 2001
From: Justin Littman <justinlittman@stanford.edu>
Date: Mon, 16 Oct 2023 16:50:09 -0400
Subject: [PATCH] Adds connection pool for redis. Increases timeout for redis.

---
 Gemfile                         |  1 +
 Gemfile.lock                    |  1 +
 app/jobs/concerns/unique_job.rb | 10 ++--------
 config/initializers/redis.rb    |  7 +++++++
 config/initializers/sidekiq.rb  |  7 +++++--
 config/settings.yml             |  4 +++-
 6 files changed, 19 insertions(+), 11 deletions(-)
 create mode 100644 config/initializers/redis.rb

diff --git a/Gemfile b/Gemfile
index 91dcdbd67..1b9905177 100644
--- a/Gemfile
+++ b/Gemfile
@@ -3,6 +3,7 @@ source 'https://rubygems.org'
 # general Ruby/Rails gems
 gem 'aws-sdk-s3', '~> 1.17'
 gem 'committee' # Validates HTTP requests/responses per OpenAPI specification
+gem 'connection_pool' # Used for redis
 gem 'config' # Settings to manage configs on different instances
 gem 'honeybadger' # for error reporting / tracking / notifications
 gem 'jbuilder', '~> 2.5' # Build JSON APIs with ease.
diff --git a/Gemfile.lock b/Gemfile.lock
index 91f0025a1..8a524a40a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -461,6 +461,7 @@ DEPENDENCIES
   capybara
   committee
   config
+  connection_pool
   debug
   dlss-capistrano
   dor-event-client (~> 1.0)
diff --git a/app/jobs/concerns/unique_job.rb b/app/jobs/concerns/unique_job.rb
index 1d7d716f7..80372d8d5 100644
--- a/app/jobs/concerns/unique_job.rb
+++ b/app/jobs/concerns/unique_job.rb
@@ -72,12 +72,6 @@ def lock_timeout
       3600
     end
 
-    def redis_connection
-      Redis.new(url: Settings.redis_url).tap do |conn|
-        yield conn if block_given?
-      end
-    end
-
     # @return [String] the key for locking this job/payload combination, e.g. 'lock:MySpecificJob-bt821jk7040;1'
     def queue_lock_key(*args)
       # Changes in ActiveModel object don't result in new lock (which they do when just calling to_s).
@@ -90,7 +84,7 @@ def before_enqueue_lock(*)
       now = Time.now.to_i
       new_expiry_time = now + lock_timeout + 1
 
-      redis_connection do |conn|
+      REDIS.with do |conn|
         # return true if we successfully acquired the lock
         # "Set key to hold string value if key does not exist" (otherwise no-op) -- https://redis.io/commands/setnx
         if conn.setnx(key, new_expiry_time)
@@ -123,7 +117,7 @@ def before_enqueue_lock(*)
     def clear_lock(*)
       key = queue_lock_key(*)
       Rails.logger.info("clearing lock for #{key}...")
-      redis_connection do |conn|
+      REDIS.with do |conn|
         conn.del(key).tap do |del_result|
           Rails.logger.info("...cleared lock for #{key} (del_result=#{del_result})")
         end
diff --git a/config/initializers/redis.rb b/config/initializers/redis.rb
new file mode 100644
index 000000000..112440a3f
--- /dev/null
+++ b/config/initializers/redis.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+pool_size = ENV.fetch('RAILS_MAX_THREADS', 5)
+
+REDIS = ConnectionPool.new(size: pool_size) do
+  Redis.new(url: Settings.redis_url, timeout: Settings.redis_timeout)
+end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index c2a01dce2..66c491a38 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
+# See separate initializer for redis used by unique job.
+
+# Note the increased timeouts to try to address Redis timeouts
 Sidekiq.configure_server do |config|
-  config.redis = { url: Settings.redis_url }
+  config.redis = { url: Settings.redis_url, network_timeout: Settings.redis_timeout, pool_timeout: Settings.redis_timeout }
   # For Sidekiq Pro
   config.super_fetch!
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = { url: Settings.redis_url }
+  config.redis = { url: Settings.redis_url, network_timeout: Settings.redis_timeout, pool_timeout: Settings.redis_timeout }
 end
 Sidekiq::Client.reliable_push! unless Rails.env.test?
diff --git a/config/settings.yml b/config/settings.yml
index e1bc37972..57375ecaf 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -59,6 +59,7 @@ rabbitmq:
   password: guest
 
 redis_url: redis://localhost:6379/
+redis_timeout: 5 # seconds
 
 honeybadger_checkins:
   moab_to_catalog: xyzzy
@@ -68,4 +69,5 @@ honeybadger_checkins:
 
 slow_queries:
   enable: false
-  threshold: 500
\ No newline at end of file
+  threshold: 500
+