From 4043e76d822ec95451a53794491b05ea59ed987a Mon Sep 17 00:00:00 2001 From: Justin Littman Date: Thu, 22 Sep 2022 13:11:15 -0400 Subject: [PATCH] Deploy with docker. --- .dockerignore | 1 + Capfile | 8 +--- Gemfile | 5 +- Gemfile.lock | 27 +++++------ app/jobs/assign_pid_job.rb | 2 +- app/jobs/deposit_status_job.rb | 2 +- app/jobs/record_embargo_release_job.rb | 2 +- config/deploy.rb | 29 +++-------- config/deploy/qa.rb | 3 +- config/environments/production.rb | 1 + config/puma.rb | 8 ++-- config/schedule.rb | 3 ++ config/settings.yml | 6 ++- db/seeds.rb | 2 +- docker-compose.prod.yml | 66 ++++++++++++++++++++++++++ docker/app-prod/Dockerfile | 53 +++++++++++++++++++++ lib/tasks/rabbitmq.rake | 6 +-- 17 files changed, 164 insertions(+), 60 deletions(-) create mode 100644 docker-compose.prod.yml create mode 100644 docker/app-prod/Dockerfile diff --git a/.dockerignore b/.dockerignore index d12e797bc..48b7cbb60 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,4 @@ postgres-data redis-data # Required for puma to have a place to put the pid file !tmp/pids/.keep +public/assets diff --git a/Capfile b/Capfile index 7fbea43ca..a8f6aa7bf 100644 --- a/Capfile +++ b/Capfile @@ -9,13 +9,9 @@ require 'capistrano/deploy' require 'capistrano/scm/git' install_plugin Capistrano::SCM::Git -require 'capistrano/bundler' -require 'capistrano/rails' -require 'capistrano/honeybadger' -require 'capistrano/passenger' require 'capistrano/maintenance' -require 'whenever/capistrano' -require 'dlss/capistrano' + +require 'dlss/docker/capistrano' # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/Gemfile b/Gemfile index 427b0b154..513c6f288 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,6 @@ group :development do gem 'faker' gem 'listen', '~> 3.2' gem 'multi_json', require: false # needed to update RBIs after adding reform-rails - gem 'puma', '~> 5.6', '>= 5.6.4' gem 'spring' gem 'spring-watcher-listen', '~> 2.0.0' gem 'state_machines-graphviz' @@ -41,9 +40,8 @@ end group :deployment do gem 'capistrano-maintenance', '~> 1.2', require: false - gem 'capistrano-passenger', require: false gem 'capistrano-rails', require: false - gem 'dlss-capistrano', require: false + gem 'dlss-capistrano-docker', github: 'sul-dlss/dlss-capistrano-docker', branch: 'initial', require: false end gem 'action_policy', '~> 0.5.3' @@ -66,6 +64,7 @@ gem 'okcomputer' gem 'pg' gem 'propshaft' gem 'pry' +gem 'puma', '~> 5.6', '>= 5.6.4' gem 'redis', '~> 4.0' # TODO: Deal with this # pinned because 2.6.0 broke the build: [Reform] Your :populator did not return a Reform::Form instance for `authors`. diff --git a/Gemfile.lock b/Gemfile.lock index b7a66eca5..9c41b3417 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,13 @@ +GIT + remote: https://github.com/sul-dlss/dlss-capistrano-docker.git + revision: f180a5fc7329c506b0ad89bd0f8f821972a04fa6 + branch: initial + specs: + dlss-capistrano-docker (0.0.1) + capistrano (~> 3.0) + capistrano-one_time_key + capistrano-shared_configs + GEM remote: https://rubygems.org/ specs: @@ -80,9 +90,6 @@ GEM bootsnap (1.13.0) msgpack (~> 1.2) builder (3.2.4) - bundler-audit (0.9.1) - bundler (>= 1.2.0, < 3) - thor (~> 1.0) bunny (2.19.0) amq-protocol (~> 2.3, >= 2.3.1) sorted_set (~> 1, >= 1.0.2) @@ -92,18 +99,12 @@ GEM i18n rake (>= 10.0.0) sshkit (>= 1.9.0) - capistrano-bundle_audit (0.4.0) - bundler-audit (~> 0.5) - capistrano (~> 3.0) - capistrano-bundler (>= 1.4) capistrano-bundler (2.1.0) capistrano (~> 3.1) capistrano-maintenance (1.2.1) capistrano (>= 3.0) capistrano-one_time_key (0.1.0) capistrano (~> 3.0) - capistrano-passenger (0.2.1) - capistrano (~> 3.0) capistrano-rails (1.6.2) capistrano (~> 3.1) capistrano-bundler (>= 1.1, < 3) @@ -181,11 +182,6 @@ GEM declarative-option (< 0.2.0) representable (>= 2.4.0, <= 3.1.0) uber (< 0.2.0) - dlss-capistrano (4.3.1) - capistrano (~> 3.0) - capistrano-bundle_audit (>= 0.3.0) - capistrano-one_time_key - capistrano-shared_configs docile (1.4.0) druid-tools (3.0.0) dry-configurable (0.15.0) @@ -549,7 +545,6 @@ DEPENDENCIES bunny (~> 2.17) byebug capistrano-maintenance (~> 1.2) - capistrano-passenger capistrano-rails capybara (~> 3.34) capybara-screenshot @@ -560,7 +555,7 @@ DEPENDENCIES cypress-rails devise (~> 4.7) devise-remote-user (~> 1.0) - dlss-capistrano + dlss-capistrano-docker! druid-tools dry-types edtf diff --git a/app/jobs/assign_pid_job.rb b/app/jobs/assign_pid_job.rb index 9593f915f..79b716bea 100644 --- a/app/jobs/assign_pid_job.rb +++ b/app/jobs/assign_pid_job.rb @@ -6,7 +6,7 @@ class AssignPidJob # This worker will connect to "h2.druid_assigned" queue # env is set to nil since by default the actual queue name would be # "h2.druid_assigned_development" - from_queue 'h2.druid_assigned', env: nil + from_queue Settings.rabbitmq.queues.druid_assigned, env: nil def work(msg) model = build_cocina_model_from_json_str(msg) diff --git a/app/jobs/deposit_status_job.rb b/app/jobs/deposit_status_job.rb index be96de414..d25a58adf 100644 --- a/app/jobs/deposit_status_job.rb +++ b/app/jobs/deposit_status_job.rb @@ -11,7 +11,7 @@ class DepositStatusJob # example, if the embargo was lifted, DSA would open and close a version. The # workflow message "end-accession" would end up here. We must be able to handle # these messages in addition to those that result from depositing in h2. - from_queue 'h2.deposit_complete', env: nil + from_queue Settings.rabbitmq.queues.deposit_complete, env: nil def work(msg) druid = parse_message(msg) diff --git a/app/jobs/record_embargo_release_job.rb b/app/jobs/record_embargo_release_job.rb index 571796e3b..d5f7feebd 100644 --- a/app/jobs/record_embargo_release_job.rb +++ b/app/jobs/record_embargo_release_job.rb @@ -6,7 +6,7 @@ class RecordEmbargoReleaseJob # This worker will connect to "h2.embargo_lifted" queue # env is set to nil since by default the actual queue name would be # "h2.embargo_lifted_development" - from_queue 'h2.embargo_lifted', env: nil + from_queue Settings.rabbitmq.queues.embargo_lifted, env: nil def work(msg) model = build_cocina_model_from_json_str(msg) diff --git a/config/deploy.rb b/config/deploy.rb index 121a7ffe5..b98a28197 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -22,14 +22,11 @@ # Default value for :pty is false # set :pty, true -# Default value for :linked_files is [] -append :linked_files, - 'config/database.yml', # in Puppet - 'config/secrets.yml', # in shared_configs - 'config/honeybadger.yml' # in shared_configs +set :linked_files, %w[config/honeybadger.yml] -# Default value for linked_dirs is [] -append :linked_dirs, 'log', 'config/settings', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'public/system' +set :linked_dirs, %w[log config/settings] + +set :dereference_dirs, %w[config/settings] # Default value for default_env is {} # set :default_env, { path: "/opt/ruby/bin:$PATH" } @@ -46,21 +43,9 @@ # honeybadger_env otherwise defaults to rails_env set :honeybadger_env, fetch(:stage) -# Manage sidekiq via systemd (from dlss-capistrano gem) -set :sidekiq_systemd_use_hooks, true - -# Manage sneakers via systemd (from dlss-capistrano gem) -set :sneakers_systemd_use_hooks, true - # Set Rails env to production in all Cap environments set :rails_env, 'production' -# Deploy passenger-standalone via systemd service -set :passenger_restart_command, 'sudo systemctl restart passenger' -set :passenger_restart_options, -> { '' } - -set :whenever_environment, fetch(:rails_env) -set :whenever_roles, [:cron] - -# update shared_configs before restarting app (from dlss-capistrano gem) -before 'deploy:restart', 'shared_configs:update' +set :docker_compose_file, 'docker-compose.prod.yml' +set :docker_compose_seed_use_hooks, true +set :docker_compose_rabbitmq_use_hooks, true diff --git a/config/deploy/qa.rb b/config/deploy/qa.rb index 645246c6d..c0dcf2af4 100644 --- a/config/deploy/qa.rb +++ b/config/deploy/qa.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true -server 'sul-h2-qa.stanford.edu', user: 'h2', roles: %w[web app db] +# Roles are passed to docker-compose as profiles. +server 'h2-docker-qa.stanford.edu', user: 'h2', roles: %w[web app db cron worker] Capistrano::OneTimeKey.generate_one_time_key! diff --git a/config/environments/production.rb b/config/environments/production.rb index cb79d2520..3cecf6a1a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -66,6 +66,7 @@ config.action_mailer.perform_caching = false config.action_mailer.asset_host = "https://#{Settings.host}" + config.action_mailer.smtp_settings = { address: ENV.fetch('SMTP_HOST', 'localhost') } # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. diff --git a/config/puma.rb b/config/puma.rb index 69e2645fb..c5a455e58 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -6,8 +6,8 @@ # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. # -max_threads_count = ENV.fetch('RAILS_MAX_THREADS', 5) -min_threads_count = ENV.fetch('RAILS_MIN_THREADS') { max_threads_count } +max_threads_count = ENV.fetch('PUMA_MAX_THREADS', 5) +min_threads_count = ENV.fetch('PUMA_MIN_THREADS') { max_threads_count } threads min_threads_count, max_threads_count # Specifies the `port` that Puma will listen on to receive requests; default is 3000. @@ -27,14 +27,14 @@ # Workers do not work on JRuby or Windows (both of which do not support # processes). # -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } +workers ENV.fetch('PUMA_WORKERS', 2) # Use the `preload_app!` method when specifying a `workers` number. # This directive tells Puma to first boot the application and load code # before forking the application. This takes advantage of Copy On Write # process behavior so workers use less memory. # -# preload_app! +preload_app! # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart diff --git a/config/schedule.rb b/config/schedule.rb index 4b183ab82..bbc4dde1e 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -3,6 +3,9 @@ # Use this file to easily define all of your cron jobs. # Learn more: http://github.com/javan/whenever +# Execute without bash. +set :job_template, nil + every :day, at: '1:00am' do runner 'WorkReminderGenerator.send_draft_reminders' end diff --git a/config/settings.yml b/config/settings.yml index ed045c032..259f70372 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -58,7 +58,7 @@ purl_url: https://purl.stanford.edu terms_url: https://library.stanford.edu/research/stanford-digital-repository/documentation/sdr-terms-deposit sdr_url: https://library.stanford.edu/research/stanford-digital-repository -host: <%= Socket.gethostname %> +host: <%= ENV.fetch('HOST', Socket.gethostname) %> rabbitmq: enabled: false @@ -66,6 +66,10 @@ rabbitmq: vhost: / username: guest password: guest + queues: + deposit_complete: 'h2.deposit_complete' + druid_assigned: 'h2.druid_assigned' + embargo_lifted: 'h2.embargo_lifted' notifications: first_draft_reminder: diff --git a/db/seeds.rb b/db/seeds.rb index 9ae6be35c..8b10156fc 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -9,4 +9,4 @@ # Character.create(name: 'Luke', movie: movies.first) # The SDR user is used in Events performed by SDR. -User.create!(name: 'SDR', email: 'sdr@stanford.edu') +User.find_or_create_by!(name: 'SDR', email: 'sdr@stanford.edu') diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 000000000..6942b3ec9 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,66 @@ +version: '3.6' + +services: + app: + build: &build + context: . + dockerfile: docker/app-prod/Dockerfile + args: + - GID=${H2_GID-1000} + - UID=${H2_UID-1000} + - SECRET_KEY_BASE + environment: &environment + - DATABASE_NAME=h2 + - DATABASE_USERNAME + - DATABASE_PASSWORD + - DATABASE_HOSTNAME=host.docker.internal + # for ActionCable + - REDIS_URL=redis://host.docker.internal:6379/ + - SMTP_HOST=host.docker.internal + - HOST + - HONEYBADGER_API_KEY + - SETTINGS__RABBITMQ__PASSWORD + - SETTINGS__RABBITMQ__QUEUES__DEPOSIT_COMPLETE + - SETTINGS__RABBITMQ__QUEUES__DRUID_ASSIGNED + - SETTINGS__RABBITMQ__QUEUES__EMBARGO_LIFTED + - SETTINGS__SDR_API__PASSWORD + - SETTINGS__PRESERVATION_CATALOG__TOKEN + volumes: &volumes + - /opt/app/h2/happy-heron/shared/log:/app/log + - /data/h2-files:/data/h2-files + ports: + - 3000:3000 + extra_hosts: &extra-hosts + - host.docker.internal:host-gateway + profiles: + - web + # restart: on-failure + sneakers: + build: *build + environment: *environment + volumes: *volumes + extra_hosts: *extra-hosts + profiles: + - web + command: bin/bundle exec rake sneakers:run 2>&1 | tee -a log/sneakers.log + # restart: on-failure + sidekiq: + build: *build + environment: *environment + volumes: *volumes + extra_hosts: *extra-hosts + command: bin/bundle exec sidekiq 2>&1 | tee -a log/sidekiq.log + profiles: + - worker + deploy: + replicas: ${SIDEKIQ_COUNT-1} + # restart: on-failure + cron: + build: *build + environment: *environment + volumes: *volumes + extra_hosts: *extra-hosts + profiles: + - cron + command: supercronic /app/config/crontab 2>&1 | tee -a log/cron.log + # restart: on-failure diff --git a/docker/app-prod/Dockerfile b/docker/app-prod/Dockerfile new file mode 100644 index 000000000..b14ede1f3 --- /dev/null +++ b/docker/app-prod/Dockerfile @@ -0,0 +1,53 @@ +FROM ruby:3.1-alpine + +# curl is to install supercronic +RUN apk add --update --no-cache \ + build-base \ + postgresql-dev \ + tzdata \ + yarn \ + git \ + curl + +ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.2.1/supercronic-linux-amd64 +ENV SUPERCRONIC=supercronic-linux-amd64 +ENV SUPERCRONIC_SHA1SUM=d7f4c0886eb85249ad05ed592902fa6865bb9d70 + +# Cron replacement, since cron must be run as root +RUN curl -fsSLO "$SUPERCRONIC_URL" \ + && echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - \ + && chmod +x "$SUPERCRONIC" \ + && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \ + && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic + +# Get bundler 2.0 +RUN gem install bundler + +ARG UID=1000 +ARG GID=1000 +ARG SECRET_KEY_BASE +ENV SECRET_KEY_BASE=${SECRET_KEY_BASE} + +RUN addgroup -g $GID h2 +RUN adduser -G h2 -u $UID -D h2 + +WORKDIR /app + +COPY Gemfile Gemfile.lock package.json yarn.lock ./ + +ENV RAILS_ENV=production +ENV RAILS_LOG_TO_STDOUT=true +ENV BUNDLE_WITHOUT="test:development:deployment" +ENV NODE_ENV=production + +RUN bundle install +RUN yarn install + +COPY --chown=h2:h2 . . +USER h2:h2 + +RUN bin/rails assets:precompile +# Write out the crontab file +RUN sh -c 'bundle exec whenever . | tee -a config/crontab' + +CMD bin/puma -C config/puma.rb config.ru 2>&1 | tee -a log/production.log diff --git a/lib/tasks/rabbitmq.rake b/lib/tasks/rabbitmq.rake index 36e4628e3..c70d959da 100644 --- a/lib/tasks/rabbitmq.rake +++ b/lib/tasks/rabbitmq.rake @@ -14,15 +14,15 @@ namespace :rabbitmq do # connect topic to the queue exchange = channel.topic('sdr.workflow') - queue = channel.queue('h2.deposit_complete', durable: true) + queue = channel.queue(Settings.rabbitmq.queues.deposit_complete, durable: true) queue.bind(exchange, routing_key: 'end-accession.completed') exchange = channel.topic('sdr.objects.created') - queue = channel.queue('h2.druid_assigned', durable: true) + queue = channel.queue(Settings.rabbitmq.queues.druid_assigned, durable: true) queue.bind(exchange, routing_key: Settings.h2.project_tag) exchange = channel.topic('sdr.objects.embargo_lifted') - queue = channel.queue('h2.embargo_lifted', durable: true) + queue = channel.queue(Settings.rabbitmq.queues.embargo_lifted, durable: true) queue.bind(exchange, routing_key: Settings.h2.project_tag) conn.close end