Adding NBP push connector

Gemfile

@@ -36,6 +36,7 @@ gem 'sassc-rails'
 gem 'shakapacker', '8.0.0'
 gem 'simple_form'
 gem 'slim-rails'
+gem 'solid_queue'
 gem 'sprockets-rails'
 gem 'terser'
 gem 'turbolinks'

Gemfile.lock

@@ -153,6 +153,8 @@ GEM
     docile (1.4.0)
     drb (2.2.1)
     erubi (1.12.0)
+    et-orbi (1.2.11)
+      tzinfo
     execjs (2.9.1)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
@@ -164,6 +166,9 @@ GEM
     faraday-net_http (3.1.0)
       net-http
     ffi (1.16.3)
+    fugit (1.11.0)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     glob (0.4.1)
     globalid (1.2.1)
       activesupport (>= 6.1)
@@ -313,6 +318,7 @@ GEM
       rspec-expectations (~> 3.12)
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
+    raabro (1.4.0)
     racc (1.8.0)
     rack (3.0.11)
     rack-mini-profiler (3.3.1)
@@ -500,6 +506,12 @@ GEM
     slim_lint (0.27.0)
       rubocop (>= 1.0, < 2.0)
       slim (>= 3.0, < 6.0)
+    solid_queue (0.3.2)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
     sorted_set (1.0.3)
       rbtree
       set (~> 1.0)
@@ -627,6 +639,7 @@ DEPENDENCIES
   simplecov
   slim-rails
   slim_lint
+  solid_queue
   sprockets-rails
   stackprof
   terser

app/jobs/application_job.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+end

app/jobs/nbp_delete_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class NbpDeleteJob < ApplicationJob
+  def perform(task_uuid)
+    Nbp::PushConnector.instance.delete_task!(task_uuid)
+
+    Rails.logger.debug { "Task ##{task_uuid} deleted from NBP" }
+  end
+end

app/jobs/nbp_push_all_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class NbpPushAllJob < ApplicationJob
+  def perform
+    Task.find_each(batch_size: 50) do |task|
+      NbpPushJob.perform_later task
+    end
+  end
+end

app/jobs/nbp_push_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class NbpPushJob < ApplicationJob
+  def perform(task)
+    builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) }
+
+    Nbp::PushConnector.instance.push_lom!(builder.to_xml)
+
+    Rails.logger.debug { "Task ##{task.id} \"#{task}\" pushed to NBP" }
+  end
+end

app/models/task.rb

@@ -5,11 +5,14 @@
 class Task < ApplicationRecord
   acts_as_taggable_on :state
 
+  before_validation :lowercase_language
+  after_destroy_commit :remove_metadata_from_nbp, if: -> { Nbp::PushConnector.enabled? }
+  after_save_commit :publish_metadata_to_nbp, if: -> { Nbp::PushConnector.enabled? }
+
   validates :title, presence: true
 
   validates :uuid, uniqueness: true
 
-  before_validation :lowercase_language
   validates :language, format: {with: /\A[a-zA-Z]{1,8}(-[a-zA-Z0-9]{1,8})*\z/, message: :not_de_or_us}
   validate :primary_language_tag_in_iso639?
 
@@ -102,6 +105,14 @@ def self.ransackable_associations(_auth_object = nil)
     %w[labels]
   end
 
+  def publish_metadata_to_nbp
+    NbpPushJob.perform_later self
+  end
+
+  def remove_metadata_from_nbp
+    NbpDeleteJob.perform_later uuid
+  end
+
   # This method creates a duplicate while leaving permissions and ownership unchanged
   def duplicate
     dup.tap do |task|

config/environments/development.rb

@@ -91,4 +91,8 @@
 
   # Raise error when a before_action's only/except options reference missing actions
   config.action_controller.raise_on_missing_callback_actions = true
+
+  # Use a real queuing backend for Active Job (and separate queues per environment).
+  config.active_job.queue_adapter = :solid_queue
+  config.active_job.queue_name_prefix = 'codeharbor_development'
 end

config/environments/production.rb

@@ -77,8 +77,8 @@
   # config.cache_store = :mem_cache_store
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter     = :resque
-  # config.active_job.queue_name_prefix = "codeharbor_production"
+  config.active_job.queue_adapter = :solid_queue
+  config.active_job.queue_name_prefix = 'codeharbor_production'
 
   config.action_mailer.perform_caching = false
 

config/environments/test.rb

@@ -66,4 +66,8 @@
 
   # Raise error when a before_action's only/except options reference missing actions
   config.action_controller.raise_on_missing_callback_actions = true
+
+  # Use a real queuing backend for Active Job (and separate queues per environment).
+  config.active_job.queue_adapter = :solid_queue
+  config.active_job.queue_name_prefix = 'codeharbor_test'
 end

config/puma.rb

@@ -82,3 +82,6 @@
 # - Phased Restarts are only supported in cluster mode with multiple workers (i.e., not in development).
 # - The Puma binary won't be upgraded on phased restarts, but since we have the unattended-upgrades, this is not a major issue.
 # - See https://github.com/casperisfine/puma/blob/master/docs/restart.md.
+
+# Run Solid Queue's supervisor
+plugin :solid_queue

config/solid_queue.yml

@@ -0,0 +1,18 @@
+# default: &default
+#   dispatchers:
+#     - polling_interval: 1
+#       batch_size: 500
+#   workers:
+#     - queues: "*"
+#       threads: 3
+#       processes: 1
+#       polling_interval: 0.1
+#
+# development:
+#  <<: *default
+#
+# test:
+#  <<: *default
+#
+# production:
+#  <<: *default

db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+# This migration comes from solid_queue (originally 20231211200639)
+class CreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index %i[queue_name finished_at], name: 'index_solid_queue_jobs_for_filtering'
+      t.index %i[scheduled_at finished_at], name: 'index_solid_queue_jobs_for_alerting'
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index %i[scheduled_at priority job_id], name: 'index_solid_queue_dispatch_all'
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index %i[priority job_id], name: 'index_solid_queue_poll_all'
+      t.index %i[queue_name priority job_id], name: 'index_solid_queue_poll_by_queue'
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index %i[process_id job_id]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index %i[expires_at concurrency_key], name: 'index_solid_queue_blocked_executions_for_maintenance'
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: {unique: true}
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: {unique: true}
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index %i[key value], name: 'index_solid_queue_semaphores_on_key_and_value'
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end

db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# This migration comes from solid_queue (originally 20240110143450)
+class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1]
+  def change
+    add_index :solid_queue_blocked_executions, %i[concurrency_key priority job_id], name: 'index_solid_queue_blocked_executions_for_release'
+  end
+end

db/migrate/20240609104041_create_recurring_executions.solid_queue.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# This migration comes from solid_queue (originally 20240218110712)
+class CreateRecurringExecutions < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_recurring_executions do |t|
+      t.references :job, index: {unique: true}, null: false
+      t.string :task_key, null: false
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false
+
+      t.index %i[task_key run_at], unique: true
+    end
+
+    add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end