diff --git a/Gemfile b/Gemfile index 56c42e027..eac3f8645 100644 --- a/Gemfile +++ b/Gemfile @@ -36,6 +36,7 @@ gem 'sassc-rails' gem 'shakapacker', '8.0.0' gem 'simple_form' gem 'slim-rails' +gem 'solid_queue' gem 'sprockets-rails' gem 'terser' gem 'turbolinks' diff --git a/Gemfile.lock b/Gemfile.lock index c4cbc75b1..5c8b0c731 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -153,6 +153,8 @@ GEM docile (1.4.0) drb (2.2.1) erubi (1.13.0) + et-orbi (1.2.11) + tzinfo execjs (2.9.1) factory_bot (6.4.6) activesupport (>= 5.0.0) @@ -164,6 +166,9 @@ GEM faraday-net_http (3.1.0) net-http ffi (1.17.0) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) glob (0.4.1) globalid (1.2.1) activesupport (>= 6.1) @@ -314,6 +319,7 @@ GEM rspec-expectations (~> 3.12) rspec-mocks (~> 3.12) rspec-support (~> 3.12) + raabro (1.4.0) racc (1.8.0) rack (3.1.3) rack-mini-profiler (3.3.1) @@ -500,6 +506,12 @@ GEM slim_lint (0.27.0) rubocop (>= 1.0, < 2.0) slim (>= 3.0, < 6.0) + solid_queue (0.3.3) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) sorted_set (1.0.3) rbtree set (~> 1.0) @@ -629,6 +641,7 @@ DEPENDENCIES simplecov slim-rails slim_lint + solid_queue sprockets-rails stackprof terser diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb new file mode 100644 index 000000000..27cfbfd7a --- /dev/null +++ b/app/jobs/application_job.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ApplicationJob < ActiveJob::Base + include ActiveRecordLogging +end diff --git a/app/jobs/concerns/active_record_logging.rb b/app/jobs/concerns/active_record_logging.rb new file mode 100644 index 000000000..5a2c38c4a --- /dev/null +++ b/app/jobs/concerns/active_record_logging.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# This module is used to log ActiveRecord queries performed in jobs. +module ActiveRecordLogging + extend ActiveSupport::Concern + + included do + around_perform do |_job, block| + # With our current Solid Queue setup, there is a difference between both logger: + # - *ActiveRecord::Base.logger*: This logger is used for SQL queries and, normally, writes to the log file only. + # - *Rails.logger*: The regular logger, which writes to the log file and the console. + # For the duration of the job, we want to write the SQL queries to the Rails logger, so they show up in the console. + # See config/solid_queue_logging.rb for more information. + previous_logger = ActiveRecord::Base.logger + ActiveRecord::Base.logger = Rails.logger + block.call + ActiveRecord::Base.logger = previous_logger + end + end +end diff --git a/app/jobs/nbp_delete_job.rb b/app/jobs/nbp_delete_job.rb new file mode 100644 index 000000000..b4ace0d3c --- /dev/null +++ b/app/jobs/nbp_delete_job.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class NbpDeleteJob < ApplicationJob + retry_on Faraday::Error, wait: :polynomially_longer + + def perform(task_uuid) + Nbp::PushConnector.instance.delete_task!(task_uuid) + + Rails.logger.debug { "Task with UUID #{task_uuid} deleted from NBP" } + end +end diff --git a/app/jobs/nbp_push_all_job.rb b/app/jobs/nbp_push_all_job.rb new file mode 100644 index 000000000..56ac90184 --- /dev/null +++ b/app/jobs/nbp_push_all_job.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class NbpPushAllJob < ApplicationJob + def perform + Task.find_each(batch_size: 50) do |task| + NbpPushJob.perform_later task + end + end +end diff --git a/app/jobs/nbp_push_job.rb b/app/jobs/nbp_push_job.rb new file mode 100644 index 000000000..c56ff7e3a --- /dev/null +++ b/app/jobs/nbp_push_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class NbpPushJob < ApplicationJob + retry_on Faraday::Error, wait: :polynomially_longer, attempts: 5 + + def perform(task) + builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) } + + Nbp::PushConnector.instance.push_lom!(builder.to_xml) + + Rails.logger.debug { "Task ##{task.id} \"#{task}\" pushed to NBP" } + end +end diff --git a/app/models/task.rb b/app/models/task.rb index 0872dfc74..2b1292ff5 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -5,11 +5,14 @@ class Task < ApplicationRecord acts_as_taggable_on :state + before_validation :lowercase_language + after_destroy_commit :remove_metadata_from_nbp, if: -> { Nbp::PushConnector.enabled? } + after_save_commit :sync_metadata_with_nbp, if: -> { Nbp::PushConnector.enabled? } + validates :title, presence: true validates :uuid, uniqueness: true - before_validation :lowercase_language validates :language, format: {with: /\A[a-zA-Z]{1,8}(-[a-zA-Z0-9]{1,8})*\z/, message: :not_de_or_us} validate :primary_language_tag_in_iso639? @@ -102,6 +105,22 @@ def self.ransackable_associations(_auth_object = nil) %w[labels] end + def sync_metadata_with_nbp + if access_level_public? + publish_metadata_to_nbp + elsif saved_change_to_access_level? + remove_metadata_from_nbp + end + end + + def publish_metadata_to_nbp + NbpPushJob.perform_later self + end + + def remove_metadata_from_nbp + NbpDeleteJob.perform_later uuid + end + # This method creates a duplicate while leaving permissions and ownership unchanged def duplicate dup.tap do |task| diff --git a/config/application.rb b/config/application.rb index 4a01363fb..bcbeffd74 100644 --- a/config/application.rb +++ b/config/application.rb @@ -50,5 +50,8 @@ class Application < Rails::Application # Fix invalid Content-Type header for incoming requests made by edu-sharing. config.middleware.insert_before 0, Middleware::EduSharingContentType + + # Configure some defaults for the Solid Queue Supervisor + require_relative 'solid_queue_defaults' end end diff --git a/config/environments/development.rb b/config/environments/development.rb index 80d25f19b..cda0c98b1 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -44,6 +44,10 @@ # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false + # Use a real queuing backend for Active Job (and separate queues per environment). + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_development' + config.action_mailer.perform_caching = false # Print deprecation notices to the Rails logger. diff --git a/config/environments/production.rb b/config/environments/production.rb index c167344d6..a5249631a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -77,8 +77,8 @@ # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "codeharbor_production" + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_production' config.action_mailer.perform_caching = false diff --git a/config/environments/test.rb b/config/environments/test.rb index 70f731662..79c2510d0 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -39,6 +39,10 @@ # Store uploaded files on the local file system in a temporary directory. config.active_storage.service = :test + # Use a real queuing backend for Active Job (and separate queues per environment). + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_test' + config.action_mailer.perform_caching = false # Tell Action Mailer not to deliver emails to the real world. diff --git a/config/schedule.rb b/config/schedule.rb index 072c66669..d0a10594a 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -28,4 +28,5 @@ every 1.day, at: '3:00 am' do rake 'import_cache_files:cleanup' + SolidQueue::Job.clear_finished_in_batches end diff --git a/config/settings/test.yml b/config/settings/test.yml index e58f8e258..ebdcd1cd8 100644 --- a/config/settings/test.yml +++ b/config/settings/test.yml @@ -13,3 +13,14 @@ omniauth: private_key: ~ oai_pmh: admin_mail: admin@example.org +nbp: + push_connector: + enable: true + client_id: testing_client_id + client_secret: testing_client_secret + token_path: 'https://test.provider/token' + api_host: 'https://test.api.host' + source: + organization: test_organization + name: CodeHarbor + slug: CoHaP2 diff --git a/config/solid_queue.yml b/config/solid_queue.yml new file mode 100644 index 000000000..2d8d922f1 --- /dev/null +++ b/config/solid_queue.yml @@ -0,0 +1,18 @@ +# default: &default +# dispatchers: +# - polling_interval: 1 +# batch_size: 500 +# workers: +# - queues: "*" +# threads: 3 +# processes: 1 +# polling_interval: 0.1 +# +# development: +# <<: *default +# +# test: +# <<: *default +# +# production: +# <<: *default diff --git a/config/solid_queue_defaults.rb b/config/solid_queue_defaults.rb new file mode 100644 index 000000000..d30aadd5f --- /dev/null +++ b/config/solid_queue_defaults.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# This file must be loaded before other initializers due to the logging configuration. +Rails.application.configure do + # On shutdown, jobs Solid Queue will wait the specified timeout before forcefully shutting down. + # Any job not finished by then will be picked up again after a restart. + config.solid_queue.shutdown_timeout = 10.seconds + # Remove *successful* jobs from the database after 30 days + config.solid_queue.clear_finished_jobs_after = 30.days + config.solid_queue.supervisor_pidfile = Rails.root.join('tmp/pids/solid_queue_supervisor.pid') + + # For Solid Queue, we want to hide regular SQL queries from the console, but still log them to a separate file. + # For the normal webserver, this dedicated setup is neither needed nor desired. + next unless Rake.application.top_level_tasks.to_s.include?('solid_queue:') + + # Specify that all logs should be written to the specified log file + file_name = "#{Rails.env}.solid_queue.log" + config.paths.add 'log', with: "log/#{file_name}" + + # Send all logs regarding SQL queries to the log file. + # This will include all queries performed by Solid Queue including periodic job checks. + log_file = ActiveSupport::Logger.new(Rails.root.join('log', file_name)) + config.active_record.logger = ActiveSupport::BroadcastLogger.new(log_file) + + config.after_initialize do + # Create a new logger that will write to the console + console = ActiveSupport::Logger.new($stdout) + console.level = Rails.logger.level + # Enable this line to have the same log format as Rails.logger + # It will include the job name, the job ID for each line + # console.formatter = Rails.logger.formatter + + ActiveSupport.on_load :solid_queue_record do + # Once SolidQueue is loaded, we can broadcast its logs to the console, too. + # Due to the initialization order, this will effectively start logging once SolidQueue is about to start. + Rails.logger.broadcast_to console + end + end +end diff --git a/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb b/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb new file mode 100644 index 000000000..874f68ba0 --- /dev/null +++ b/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20231211200639) +class CreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + create_table :solid_queue_jobs do |t| + t.string :queue_name, null: false + t.string :class_name, null: false, index: true + t.text :arguments + t.integer :priority, default: 0, null: false + t.string :active_job_id, index: true + t.datetime :scheduled_at + t.datetime :finished_at, index: true + t.string :concurrency_key + + t.timestamps + + t.index %i[queue_name finished_at], name: 'index_solid_queue_jobs_for_filtering' + t.index %i[scheduled_at finished_at], name: 'index_solid_queue_jobs_for_alerting' + end + + create_table :solid_queue_scheduled_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.datetime :scheduled_at, null: false + + t.datetime :created_at, null: false + + t.index %i[scheduled_at priority job_id], name: 'index_solid_queue_dispatch_all' + end + + create_table :solid_queue_ready_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + + t.datetime :created_at, null: false + + t.index %i[priority job_id], name: 'index_solid_queue_poll_all' + t.index %i[queue_name priority job_id], name: 'index_solid_queue_poll_by_queue' + end + + create_table :solid_queue_claimed_executions do |t| + t.references :job, index: {unique: true}, null: false + t.bigint :process_id + t.datetime :created_at, null: false + + t.index %i[process_id job_id] + end + + create_table :solid_queue_blocked_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.string :concurrency_key, null: false + t.datetime :expires_at, null: false + + t.datetime :created_at, null: false + + t.index %i[expires_at concurrency_key], name: 'index_solid_queue_blocked_executions_for_maintenance' + end + + create_table :solid_queue_failed_executions do |t| + t.references :job, index: {unique: true}, null: false + t.text :error + t.datetime :created_at, null: false + end + + create_table :solid_queue_pauses do |t| + t.string :queue_name, null: false, index: {unique: true} + t.datetime :created_at, null: false + end + + create_table :solid_queue_processes do |t| + t.string :kind, null: false + t.datetime :last_heartbeat_at, null: false, index: true + t.bigint :supervisor_id, index: true + + t.integer :pid, null: false + t.string :hostname + t.text :metadata + + t.datetime :created_at, null: false + end + + create_table :solid_queue_semaphores do |t| + t.string :key, null: false, index: {unique: true} + t.integer :value, default: 1, null: false + t.datetime :expires_at, null: false, index: true + + t.timestamps + + t.index %i[key value], name: 'index_solid_queue_semaphores_on_key_and_value' + end + + add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb b/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb new file mode 100644 index 000000000..491e2181d --- /dev/null +++ b/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20240110143450) +class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1] + def change + add_index :solid_queue_blocked_executions, %i[concurrency_key priority job_id], name: 'index_solid_queue_blocked_executions_for_release' + end +end diff --git a/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb b/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb new file mode 100644 index 000000000..69847c125 --- /dev/null +++ b/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20240218110712) +class CreateRecurringExecutions < ActiveRecord::Migration[7.1] + def change + create_table :solid_queue_recurring_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :task_key, null: false + t.datetime :run_at, null: false + t.datetime :created_at, null: false + + t.index %i[task_key run_at], unique: true + end + + add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index d3f99fb0f..d5afcfd38 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_05_31_160738) do +ActiveRecord::Schema[7.1].define(version: 2024_06_09_104041) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -199,6 +199,109 @@ t.index ["user_id"], name: "index_reports_on_user_id" end + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + create_table "taggings", id: :serial, force: :cascade do |t| t.integer "tag_id" t.string "taggable_type" @@ -350,6 +453,12 @@ add_foreign_key "ratings", "users" add_foreign_key "reports", "tasks" add_foreign_key "reports", "users" + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade add_foreign_key "task_labels", "tasks" add_foreign_key "tasks", "licenses" add_foreign_key "tests", "tasks" diff --git a/docs/LOCAL_SETUP.md b/docs/LOCAL_SETUP.md index 14d419740..a894ea281 100644 --- a/docs/LOCAL_SETUP.md +++ b/docs/LOCAL_SETUP.md @@ -181,7 +181,9 @@ rake db:setup ### Start CodeHarbor -For the development environment, two server processes are required: the Rails server for the main application and a Webpack server providing JavaScript and CSS assets. +For the development environment, three server processes are required: +the Rails server for the main application, a Webpack server providing JavaScript and CSS assets +and the Solid Queue supervisor to process background jobs. 1. Webpack dev server: @@ -201,6 +203,15 @@ This will launch a dedicated server on port 3045 (default setting) and allow inc This will launch the CodeHarbor web application server on port 7500 (default setting) and allow incoming connections from your browser. +3. Solid Queue supervisor (optional): + + ```shell + bundle exec rake solid_queue:start + ``` + +Solid Queue is currently only used to synchronize task metadata with the "NBP Datenraum". +The synchronization is disabled by default. Unless you enable it, you can skip starting the supervisor. + **Check with:** Open your web browser at diff --git a/lib/nbp/push_connector.rb b/lib/nbp/push_connector.rb new file mode 100644 index 000000000..0151c1f13 --- /dev/null +++ b/lib/nbp/push_connector.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Nbp + class SettingsError < StandardError; end + class ConnectorError < StandardError; end + + class PushConnector + include Singleton + + def initialize + super + create_source! unless source_exists? + end + + def self.enabled? + Settings.nbp&.push_connector&.enable || false + end + + def push_lom!(xml) + response = api_conn.put("/push-connector/api/lom-v2/#{settings.source.slug}") do |req| + req.body = {metadata: xml}.to_json + end + raise_connector_error('Could not push task LOM', response) unless success_status?(response.status) + end + + def delete_task!(task_uuid) + response = api_conn.delete("/push-connector/api/course/#{settings.source.slug}/#{task_uuid}") + raise_connector_error('Could delete task', response) unless success_status?(response.status) || response.status == 404 + end + + def source_exists? + response = api_conn.get("/datenraum/api/core/sources/slug/#{settings.source.slug}") + if response.status == 200 + true + elsif response.status == 404 + false + else + raise_connector_error('Could not determine if source exists', response) + end + end + + def create_source! + response = api_conn.post('/datenraum/api/core/sources') do |req| + req.body = settings.source.to_json + end + raise_connector_error('Failed to create source', response) unless success_status?(response.status) + end + + def token + if @token.present? && @token_expiration > 10.seconds.from_now + @token + else + update_token + end + end + + def update_token + response = Faraday.post(settings.token_path, auth) + result = JSON.parse(response.body) + + if success_status?(response.status) + @token_expiration = Time.zone.now + result['expires_in'] + @token = result['access_token'] + else + raise_connector_error('Failed to get fresh access token', response) + end + end + + def auth + { + grant_type: 'client_credentials', + client_id: settings.client_id, + client_secret: settings.client_secret, + } + end + + def api_conn + Faraday.new(url: settings.api_host, headers:) + end + + def settings + return @connector_settings if @connector_settings + + check_settings! + @connector_settings = Settings.nbp&.push_connector + end + + def check_settings! # rubocop:disable Metrics/AbcSize + settings_hash = Settings.nbp&.push_connector.to_h + + if PushConnector.enabled? + missing_keys = %i[client_id client_secret token_path api_host source] - settings_hash.keys + raise SettingsError.new("Nbp::PushConnector is missing some settings: #{missing_keys}") if missing_keys.any? + + missing_source_keys = %i[organization name slug] - settings_hash[:source].keys + raise SettingsError.new("Nbp::PushConnector source is missing some settings: #{missing_source_keys}") if missing_source_keys.any? + else + raise SettingsError.new('Nbp::PushConnector is disabled but got accessed') + end + end + + def success_status?(status_code) + (200..299).cover?(status_code) + end + + def raise_connector_error(message, faraday_response) + raise ConnectorError.new("#{message} (code #{faraday_response.status}). Response was: '#{faraday_response.body}'") + end + + def headers + {authorization: "Bearer #{token}", 'content-type': 'application/json', accept: 'application/json'} + end + end +end diff --git a/lib/tasks/nbp_push_all_tasks.rake b/lib/tasks/nbp_push_all_tasks.rake new file mode 100644 index 000000000..19e011b16 --- /dev/null +++ b/lib/tasks/nbp_push_all_tasks.rake @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +namespace :nbp do + desc 'Pushes all tasks to the NBP for an initial sync' + task push_all: :environment do + NbpPushAllJob.perform_later + end +end diff --git a/spec/lib/nbp/push_connector_spec.rb b/spec/lib/nbp/push_connector_spec.rb new file mode 100644 index 000000000..beca53fec --- /dev/null +++ b/spec/lib/nbp/push_connector_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Nbp::PushConnector do + let(:api_host) { Settings.nbp.push_connector.api_host } + let(:source_slug) { Settings.nbp.push_connector.source.slug } + + let(:task) { create(:task) } + let(:task_xml) { (Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) }).to_xml } + + let(:connector) { Class.new(described_class).instance } # https://stackoverflow.com/a/23901644 + + before do + stub_request(:post, Settings.nbp.push_connector.token_path).to_return_json(body: {token: 'sometoken', expires_in: 600}) + stub_request(:post, "#{api_host}/datenraum/api/core/sources") + stub_request(:put, "#{api_host}/push-connector/api/lom-v2/#{source_slug}") + stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 404) + end + + describe 'push_lom!' do + subject(:push_lom!) { connector.push_lom!(task_xml) } + + context 'when no source exists' do + it 'creates a source' do + push_lom! + expect(WebMock).to have_requested(:post, "#{api_host}/datenraum/api/core/sources") + end + end + + context 'when a source exists' do + before { stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 200) } + + it 'does not create a source' do + push_lom! + expect(WebMock).not_to have_requested(:post, "#{api_host}/datenraum/api/core/sources") + end + end + + context 'without any errors' do + it 'pushes the metadata' do + push_lom! + expect(WebMock).to have_requested(:put, "#{api_host}/push-connector/api/lom-v2/#{source_slug}") + end + end + end +end diff --git a/spec/models/task_spec.rb b/spec/models/task_spec.rb index b3e5ea0f4..e413dd24d 100644 --- a/spec/models/task_spec.rb +++ b/spec/models/task_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' RSpec.describe Task do + include ActiveJob::TestHelper + describe '#valid?' do it { is_expected.to validate_presence_of(:title) } it { is_expected.to validate_uniqueness_of(:uuid).case_insensitive } @@ -186,5 +188,51 @@ destroy expect(collection.reload.tasks).to be_empty end + + it 'enqueues an NbpDeleteJob' do + expect { destroy }.to have_enqueued_job(NbpDeleteJob).with(task.uuid) + end + end + + describe '#update' do + subject(:update) { task.update(new_attributes) } + + let(:task) { create(:task, access_level:) } + + context 'when updating a public task' do + let(:access_level) { :public } + let(:new_attributes) { {title: 'some new title'} } + + it 'enqueues an NbpPushJob' do + expect { update }.to have_enqueued_job(NbpPushJob).with(task) + end + end + + context 'when updating a private task' do + let(:access_level) { :private } + let(:new_attributes) { {title: 'some new title'} } + + it 'does not enqueue an NbpPushJob' do + expect { update }.not_to have_enqueued_job(NbpPushJob) + end + end + + context 'when changing the access level from private to public' do + let(:access_level) { :private } + let(:new_attributes) { {access_level: :public} } + + it 'enqueues an NbpPushJob' do + expect { update }.to have_enqueued_job(NbpPushJob).with(task) + end + end + + context 'when changing the access level from public to private' do + let(:access_level) { :public } + let(:new_attributes) { {access_level: :private} } + + it 'enqueues an NbpDeleteJob' do + expect { update }.to have_enqueued_job(NbpDeleteJob).with(task.uuid) + end + end end end