diff --git a/Gemfile b/Gemfile index 838308992..db9594753 100644 --- a/Gemfile +++ b/Gemfile @@ -19,6 +19,7 @@ gem 'json_schemer' gem 'js-routes' gem 'kramdown' gem 'kramdown-parser-gfm' +gem 'loofah' gem 'nested_form_fields' gem 'net-http' gem 'net-imap', require: false @@ -30,6 +31,7 @@ gem 'proformaxml', '~> 1.4.0' gem 'puma' gem 'rails', '~> 7.1.3' gem 'rails_admin' +gem 'rails-html-sanitizer' gem 'rails-i18n' gem 'ransack' gem 'rqrcode' @@ -39,6 +41,7 @@ gem 'sassc-rails' gem 'shakapacker', '8.0.1' gem 'simple_form' gem 'slim-rails' +gem 'solid_queue' gem 'sprockets-rails' gem 'terser' gem 'turbolinks' diff --git a/Gemfile.lock b/Gemfile.lock index acb59c544..f8f49477c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -154,6 +154,8 @@ GEM docile (1.4.0) drb (2.2.1) erubi (1.13.0) + et-orbi (1.2.11) + tzinfo event_stream_parser (1.0.0) execjs (2.9.1) factory_bot (6.4.6) @@ -169,6 +171,9 @@ GEM faraday-net_http (3.1.0) net-http ffi (1.17.0) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) glob (0.4.1) globalid (1.2.1) activesupport (>= 6.1) @@ -326,6 +331,7 @@ GEM rspec-expectations (~> 3.12) rspec-mocks (~> 3.12) rspec-support (~> 3.12) + raabro (1.4.0) racc (1.8.0) rack (3.1.7) rack-mini-profiler (3.3.1) @@ -521,6 +527,12 @@ GEM slim_lint (0.27.0) rubocop (>= 1.0, < 2.0) slim (>= 3.0, < 6.0) + solid_queue (0.3.3) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) sorted_set (1.0.3) rbtree set (~> 1.0) @@ -607,6 +619,7 @@ DEPENDENCIES kramdown-parser-gfm letter_opener listen + loofah mnemosyne-ruby nested_form_fields net-http @@ -627,6 +640,7 @@ DEPENDENCIES rack-mini-profiler rails (~> 7.1.3) rails-controller-testing + rails-html-sanitizer rails-i18n rails_admin ransack @@ -653,6 +667,7 @@ DEPENDENCIES simplecov slim-rails slim_lint + solid_queue sprockets-rails stackprof terser diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb new file mode 100644 index 000000000..f378bca4a --- /dev/null +++ b/app/jobs/application_job.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ApplicationJob < ActiveJob::Base + include ActiveRecordLogging + + # Automatically retry jobs that encountered a deadlock + retry_on ActiveRecord::Deadlocked + + # Most jobs are safe to ignore if the underlying records are no longer available + discard_on ActiveJob::DeserializationError +end diff --git a/app/jobs/concerns/active_record_logging.rb b/app/jobs/concerns/active_record_logging.rb new file mode 100644 index 000000000..5a2c38c4a --- /dev/null +++ b/app/jobs/concerns/active_record_logging.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# This module is used to log ActiveRecord queries performed in jobs. +module ActiveRecordLogging + extend ActiveSupport::Concern + + included do + around_perform do |_job, block| + # With our current Solid Queue setup, there is a difference between both logger: + # - *ActiveRecord::Base.logger*: This logger is used for SQL queries and, normally, writes to the log file only. + # - *Rails.logger*: The regular logger, which writes to the log file and the console. + # For the duration of the job, we want to write the SQL queries to the Rails logger, so they show up in the console. + # See config/solid_queue_logging.rb for more information. + previous_logger = ActiveRecord::Base.logger + ActiveRecord::Base.logger = Rails.logger + block.call + ActiveRecord::Base.logger = previous_logger + end + end +end diff --git a/app/jobs/nbp_sync_all_job.rb b/app/jobs/nbp_sync_all_job.rb new file mode 100644 index 000000000..c560e4b45 --- /dev/null +++ b/app/jobs/nbp_sync_all_job.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class NbpSyncAllJob < ApplicationJob + def perform + uuids = Set[] + + # First, add all uploaded UUIDs. + # This allows us to delete the ones that are still present remote but no longer in the local database. + Nbp::PushConnector.instance.process_uploaded_task_uuids do |uuid| + uuids.add(uuid) + end + + # Then, add all local UUIDs. + # This allows us to upload tasks missing remote (and remove private tasks not yet removed). + Task.select(:id, :uuid).find_each {|task| uuids.add(task.uuid) } + + # Finally, schedule a full sync for each UUID identified. + uuids.each do |uuid| + NbpSyncJob.perform_later uuid + end + end +end diff --git a/app/jobs/nbp_sync_job.rb b/app/jobs/nbp_sync_job.rb new file mode 100644 index 000000000..b52952002 --- /dev/null +++ b/app/jobs/nbp_sync_job.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class NbpSyncJob < ApplicationJob + retry_on Faraday::Error, Nbp::PushConnector::ServerError, wait: :polynomially_longer, attempts: 5 + + def perform(uuid) + task = Task.find_by(uuid:) + + if task.present? && task.access_level_public? + builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) } + Nbp::PushConnector.instance.push_lom!(builder.to_xml) + Rails.logger.debug { "Task ##{task.id} \"#{task}\" pushed to NBP" } + else + Nbp::PushConnector.instance.delete_task!(uuid) + Rails.logger.debug { "Task with UUID #{uuid} deleted from NBP" } + end + end +end diff --git a/app/models/task.rb b/app/models/task.rb index 0872dfc74..28fe2d2df 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -5,11 +5,13 @@ class Task < ApplicationRecord acts_as_taggable_on :state + before_validation :lowercase_language + after_commit :sync_metadata_with_nbp, if: -> { Nbp::PushConnector.enabled? } + validates :title, presence: true validates :uuid, uniqueness: true - before_validation :lowercase_language validates :language, format: {with: /\A[a-zA-Z]{1,8}(-[a-zA-Z0-9]{1,8})*\z/, message: :not_de_or_us} validate :primary_language_tag_in_iso639? @@ -102,6 +104,13 @@ def self.ransackable_associations(_auth_object = nil) %w[labels] end + def sync_metadata_with_nbp + if access_level_public? || saved_change_to_access_level? + NbpSyncJob.perform_later uuid + NbpSyncJob.perform_later uuid_previously_was if saved_change_to_uuid? && access_level_previously_was == 'public' + end + end + # This method creates a duplicate while leaving permissions and ownership unchanged def duplicate dup.tap do |task| diff --git a/config/application.rb b/config/application.rb index 4a01363fb..bcbeffd74 100644 --- a/config/application.rb +++ b/config/application.rb @@ -50,5 +50,8 @@ class Application < Rails::Application # Fix invalid Content-Type header for incoming requests made by edu-sharing. config.middleware.insert_before 0, Middleware::EduSharingContentType + + # Configure some defaults for the Solid Queue Supervisor + require_relative 'solid_queue_defaults' end end diff --git a/config/environments/development.rb b/config/environments/development.rb index 80d25f19b..cda0c98b1 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -44,6 +44,10 @@ # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false + # Use a real queuing backend for Active Job (and separate queues per environment). + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_development' + config.action_mailer.perform_caching = false # Print deprecation notices to the Rails logger. diff --git a/config/environments/production.rb b/config/environments/production.rb index c167344d6..a5249631a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -77,8 +77,8 @@ # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "codeharbor_production" + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_production' config.action_mailer.perform_caching = false diff --git a/config/environments/test.rb b/config/environments/test.rb index 70f731662..79c2510d0 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -39,6 +39,10 @@ # Store uploaded files on the local file system in a temporary directory. config.active_storage.service = :test + # Use a real queuing backend for Active Job (and separate queues per environment). + config.active_job.queue_adapter = :solid_queue + config.active_job.queue_name_prefix = 'codeharbor_test' + config.action_mailer.perform_caching = false # Tell Action Mailer not to deliver emails to the real world. diff --git a/config/schedule.rb b/config/schedule.rb index 072c66669..9c8afb02d 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -28,4 +28,5 @@ every 1.day, at: '3:00 am' do rake 'import_cache_files:cleanup' + runner 'SolidQueue::Job.clear_finished_in_batches' end diff --git a/config/settings/test.yml b/config/settings/test.yml index 67503445e..89acaaa75 100644 --- a/config/settings/test.yml +++ b/config/settings/test.yml @@ -20,6 +20,17 @@ omniauth: private_key: ~ oai_pmh: admin_mail: admin@example.org +nbp: + push_connector: + enable: true + client_id: testing_client_id + client_secret: testing_client_secret + token_path: 'https://test.provider/token' + api_host: 'https://test.api.host' + source: + organization: test_organization + name: CodeHarbor + slug: CoHaP2 open_ai: access_token: ~ # Add a valid API key from https://platform.openai.com/api-keys model: gpt-3.5-turbo diff --git a/config/solid_queue.yml b/config/solid_queue.yml new file mode 100644 index 000000000..73dd19e0c --- /dev/null +++ b/config/solid_queue.yml @@ -0,0 +1,25 @@ + default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: 1 + polling_interval: 0.1 + + development: + <<: *default + + test: + <<: *default + + production: + <<: *default + dispatchers: + - polling_interval: 1 + batch_size: 500 + recurring_tasks: + nbp_sync_all_job: + class: NbpSyncAllJob + schedule: "0 3 * * *" diff --git a/config/solid_queue_defaults.rb b/config/solid_queue_defaults.rb new file mode 100644 index 000000000..d30aadd5f --- /dev/null +++ b/config/solid_queue_defaults.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# This file must be loaded before other initializers due to the logging configuration. +Rails.application.configure do + # On shutdown, jobs Solid Queue will wait the specified timeout before forcefully shutting down. + # Any job not finished by then will be picked up again after a restart. + config.solid_queue.shutdown_timeout = 10.seconds + # Remove *successful* jobs from the database after 30 days + config.solid_queue.clear_finished_jobs_after = 30.days + config.solid_queue.supervisor_pidfile = Rails.root.join('tmp/pids/solid_queue_supervisor.pid') + + # For Solid Queue, we want to hide regular SQL queries from the console, but still log them to a separate file. + # For the normal webserver, this dedicated setup is neither needed nor desired. + next unless Rake.application.top_level_tasks.to_s.include?('solid_queue:') + + # Specify that all logs should be written to the specified log file + file_name = "#{Rails.env}.solid_queue.log" + config.paths.add 'log', with: "log/#{file_name}" + + # Send all logs regarding SQL queries to the log file. + # This will include all queries performed by Solid Queue including periodic job checks. + log_file = ActiveSupport::Logger.new(Rails.root.join('log', file_name)) + config.active_record.logger = ActiveSupport::BroadcastLogger.new(log_file) + + config.after_initialize do + # Create a new logger that will write to the console + console = ActiveSupport::Logger.new($stdout) + console.level = Rails.logger.level + # Enable this line to have the same log format as Rails.logger + # It will include the job name, the job ID for each line + # console.formatter = Rails.logger.formatter + + ActiveSupport.on_load :solid_queue_record do + # Once SolidQueue is loaded, we can broadcast its logs to the console, too. + # Due to the initialization order, this will effectively start logging once SolidQueue is about to start. + Rails.logger.broadcast_to console + end + end +end diff --git a/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb b/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb new file mode 100644 index 000000000..874f68ba0 --- /dev/null +++ b/db/migrate/20240609104039_create_solid_queue_tables.solid_queue.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20231211200639) +class CreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + create_table :solid_queue_jobs do |t| + t.string :queue_name, null: false + t.string :class_name, null: false, index: true + t.text :arguments + t.integer :priority, default: 0, null: false + t.string :active_job_id, index: true + t.datetime :scheduled_at + t.datetime :finished_at, index: true + t.string :concurrency_key + + t.timestamps + + t.index %i[queue_name finished_at], name: 'index_solid_queue_jobs_for_filtering' + t.index %i[scheduled_at finished_at], name: 'index_solid_queue_jobs_for_alerting' + end + + create_table :solid_queue_scheduled_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.datetime :scheduled_at, null: false + + t.datetime :created_at, null: false + + t.index %i[scheduled_at priority job_id], name: 'index_solid_queue_dispatch_all' + end + + create_table :solid_queue_ready_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + + t.datetime :created_at, null: false + + t.index %i[priority job_id], name: 'index_solid_queue_poll_all' + t.index %i[queue_name priority job_id], name: 'index_solid_queue_poll_by_queue' + end + + create_table :solid_queue_claimed_executions do |t| + t.references :job, index: {unique: true}, null: false + t.bigint :process_id + t.datetime :created_at, null: false + + t.index %i[process_id job_id] + end + + create_table :solid_queue_blocked_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.string :concurrency_key, null: false + t.datetime :expires_at, null: false + + t.datetime :created_at, null: false + + t.index %i[expires_at concurrency_key], name: 'index_solid_queue_blocked_executions_for_maintenance' + end + + create_table :solid_queue_failed_executions do |t| + t.references :job, index: {unique: true}, null: false + t.text :error + t.datetime :created_at, null: false + end + + create_table :solid_queue_pauses do |t| + t.string :queue_name, null: false, index: {unique: true} + t.datetime :created_at, null: false + end + + create_table :solid_queue_processes do |t| + t.string :kind, null: false + t.datetime :last_heartbeat_at, null: false, index: true + t.bigint :supervisor_id, index: true + + t.integer :pid, null: false + t.string :hostname + t.text :metadata + + t.datetime :created_at, null: false + end + + create_table :solid_queue_semaphores do |t| + t.string :key, null: false, index: {unique: true} + t.integer :value, default: 1, null: false + t.datetime :expires_at, null: false, index: true + + t.timestamps + + t.index %i[key value], name: 'index_solid_queue_semaphores_on_key_and_value' + end + + add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb b/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb new file mode 100644 index 000000000..491e2181d --- /dev/null +++ b/db/migrate/20240609104040_add_missing_index_to_blocked_executions.solid_queue.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20240110143450) +class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1] + def change + add_index :solid_queue_blocked_executions, %i[concurrency_key priority job_id], name: 'index_solid_queue_blocked_executions_for_release' + end +end diff --git a/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb b/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb new file mode 100644 index 000000000..69847c125 --- /dev/null +++ b/db/migrate/20240609104041_create_recurring_executions.solid_queue.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20240218110712) +class CreateRecurringExecutions < ActiveRecord::Migration[7.1] + def change + create_table :solid_queue_recurring_executions do |t| + t.references :job, index: {unique: true}, null: false + t.string :task_key, null: false + t.datetime :run_at, null: false + t.datetime :created_at, null: false + + t.index %i[task_key run_at], unique: true + end + + add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index 7c9e8a585..63e7ecf87 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_05_31_161908) do +ActiveRecord::Schema[7.1].define(version: 2024_06_09_104041) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -200,6 +200,109 @@ t.index ["user_id"], name: "index_reports_on_user_id" end + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + create_table "taggings", id: :serial, force: :cascade do |t| t.integer "tag_id" t.string "taggable_type" @@ -351,6 +454,12 @@ add_foreign_key "ratings", "users" add_foreign_key "reports", "tasks" add_foreign_key "reports", "users" + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade add_foreign_key "task_labels", "tasks" add_foreign_key "tasks", "licenses" add_foreign_key "tests", "tasks" diff --git a/docs/LOCAL_SETUP.md b/docs/LOCAL_SETUP.md index 14d419740..5ac404d7d 100644 --- a/docs/LOCAL_SETUP.md +++ b/docs/LOCAL_SETUP.md @@ -181,7 +181,9 @@ rake db:setup ### Start CodeHarbor -For the development environment, two server processes are required: the Rails server for the main application and a Webpack server providing JavaScript and CSS assets. +For the development environment, three server processes are required: +the Rails server for the main application, a Webpack server providing JavaScript and CSS assets +and the Solid Queue supervisor to process background jobs. 1. Webpack dev server: @@ -201,6 +203,12 @@ This will launch a dedicated server on port 3045 (default setting) and allow inc This will launch the CodeHarbor web application server on port 7500 (default setting) and allow incoming connections from your browser. +3. Solid Queue supervisor: + + ```shell + bundle exec rake solid_queue:start + ``` + **Check with:** Open your web browser at diff --git a/lib/nbp/push_connector.rb b/lib/nbp/push_connector.rb new file mode 100644 index 000000000..bfa2c089d --- /dev/null +++ b/lib/nbp/push_connector.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +require 'singleton' +require 'concurrent' + +module Nbp + class PushConnector + include Singleton + + class Error < StandardError; end + class SettingsError < Error; end + class ServerError < Error; end + + def initialize + super + @token = Concurrent::ThreadLocalVar.new + @token_expiration = Concurrent::ThreadLocalVar.new + + create_source! unless source_exists? + end + + def self.enabled? + @enabled = Settings.nbp&.push_connector&.enable || false if @enabled.nil? + @enabled + end + + def push_lom!(xml) + response = api_conn.put("/push-connector/api/lom-v2/#{source_slug}") do |req| + req.body = {metadata: xml}.to_json + end + raise ServerError if response.status == 500 + + raise_connector_error('Could not push task LOM', response) unless response.success? + end + + def delete_task!(task_uuid) + response = api_conn.delete("/push-connector/api/course/#{source_slug}/#{task_uuid}") + raise ServerError if response.status == 500 + + raise_connector_error('Could not delete task', response) unless response.success? || response.status == 404 + end + + def process_uploaded_task_uuids(&) + offset = 0 + until (uuids = get_uploaded_task_uuids(offset)).empty? + offset += uuids.length + uuids.map(&) + end + end + + def source_exists? + response = api_conn.get("/datenraum/api/core/sources/slug/#{source_slug}") + if response.status == 200 + true + elsif response.status == 404 + false + else + raise_connector_error('Could not determine if source exists', response) + end + end + + private + + def get_uploaded_task_uuids(offset, limit = 100) # rubocop:disable Metrics/AbcSize + raise Error('The NBP API does not accept limits over 100') if limit > 100 + + response = api_conn.get('/datenraum/api/core/nodes') do |req| + req.params[:sourceSlug] = source_slug + req.params[:offset] = offset + req.params[:limit] = limit + end + raise ServerError if response.status == 500 + + raise_connector_error('Could query existing tasks', response) unless response.success? + + nodes = JSON.parse(response.body).deep_symbolize_keys.dig(:_embedded, :nodes) + raise_connector_error('Nodes response did not contain nodes list', response) if nodes.nil? + + nodes.pluck(:externalId).compact + end + + def create_source! + response = api_conn.post('/datenraum/api/core/sources') do |req| + req.body = settings.source.to_json + end + raise_connector_error('Failed to create source', response) unless response.success? + end + + def token + if @token.value.present? && @token_expiration.value > 10.seconds.from_now + @token.value + else + update_token + end + end + + def update_token + response = Faraday.post(settings.token_path, auth) + result = JSON.parse(response.body) + + if response.success? + @token_expiration.value = Time.zone.now + result['expires_in'] + @token.value = result['access_token'] + else + raise_connector_error('Failed to get fresh access token', response) + end + end + + def auth + { + grant_type: 'client_credentials', + client_id: settings.client_id, + client_secret: settings.client_secret, + } + end + + def api_conn + Faraday.new(url: settings.api_host, headers:) + end + + def source_slug + settings.source.slug + end + + def settings + return @connector_settings if @connector_settings + + check_settings! + @connector_settings = Settings.nbp&.push_connector + end + + def check_settings! # rubocop:disable Metrics/AbcSize + settings_hash = Settings.nbp&.push_connector.to_h + + if PushConnector.enabled? + missing_keys = %i[client_id client_secret token_path api_host source] - settings_hash.keys + raise SettingsError.new("Nbp::PushConnector is missing some settings: #{missing_keys}") if missing_keys.any? + + missing_source_keys = %i[organization name slug] - settings_hash[:source].keys + raise SettingsError.new("Nbp::PushConnector source is missing some settings: #{missing_source_keys}") if missing_source_keys.any? + else + raise SettingsError.new('Nbp::PushConnector is disabled but got accessed') + end + end + + def raise_connector_error(message, faraday_response) + raise Error.new("#{message} (code #{faraday_response.status}). Response was: '#{faraday_response.body}'") + end + + def headers + {authorization: "Bearer #{token}", 'content-type': 'application/json', accept: 'application/json'} + end + end +end diff --git a/lib/tasks/nbp_push_all_tasks.rake b/lib/tasks/nbp_push_all_tasks.rake new file mode 100644 index 000000000..fc5752f5d --- /dev/null +++ b/lib/tasks/nbp_push_all_tasks.rake @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +namespace :nbp do + desc 'Pushes all tasks to the NBP for an initial sync' + task push_all: :environment do + NbpSyncAllJob.perform_later + end +end diff --git a/spec/fixtures/files/nbp/empty_nodes.json b/spec/fixtures/files/nbp/empty_nodes.json new file mode 100644 index 000000000..36f2fb2cc --- /dev/null +++ b/spec/fixtures/files/nbp/empty_nodes.json @@ -0,0 +1,12 @@ +{ + "total":0, + "offset":3, + "limit":100, + "_embedded":{ + "nodes":[] + }, + "_links":{ + "self":null, + "next":null + } +} diff --git a/spec/fixtures/files/nbp/nodes.json b/spec/fixtures/files/nbp/nodes.json new file mode 100644 index 000000000..5f130d748 --- /dev/null +++ b/spec/fixtures/files/nbp/nodes.json @@ -0,0 +1,151 @@ +{ + "total":3, + "offset":0, + "limit":100, + "_embedded":{ + "nodes":[ + { + "id":"d381bbac-4686-4d6c-8679-5637b5efd104", + "title":"Minimal Hello World", + "description":"Write a simple program that prints \"Hello World\".", + "externalId":"external-id-1", + "sourceId":"a3634a05-ab0a-4550-8961-65796b05ca47", + "metadata":{ + "Amb":{ + "id":"http://localhost:7500/tasks/2", + "name":"Minimal Hello World", + "type":[ + "LearningResource", + "Course" + ], + "creator":[ + { + "name":"some first name jackson351467", + "type":"Organization" + } + ], + "@context":[ + "https://w3id.org/kim/amb/context.jsonld", + "https://schema.org", + { + "@language":"en" + } + ], + "inLanguage":[ + "en" + ], + "dateCreated":"2024-05-24T00:00:00", + "description":"Write a simple program that prints \"Hello World\".", + "isAccessibleForFree":true + } + }, + "class":"LearningOpportunity", + "_embedded":null, + "_links":{ + "self":{ + "href":"https://dam.demo.meinbildungsraum.de/datenraum/api/core/nodes/d381bbac-4686-4d6c-8679-5637b5efd104" + }, + "headNodes":null, + "tailNodes":null, + "childNodes":null + } + }, + { + "id":"500ad140-6a62-43f4-b78e-3554697b27e6", + "title":"nbp test task", + "description":"", + "externalId":"external-id-2", + "sourceId":"a3634a05-ab0a-4550-8961-65796b05ca47", + "metadata":{ + "Amb":{ + "id":"http://localhost:7500/tasks/8", + "name":"nbp test task", + "type":[ + "LearningResource", + "Course" + ], + "creator":[ + { + "name":"john oliver", + "type":"Organization" + } + ], + "@context":[ + "https://w3id.org/kim/amb/context.jsonld", + "https://schema.org", + { + "@language":"en" + } + ], + "inLanguage":[ + "en" + ], + "dateCreated":"2024-06-17T00:00:00", + "description":"", + "isAccessibleForFree":true + } + }, + "class":"LearningOpportunity", + "_embedded":null, + "_links":{ + "self":{ + "href":"https://dam.demo.meinbildungsraum.de/datenraum/api/core/nodes/500ad140-6a62-43f4-b78e-3554697b27e6" + }, + "headNodes":null, + "tailNodes":null, + "childNodes":null + } + }, + { + "id":"50c9dc43-5ede-4748-a1f4-3b045e506d30", + "title":"nbp test task", + "description":"", + "externalId":"external-id-3", + "sourceId":"a3634a05-ab0a-4550-8961-65796b05ca47", + "metadata":{ + "Amb":{ + "id":"http://localhost:7500/tasks/6", + "name":"nbp test task", + "type":[ + "LearningResource", + "Course" + ], + "creator":[ + { + "name":"john oliver", + "type":"Organization" + } + ], + "@context":[ + "https://w3id.org/kim/amb/context.jsonld", + "https://schema.org", + { + "@language":"en" + } + ], + "inLanguage":[ + "en" + ], + "dateCreated":"2024-06-10T00:00:00", + "description":"", + "isAccessibleForFree":true + } + }, + "class":"LearningOpportunity", + "_embedded":null, + "_links":{ + "self":{ + "href":"https://dam.demo.meinbildungsraum.de/datenraum/api/core/nodes/50c9dc43-5ede-4748-a1f4-3b045e506d30" + }, + "headNodes":null, + "tailNodes":null, + "childNodes":null + } + } + ] + }, + "_links":{ + "self":null, + "next":null + } +} diff --git a/spec/jobs/nbp_sync_all_job_spec.rb b/spec/jobs/nbp_sync_all_job_spec.rb new file mode 100644 index 000000000..76957fcd7 --- /dev/null +++ b/spec/jobs/nbp_sync_all_job_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe NbpSyncAllJob do + include ActiveJob::TestHelper + + let(:api_host) { Settings.nbp.push_connector.api_host } + let(:source_slug) { Settings.nbp.push_connector.source.slug } + + let!(:local_uuids) { create_list(:task, 3, access_level: :public).pluck(:uuid) } + let!(:uploaded_uuids) { [Random.uuid, local_uuids.first, local_uuids.second] } + let!(:all_uuids) { local_uuids | uploaded_uuids } + + before do + stub_request(:post, Settings.nbp.push_connector.token_path).to_return_json(body: {token: 'sometoken', expires_in: 600}) + stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 200) + + allow(Nbp::PushConnector.instance).to receive(:get_uploaded_task_uuids).with(0).and_return(uploaded_uuids) + allow(Nbp::PushConnector.instance).to receive(:get_uploaded_task_uuids).with(3).and_return([]) + end + + describe 'perform' do + subject(:perform_job) { described_class.perform_now } + + it 'schedules the correct sync jobs' do + expect { perform_job }.to have_enqueued_job(NbpSyncJob).with(all_uuids.first) + .and have_enqueued_job(NbpSyncJob).with(all_uuids.second) + .and have_enqueued_job(NbpSyncJob).with(all_uuids.third) + .and have_enqueued_job(NbpSyncJob).with(all_uuids.fourth) + end + end + + describe 'rake' do + subject(:push_all) { Rake::Task['nbp:push_all'].invoke } + + before { Rails.application.load_tasks if Rake::Task.tasks.empty? } + + it 'schedules the desired job' do + expect { push_all }.to have_enqueued_job(described_class) + end + end +end diff --git a/spec/jobs/nbp_sync_job_spec.rb b/spec/jobs/nbp_sync_job_spec.rb new file mode 100644 index 000000000..4b89f7129 --- /dev/null +++ b/spec/jobs/nbp_sync_job_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe NbpSyncJob do + let(:task) { create(:task, access_level:) } + let(:uuid) { task.uuid } + + before do + allow(Nbp::PushConnector).to receive(:instance) + allow(Nbp::PushConnector.instance).to receive(:push_lom!) + allow(Nbp::PushConnector.instance).to receive(:delete_task!) + end + + describe 'perform' do + subject(:perform_job) { described_class.perform_now(uuid) } + + context 'when the task is public' do + let(:access_level) { :public } + + it 'pushes the task' do + perform_job + expect(Nbp::PushConnector.instance).to have_received(:push_lom!) + end + end + + context 'when the task does not exist' do + let(:uuid) { :not_existing_uuid } + + it 'deletes the task' do + perform_job + expect(Nbp::PushConnector.instance).to have_received(:delete_task!) + end + end + + context 'when the task is private' do + let(:access_level) { :private } + + it 'deletes the task' do + perform_job + expect(Nbp::PushConnector.instance).to have_received(:delete_task!) + end + end + end +end diff --git a/spec/lib/nbp/push_connector_spec.rb b/spec/lib/nbp/push_connector_spec.rb new file mode 100644 index 000000000..a764e5857 --- /dev/null +++ b/spec/lib/nbp/push_connector_spec.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Nbp::PushConnector do + let(:api_host) { Settings.nbp.push_connector.api_host } + let(:source_slug) { Settings.nbp.push_connector.source.slug } + let(:token_expiration) { 600 } + + let(:task) { create(:task) } + let(:task_xml) { (Nokogiri::XML::Builder.new(encoding: 'UTF-8') {|xml| LomService::ExportLom.call(task:, xml:) }).to_xml } + + let(:connector) { Class.new(described_class).instance } # https://stackoverflow.com/a/23901644 + + before do + stub_request(:post, Settings.nbp.push_connector.token_path).to_return_json(body: {access_token: 'sometoken', expires_in: token_expiration}) + stub_request(:post, "#{api_host}/datenraum/api/core/sources") + stub_request(:put, "#{api_host}/push-connector/api/lom-v2/#{source_slug}") + stub_request(:delete, %r{#{api_host}/push-connector/api/course/#{source_slug}/}) + stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 404) + + stub_request(:get, %r{#{api_host}/datenraum/api/core/nodes}) + .to_return(body: file_fixture('nbp/empty_nodes.json')) + + stub_request(:get, %r{#{api_host}/datenraum/api/core/nodes}) + .with(query: hash_including('offset' => '0')) + .to_return(body: file_fixture('nbp/nodes.json')) + end + + describe 'initialize' do + context 'when no source exists' do + it 'creates a source' do + connector + expect(WebMock).to have_requested(:post, "#{api_host}/datenraum/api/core/sources") + end + + context 'when the push connector is disabled' do + before do + # Disable push connector temporarily + Settings.nbp.push_connector.enable = false + described_class.instance_variable_set :@enabled, nil + end + + after do + # Allow push connector to be re-enabled + Settings.nbp.push_connector.enable = true + described_class.instance_variable_set :@enabled, nil + end + + it 'raises an error' do + expect { connector }.to raise_error(Nbp::PushConnector::SettingsError) + end + end + end + + context 'when a source exists' do + before { stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 200) } + + it 'does not create a source' do + connector + expect(WebMock).not_to have_requested(:post, "#{api_host}/datenraum/api/core/sources") + end + end + + context 'when the source could not be determined' do + before { stub_request(:get, "#{api_host}/datenraum/api/core/sources/slug/#{source_slug}").to_return(status: 500) } + + it 'does not create a source' do + begin + connector + rescue Nbp::PushConnector::Error + # no op for the spec + end + + expect(WebMock).not_to have_requested(:post, "#{api_host}/datenraum/api/core/sources") + end + + it 'raises an error' do + expect { connector }.to raise_error(Nbp::PushConnector::Error) + end + end + end + + describe 'push_lom!' do + subject(:push_lom!) { connector.push_lom!(task_xml) } + + context 'without any errors' do + it 'pushes the metadata' do + push_lom! + expect(WebMock).to have_requested(:put, "#{api_host}/push-connector/api/lom-v2/#{source_slug}") + end + end + + context 'when the token is still valid' do + before do + connector + WebMock.reset_executed_requests! + end + + it 'does not renew the token' do + push_lom! + expect(WebMock).not_to have_requested(:post, Settings.nbp.push_connector.token_path) + end + end + + context 'when the token expired' do + let(:token_expiration) { 0 } + + before do + connector + WebMock.reset_executed_requests! + end + + it 'renews the token' do + push_lom! + expect(WebMock).to have_requested(:post, Settings.nbp.push_connector.token_path) + end + end + + context 'when the token cannot be renewed' do + let(:token_expiration) { 0 } + + before do + connector + stub_request(:post, Settings.nbp.push_connector.token_path).to_return_json(body: {}, status: 500) + WebMock.reset_executed_requests! + end + + it 'raises an error' do + expect { push_lom! }.to raise_error(Nbp::PushConnector::Error) + end + + it 'does not push the metadata' do + begin + push_lom! + rescue Nbp::PushConnector::Error + # no op for the spec + end + + expect(WebMock).not_to have_requested(:put, "#{api_host}/push-connector/api/lom-v2/#{source_slug}") + end + end + end + + describe 'delete_task!' do + subject(:delete_task!) { connector.delete_task!(task.uuid) } + + context 'without any errors' do + it 'pushes the metadata' do + delete_task! + expect(WebMock).to have_requested(:delete, "#{api_host}/push-connector/api/course/#{source_slug}/#{task.uuid}") + end + end + end + + describe 'process_uploaded_task_uuids' do + it 'iterates the correct UUIDs' do + uuids = [] + connector.process_uploaded_task_uuids {|uuid| uuids << uuid } + expect(uuids).to eq(%w[external-id-1 external-id-2 external-id-3]) + end + end +end diff --git a/spec/models/task_spec.rb b/spec/models/task_spec.rb index b3e5ea0f4..39816f2ad 100644 --- a/spec/models/task_spec.rb +++ b/spec/models/task_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' RSpec.describe Task do + include ActiveJob::TestHelper + describe '#valid?' do it { is_expected.to validate_presence_of(:title) } it { is_expected.to validate_uniqueness_of(:uuid).case_insensitive } @@ -107,7 +109,7 @@ end it 'has the correct parent_uuid' do - expect(duplicate.parent_uuid).to be task.uuid + expect(duplicate.parent_uuid).to eq task.uuid end it 'has the same attributes' do @@ -186,5 +188,86 @@ destroy expect(collection.reload.tasks).to be_empty end + + it 'enqueues an NbpSyncJob' do + expect { destroy }.to have_enqueued_job(NbpSyncJob).with(task.uuid) + end + end + + describe '#update' do + subject(:update) { task.update(new_attributes) } + + let!(:task) { create(:task, access_level:) } + + context 'when updating a public task' do + let(:access_level) { :public } + let(:new_attributes) { {title: 'some new title'} } + + it 'enqueues an NbpSyncJob' do + expect { update }.to have_enqueued_job(NbpSyncJob).with(task.uuid) + end + + context 'when updating the uuid' do + let(:new_attributes) { {uuid: Random.uuid} } + let!(:old_uuid) { task.uuid } + + it 'enqueues an NbpSyncJob for the old uuid' do + expect { update }.to have_enqueued_job(NbpSyncJob).with(old_uuid) + end + end + end + + context 'when updating a private task' do + let(:access_level) { :private } + let(:new_attributes) { {title: 'some new title'} } + + it 'does not enqueue an NbpSyncJob' do + expect { update }.not_to have_enqueued_job(NbpSyncJob) + end + + context 'when updating the uuid' do + let(:new_attributes) { {uuid: Random.uuid} } + + it 'does not enqueue an NbpSyncJob' do + expect { update }.not_to have_enqueued_job(NbpSyncJob) + end + end + end + + context 'when changing the access level from private to public' do + let(:access_level) { :private } + let(:new_attributes) { {access_level: :public} } + + it 'enqueues an NbpSyncJob' do + expect { update }.to have_enqueued_job(NbpSyncJob).with(task.uuid) + end + + context 'when also updating the uuid' do + let(:new_attributes) { {access_level: :public, uuid: Random.uuid} } + let!(:old_uuid) { task.uuid } + + it 'does not enqueue an NbpSyncJob for the old uuid' do + expect { update }.not_to have_enqueued_job(NbpSyncJob).with(old_uuid) + end + end + end + + context 'when changing the access level from public to private' do + let(:access_level) { :public } + let(:new_attributes) { {access_level: :private} } + + it 'enqueues an NbpSyncJob' do + expect { update }.to have_enqueued_job(NbpSyncJob).with(task.uuid) + end + + context 'when also updating the uuid' do + let(:new_attributes) { {access_level: :private, uuid: Random.uuid} } + let!(:old_uuid) { task.uuid } + + it 'enqueues an NbpSyncJob for the old uuid' do + expect { update }.to have_enqueued_job(NbpSyncJob).with(old_uuid) + end + end + end end end