DBAI-29 Canister Implementation (#37)

* DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * testing if this solution works * added default value * Update tests.yml * added config loglevel * config loglevel updated * updated test.yml * updated env * updated env * updated env * Update tests.yml * TYPE added * upd type * upd type * upd yaml * some more clean up * Update services.rb * removed config dependencies * removed config dependencies * updated gem version and removed canister reg for logger * Fix missed usage of get_value without key * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * code cleanup * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * DBAI-29 Canister Implementation * code cleanup * code cleanup * code clean up * code clean up * Added loglevel method --------- Co-authored-by: Samuel Sciolla <ssciolla@umich.edu>
mlibrary · Apr 20, 2024 · 824edd5 · 824edd5
1 parent 47b8695
commit 824edd5
Show file tree

Hide file tree

Showing 18 changed files with 96 additions and 83 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -35,10 +35,11 @@ jobs:
           ruby-version: '3.2.3'
           bundler-cache: true
       - name: Run tests
-        run: bundle exec rake test
-        env: 
+        env:
+          SETTINGS_LOG_LEVEL: debug
           DATABASE_HOST: 127.0.0.1
           DATABASE_DATABASE: test_database
           DATABASE_PORT: 3306
           DATABASE_USER: user
           DATABASE_PASSWORD: password
+        run: bundle exec rake test
diff --git a/Gemfile b/Gemfile
@@ -2,6 +2,7 @@ source "https://rubygems.org"
 
 gem "aws-sdk-s3", "~> 1.136"
 gem "bagit", "~> 0.4.6"
+gem "canister", "~> 0.9.2"
 gem "faraday", "~> 2.9"
 gem "faraday-retry", "~> 2.2"
 gem "minitar", "~> 0.9"

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -30,6 +30,7 @@ GEM
       validatable (~> 1.6)
     base64 (0.2.0)
     bigdecimal (3.1.6)
+    canister (0.9.2)
     concurrent-ruby (1.2.3)
     docopt (0.5.0)
     dotenv (3.1.0)
@@ -62,6 +63,7 @@ PLATFORMS
 DEPENDENCIES
   aws-sdk-s3 (~> 1.136)
   bagit (~> 0.4.6)
+  canister (~> 0.9.2)
   dotenv (~> 3.1.0)
   faraday (~> 2.9)
   faraday-retry (~> 2.2)

diff --git a/Rakefile b/Rakefile
@@ -9,26 +9,15 @@ namespace :db do
   task :migrate, [:version] do |t, args|
     require "sequel/core"
     require_relative "db/database_error"
-    require_relative "lib/config"
+    require_relative "services"
 
     Sequel.extension :migration
 
-    db_config = Config::ConfigService.database_config_from_env
-    if !db_config
+    if !S.db_config
       raise DatabaseError, "Migration failed. A database connection is not configured."
     end
 
     version = args[:version].to_i if args[:version]
-    Sequel.connect(
-      adapter: "mysql2",
-      host: db_config.host,
-      port: db_config.port,
-      database: db_config.database,
-      user: db_config.user,
-      password: db_config.password,
-      fractional_seconds: true
-    ) do |db|
-      Sequel::Migrator.run(db, "db/migrations", target: version)
-    end
+    Sequel::Migrator.run(S.dbconnect, "db/migrations", target: version)
   end
 end
diff --git a/example.env b/example.env
@@ -34,7 +34,8 @@ ARCHIVEMATICA_DEV_REPOSITORY_NAME=
 # REST API
 ARCHIVEMATICA_DEV_API_USERNAME=
 ARCHIVEMATICA_DEV_API_API_KEY=
-ARCHIVEMATICA_DEV_BASE_URL=
+ARCHIVEMATICA_DEV_API_BASE_URL=
+ARCHIVEMATICA_DEV_API_LOCATION_UUID=
 # Remote
 ARCHIVEMATICA_DEV_REMOTE_TYPE=file_system
 ARCHIVEMATICA_DEV_REMOTE_SETTINGS_FILE_SYSTEM_REMOTE_PATH=

diff --git a/lib/aptrust.rb b/lib/aptrust.rb
@@ -1,7 +1,6 @@
-require "semantic_logger"
-
 require_relative "api_backend"
 require_relative "bag_status"
+require_relative "../services"
 
 module APTrust
   module IngestStatus
@@ -13,7 +12,7 @@ module IngestStatus
   end
 
   class APTrustAPI
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     API_V3 = "/member-api/v3/"
     DEFAULT_OBJECT_ID_PREFIX = "umich.edu/"
@@ -75,7 +74,7 @@ def get_ingest_status(bag_identifier:, deposited_at:)
   end
 
   class APTrustVerifier
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     def initialize(aptrust_api:, status_event_repo:)
       @aptrust_api = aptrust_api

diff --git a/lib/archivematica.rb b/lib/archivematica.rb
@@ -1,7 +1,6 @@
-require "semantic_logger"
-
 require_relative "api_backend"
 require_relative "repository_data"
+require_relative "../services"
 
 module Archivematica
   Package = Struct.new(
@@ -19,7 +18,7 @@ module PackageStatus
   end
 
   class ArchivematicaAPI
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     LOCATION_PATH = "location/"
     PACKAGE_PATH = "file/"
@@ -129,7 +128,7 @@ def filter(packages)
   end
 
   class SizePackageFilter < PackageFilterBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     def initialize(size_limit)
       @size_limit = size_limit
@@ -143,7 +142,7 @@ def filter(packages)
   end
 
   class ArchivematicaService
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     NA = "Not available"
 

diff --git a/lib/bag_adapter.rb b/lib/bag_adapter.rb
@@ -1,9 +1,10 @@
 require "bagit"
-require "semantic_logger"
+
+require_relative "../services"
 
 module BagAdapter
   class BagAdapter
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     attr_reader :additional_tag_files
 

diff --git a/lib/bag_courier.rb b/lib/bag_courier.rb
@@ -1,9 +1,9 @@
 require "minitar"
-require "semantic_logger"
 
 require_relative "bag_adapter"
 require_relative "bag_status"
 require_relative "remote_client"
+require_relative "../services"
 
 module BagCourier
   class BagId
@@ -23,7 +23,7 @@ def to_s
   end
 
   class BagCourier
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     EXT_TAR = ".tar"
 

diff --git a/lib/bag_repository.rb b/lib/bag_repository.rb
@@ -1,6 +1,5 @@
-require "semantic_logger"
-
 require_relative "../db/database_schema" if DB
+require_relative "../services"
 
 module BagRepository
   Bag = Struct.new(
@@ -30,7 +29,7 @@ def get_all
   end
 
   class BagInMemoryRepository < BagRepositoryBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     attr_reader :bags
 
@@ -72,7 +71,7 @@ def get_all
   end
 
   class BagDatabaseRepository < BagRepositoryBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     def create(identifier:, group_part:, repository_package_identifier:)
       package = DatabaseSchema::RepositoryPackage.find(identifier: repository_package_identifier)

diff --git a/lib/bag_validator.rb b/lib/bag_validator.rb
@@ -5,6 +5,7 @@ def validate(path)
     raise NotImplementedError, "subclass must implement the validate method"
   end
 end
+
 class InnerBagValidator < BagValidator
   def initialize(inner_bag_name)
     @inner_bag_name = inner_bag_name
@@ -20,9 +21,9 @@ def validate(data_path)
     validity = @bag.valid?
 
     if !validity
-      raise BagValidationError, "Inner bag is not valid: #{@bag.errors.full_messages.join(', ')}"
+      raise BagValidationError, "Inner bag is not valid: #{@bag.errors.full_messages.join(", ")}"
     else
-      return validity
+      validity
     end
   end
 end

diff --git a/lib/config.rb b/lib/config.rb
@@ -312,5 +312,9 @@ def self.database_config_from_env
     def self.from_env
       create_config(ENV.to_hash)
     end
+
+    def self.log_level_from_env
+      CheckableData.new(ENV.to_hash).get_value(key: "SETTINGS_LOG_LEVEL", checks: [LOG_LEVEL_CHECK]).to_sym
+    end
   end
 end
diff --git a/lib/remote_client.rb b/lib/remote_client.rb
@@ -1,9 +1,10 @@
 require "bundler/setup"
 
 require "aws-sdk-s3"
-require "semantic_logger"
 require "sftp"
 
+require_relative "../services"
+
 module RemoteClient
   class RemoteClientError < StandardError
   end
@@ -27,7 +28,7 @@ def retrieve_from_path(local_path:, remote_path: nil)
   end
 
   class FileSystemRemoteClient < RemoteClientBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     def initialize(base_dir_path)
       @base_dir_path = base_dir_path
@@ -67,7 +68,7 @@ def retrieve_from_path(local_path:, remote_path: nil)
   end
 
   class AwsS3RemoteClient < RemoteClientBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     UPLOAD_PROGRESS = proc do |bytes, totals|
       percentage = (100.0 * bytes.sum / totals.sum).round(2)

diff --git a/lib/repository_package_repository.rb b/lib/repository_package_repository.rb
@@ -1,6 +1,5 @@
-require "semantic_logger"
-
 require_relative "../db/database_schema" if DB
+require_relative "../services"
 
 module RepositoryPackageRepository
   RepositoryPackage = Struct.new(
@@ -38,7 +37,7 @@ def get_max_updated_at_for_repository(repository_name)
   end
 
   class RepositoryPackageInMemoryRepository < RepositoryPackageRepositoryBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     attr_reader :repository_packages
 
@@ -96,7 +95,7 @@ def get_max_updated_at_for_repository(repository_name)
   end
 
   class RepositoryPackageDatabaseRepository < RepositoryPackageRepositoryBase
-    include SemanticLogger::Loggable
+    include DarkBlueLogger
 
     def create(identifier:, repository_name:, updated_at:)
       repository = DatabaseSchema::Repository.find_or_create(name: repository_name)

diff --git a/lib/status_event_repository.rb b/lib/status_event_repository.rb
@@ -1,6 +1,5 @@
-require "semantic_logger"
-
 require_relative "bag_status"
+require_relative "../services"
 
 require_relative "../db/database_schema" if DB
 
@@ -91,8 +90,7 @@ def get_latest_event_for_bag(bag_identifier:)
   end
 
   class StatusEventDatabaseRepository
-    include SemanticLogger::Loggable
-
+    include DarkBlueLogger
     def find_or_create_status(status_name)
       DatabaseSchema::Status.find_or_create(name: status_name)
     end

diff --git a/run_dark_blue.rb b/run_dark_blue.rb
@@ -1,23 +1,9 @@
 require "optparse"
 
-require "semantic_logger"
-require "sequel"
+require_relative "services"
 
-require_relative "lib/config"
-
-SemanticLogger.add_appender(io: $stderr, formatter: :color)
-config = Config::ConfigService.from_env
-SemanticLogger.default_level = config.settings.log_level
-
-DB = config.database && Sequel.connect(
-  adapter: "mysql2",
-  host: config.database.host,
-  port: config.database.port,
-  database: config.database.database,
-  user: config.database.user,
-  password: config.database.password,
-  fractional_seconds: true
-)
+config = S.config
+DB = config.database && S.dbconnect
 
 require_relative "lib/archivematica"
 require_relative "lib/bag_repository"
@@ -32,7 +18,7 @@ class DarkBlueError < StandardError
 end
 
 class DarkBlueJob
-  include SemanticLogger::Loggable
+  include DarkBlueLogger
 
   module ExtraBagInfoData
     CONTENT_TYPE_KEY = "Dark-Blue-Content-Type"

diff --git a/services.rb b/services.rb
@@ -0,0 +1,46 @@
+require "canister"
+require "semantic_logger"
+require "sequel"
+
+require_relative "lib/config"
+
+Services = Canister.new
+
+S = Services
+
+# Config
+S.register(:config) do
+  Config::ConfigService.from_env
+end
+
+S.register(:db_config) do
+  Config::ConfigService.database_config_from_env
+end
+
+# Logger
+module DarkBlueLogger
+  def self.included(klass)
+    klass.class_exec do
+      include SemanticLogger
+      include SemanticLogger::Loggable
+      logger = SemanticLogger[klass]
+      if !SemanticLogger::Logger.processor.appenders.console_output?
+        SemanticLogger.add_appender(io: $stderr, formatter: :color)
+        SemanticLogger.default_level = Config::ConfigService.log_level_from_env
+      end
+      logger
+    end
+  end
+end
+
+# Database Connection
+S.register(:dbconnect) do
+  db_config = S.db_config
+  Sequel.connect(adapter: "mysql2",
+    host: db_config.host,
+    port: db_config.port,
+    database: db_config.database,
+    user: db_config.user,
+    password: db_config.password,
+    fractional_seconds: true)
+end