Add remove_export; fix bug in run_dark_blue; tweak .gitignore

.gitignore

@@ -1,8 +1,7 @@
 # Data directories
-export
-prep
-restore
-source
+export/*
+prep/*
+restore/*
 
 # Configuration
 .ssh

example.env

@@ -5,6 +5,8 @@
 SETTINGS_LOG_LEVEL=debug
 SETTINGS_WORKING_DIR=./prep
 SETTINGS_EXPORT_DIR=./export
+# Determines whether to remove tar files from export, either true or false (should be true in production)
+SETTINGS_REMOVE_EXPORT=true
 # Determines whether the process skips sending bag(s)
 SETTINGS_DRY_RUN=false
 # Limit for the size (in bytes) of objects to be processed (optional)

lib/bag_courier.rb

@@ -37,6 +37,7 @@ def initialize(
       target_client:,
       working_dir:,
       export_dir:,
+      remove_export:,
       dry_run:,
       status_event_repo:,
       validator:
@@ -51,6 +52,7 @@ def initialize(
 
       @working_dir = working_dir
       @export_dir = export_dir
+      @remove_export = remove_export
       @dry_run = dry_run
       @validator = validator
     end
@@ -73,7 +75,7 @@ def tar(target_path:, output_dir_path:)
 
       parent = File.dirname(target_path)
       tar_src = File.basename(target_path)
-      tar_file = File.basename(target_path) + EXT_TAR
+      tar_file = tar_src + EXT_TAR
       new_path = File.join(output_dir_path, tar_file)
 
       Dir.chdir(parent) do
@@ -132,6 +134,8 @@ def deliver
 
         export_tar_file_path = tar(target_path: bag.bag_dir, output_dir_path: @export_dir)
         deposit(file_path: export_tar_file_path)
+
+        FileUtils.rm(export_tar_file_path) if @remove_export
       rescue => e
         note = "failed with error #{e.class}: #{e.full_message}"
         track!(status: BagStatus::FAILED, note: note)

lib/config.rb

@@ -106,6 +106,7 @@ def get_subset_by_key_stem(stem)
     :log_level,
     :working_dir,
     :export_dir,
+    :remove_export,
     :dry_run,
     :object_size_limit,
     keyword_init: true
@@ -283,6 +284,7 @@ def self.create_config(data)
           working_dir: data.get_value(key: "SETTINGS_WORKING_DIR"),
           export_dir: data.get_value(key: "SETTINGS_EXPORT_DIR"),
           dry_run: data.get_value(key: "SETTINGS_DRY_RUN", checks: [BOOLEAN_CHECK]) == "true",
+          remove_export: data.get_value(key: "SETTINGS_REMOVE_EXPORT", checks: [BOOLEAN_CHECK]) == "true",
           object_size_limit: data.get_value(
             key: "SETTINGS_OBJECT_SIZE_LIMIT", checks: [IntegerCheck.new], optional: true
           )&.to_i

lib/dispatcher.rb

@@ -75,6 +75,7 @@ def dispatch(
         status_event_repo: @status_event_repo,
         working_dir: @settings.working_dir,
         export_dir: @settings.export_dir,
+        remove_export: @settings.remove_export,
         dry_run: @settings.dry_run,
         validator: validator
       )

run_dark_blue.rb

@@ -105,7 +105,7 @@ def redeliver_package(identifier)
     end
 
     extra_bag_info_data = create_extra_bag_info_data(
-      content_type: arch_config.name, location_uuid: api_config.location_uuid
+      content_type: arch_config.name, location_uuid: arch_config.api.location_uuid
     )
     arch_service = prepare_arch_service(name: arch_config.name, api_config: arch_config.api)
     package_data = arch_service.get_package_data_object(package.identifier)

test/test_bag_courier.rb

@@ -1,6 +1,7 @@
 require "minitar"
 require "minitest/autorun"
 require "minitest/pride"
+require "semantic_logger"
 
 require_relative "setup_db"
 require_relative "../lib/bag_adapter"
@@ -15,6 +16,9 @@
 require_relative "../lib/repository_package_repository"
 require_relative "../lib/status_event_repository"
 
+SemanticLogger.add_appender(io: $stderr, formatter: :color)
+SemanticLogger.default_level = :debug
+
 class BagIdTest < Minitest::Test
   def test_to_s
     expected = "somerepo.uniqueid"
@@ -120,14 +124,15 @@ def setup
     )
   end
 
-  def create_courier(dry_run:, target_client:, validator: @validator)
+  def create_courier(dry_run:, target_client:, validator: @validator, remove_export: false)
     BagCourier::BagCourier.new(
       bag_id: @bag_id,
       bag_info: @bag_info,
       tags: [@aptrust_info],
       data_transfer: @data_transfer,
       working_dir: @prep_path,
       export_dir: @export_path,
+      remove_export: remove_export,
       dry_run: dry_run,
       status_event_repo: @status_event_repo,
       target_client: target_client,
@@ -187,4 +192,15 @@ def test_deliver_when_deposit_raises_error
     statuses = @status_event_repo.get_all.sort_by(&:timestamp).map(&:status)
     assert_equal expected_statuses, statuses
   end
+
+  def test_deliver_with_remove_export
+    courier = create_courier(dry_run: false, target_client: @mock_target_client, remove_export: true)
+    expected_tar_file_path = File.join(@export_path, @bag_id.to_s + ".tar")
+    @mock_target_client.expect(:remote_text, "AWS S3 remote location in bucket fake")
+    @mock_target_client.expect(:send_file, nil, local_file_path: expected_tar_file_path)
+    courier.deliver
+    @mock_target_client.verify
+
+    refute File.exist?(expected_tar_file_path)
+  end
 end