From 9778b0fbcc842d6104df5c02926ae3e033209dbb Mon Sep 17 00:00:00 2001 From: Ross Spencer Date: Sun, 23 Feb 2020 19:47:53 -0500 Subject: [PATCH] Replicate AIP bag contents not container dir Ensure that during replication we're copying the AIP (bag) contents and not the container directory. Copying the container directory pushes the replicated structure one level further down than we expect and so when we come to test for something, e.g. fixity, we cannot find the structural aspects of the bag, e.g. manifests, to perform validation. --- storage_service/locations/models/package.py | 2 +- .../locations/tests/test_package.py | 38 +++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/storage_service/locations/models/package.py b/storage_service/locations/models/package.py index e572fa19c..ac42e445c 100644 --- a/storage_service/locations/models/package.py +++ b/storage_service/locations/models/package.py @@ -610,7 +610,7 @@ def replicate(self, replicator_location): # Copy replicandum AIP from its source location to the SS src_space.move_to_storage_service( source_path=os.path.join( - replicandum_location.relative_path, replicandum_path + replicandum_location.relative_path, replicandum_path, "" ), destination_path=replica_package.current_path, destination_space=dest_space, diff --git a/storage_service/locations/tests/test_package.py b/storage_service/locations/tests/test_package.py index e17cc4041..657b57b93 100644 --- a/storage_service/locations/tests/test_package.py +++ b/storage_service/locations/tests/test_package.py @@ -11,6 +11,7 @@ from django.core.urlresolvers import reverse from django.test import TestCase +from common import utils from locations import models import bagit @@ -388,6 +389,30 @@ def test_run_post_store_callbacks_dip(self): body = '{"download_url": "http://ss.com/api/v2/file/%s/download/"}' % uuid mocked_execute.assert_called_with(url, body) + @staticmethod + def _test_bagit_structure(replica, replication_dir): + """Ensure the replicated bagit structure remains.""" + bag_contents = [ + "tagmanifest-md5.txt", + "bagit.txt", + "manifest-md5.txt", + "bag-info.txt", + os.path.join("data", "test.txt"), + ] + expected_bag_path = os.path.join( + replication_dir, utils.uuid_to_path(replica.uuid), "working_bag" + ) + expected_bagit_structure = [ + os.path.join(expected_bag_path, bag_path) for bag_path in bag_contents + ] + for subdir, _, files in os.walk(replica.current_location.full_path): + for file_ in files: + file_path = os.path.join(subdir, file_) + if file_path not in expected_bagit_structure: + pytest.fail( + "Unexpected file in Bagit structure: {}".format(file_path) + ) + def test_replicate_aip(self): space_dir = tempfile.mkdtemp(dir=self.tmp_dir, prefix="space") replication_dir = tempfile.mkdtemp(dir=self.tmp_dir, prefix="replication") @@ -401,13 +426,12 @@ def test_replicate_aip(self): ) aip.create_replicas() - assert aip.replicas.count() == 1 - replica = aip.replicas.first() assert replica is not None assert replica.origin_pipeline == aip.origin_pipeline assert replica.replicas.count() == 0 + self._test_bagit_structure(aip.replicas.first(), replication_dir) def test_replicate_aip_twice(self): space_dir = tempfile.mkdtemp(dir=self.tmp_dir, prefix="space") @@ -430,8 +454,13 @@ def test_replicate_aip_twice(self): aip.create_replicas() assert aip.replicas.count() == 2 - for replica in aip.replicas.all(): - assert replica.replicas.count() == 0 + assert aip.replicas.first() != aip.replicas.last() + + assert aip.replicas.first().replicas.count() == 0 + assert aip.replicas.last().replicas.count() == 0 + + self._test_bagit_structure(aip.replicas.first(), replication_dir) + self._test_bagit_structure(aip.replicas.last(), replication_dir2) @mock.patch("locations.models.gpg._gpg_encrypt") def test_replicate_aip_gpg_encrypted(self, mock_encrypt): @@ -460,6 +489,7 @@ def test_replicate_aip_gpg_encrypted(self, mock_encrypt): assert replica is not None assert mock_encrypt.call_args_list == [mock.call(replica.full_path, u"")] + self._test_bagit_structure(replica, replication_dir) class TestTransferPackage(TestCase):