Merge pull request #3146 from OpenNeuroOrg/remote-preference-fix

fix(worker): Prefer the s3-PUBLIC remote when found

services/datalad/datalad_service/common/annex.py

@@ -93,7 +93,7 @@ def parse_remote_line(remoteLine):
     if remoteConfig['type'] == 'S3' and remoteConfig['bucket'] in S3_BUCKETS_WHITELIST:
         remoteUuid = remoteLine[0:36]
         remoteUrl = remoteConfig['publicurl'] if 'publicurl' in remoteConfig else None
-        return {'uuid': remoteUuid, 'url': remoteUrl}
+        return {'uuid': remoteUuid, 'url': remoteUrl, 'name': remoteConfig['name']}
 
 
 def parse_rmet_line(remote, rmetLine):
@@ -174,6 +174,10 @@ def get_repo_urls(path, files):
             matched_remote = parse_remote_line(line)
             if matched_remote:
                 remote = matched_remote
+                # Prefer the s3-PUBLIC remote if we find one
+                # TODO - Identify the best remote dynamically
+                if matched_remote['name'] == 's3-PUBLIC':
+                    break
     # Check if we found a useful external remote
     if remote:
         # Read the rest of the files.

services/datalad/tests/test_annex.py

@@ -108,7 +108,8 @@ def test_compute_rmet_sha256_annex():
 def test_parse_remote_line():
     remote = parse_remote_line("""57894849-d0c8-4c62-8418-3627be18a196 autoenable=true bucket=openneuro.org datacenter=US encryption=none exporttree=yes fileprefix=ds002778/ host=s3.amazonaws.com name=s3-PUBLIC partsize=1GiB port=80 public=yes publicurl=http://openneuro.org.s3.amazonaws.com/ storageclass=STANDARD type=S3 versioning=yes timestamp=1588743361.538097946s""")
     assert remote == {'url': 'http://openneuro.org.s3.amazonaws.com/',
-                      'uuid': '57894849-d0c8-4c62-8418-3627be18a196'}
+                      'uuid': '57894849-d0c8-4c62-8418-3627be18a196',
+                      'name': 's3-PUBLIC'}
 
 
 def test_parse_rmet_line():