Merge pull request #3064 from OpenNeuroOrg/validate-s3-remote-failure

fix(worker): Allow for remote.log existing without s3-PUBLIC in validation

services/datalad/datalad_service/common/s3.py

@@ -62,14 +62,15 @@ def validate_s3_config(dataset_path):
         else:
             raise
     options_line = ''
-    for line in remote_log.stdout:
+    for line in remote_log.stdout.split('\n'):
         if f'name={get_s3_remote()}' in line:
             options_line = line
-    # check that each of the expected annex options is what's actually configured
-    expected_options = generate_s3_annex_options(dataset_path)
-    for expected_option in expected_options:
-        if not expected_option in options_line:
-            return False
+    if options_line:
+        # check that each of the expected annex options is what's actually configured
+        expected_options = generate_s3_annex_options(dataset_path)
+        for expected_option in expected_options:
+            if not expected_option in options_line:
+                return False
     return True
 
 

services/datalad/tests/test_common_s3.py

@@ -41,9 +41,11 @@ def test_update_s3_sibling(monkeypatch, no_init_remote, new_dataset):
         update_s3_sibling(new_dataset.path)
 
 
-def test_validate_s3_config(monkeypatch, no_init_remote, new_dataset):
-    monkeypatch.setattr(datalad_service.config,
-                        'AWS_S3_PUBLIC_BUCKET', 'a-fake-test-public-bucket')
-    create_remotes(new_dataset.path)
-    # Should fail since the no_init_remote version doesn't match the expected config
+def test_validate_s3_config(no_init_remote, new_dataset):
+    annex_options = generate_s3_annex_options(new_dataset.path)
+    # Set a bad option
+    annex_options = ['type=web']
+    # check=False because this will fail when talking to S3
+    subprocess.run(['git-annex', 'initremote', get_s3_remote()] +
+                   annex_options, check=False, cwd=new_dataset.path)
     assert not validate_s3_config(new_dataset.path)