Merge pull request #46 from datalad/gh-32

Support invoking `registry-submit` with neither `--sibling` nor `--url`

datalad_registry_client/opts.py

@@ -3,7 +3,9 @@
 
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Optional
+from typing import Tuple
 
 from datalad.distribution.dataset import Dataset
 from datalad.distribution.dataset import EnsureDataset
@@ -12,7 +14,6 @@
 from datalad.support.constraints import EnsureStr
 from datalad.support.param import Parameter
 
-from datalad_registry.utils import url_encode
 from datalad_registry_client.consts import DEFAULT_ENDPOINT
 
 common_params = dict(
@@ -65,17 +66,23 @@ def process_args(
         if sibling not in remotes:
             raise ValueError("Unknown sibling: {}".format(sibling))
 
-    if not url:
-        if not sibling:
-            raise ValueError(
-                "Must specify URL to use when sibling isn't given")
+    urls: List[str] = []
+    if url:
+        urls.append(url)
+    elif sibling:
         url = repo.config.get("remote.{}.url".format(sibling))
-        if not url:
+        if url is None:
             raise ValueError("Could not find URL for {}".format(sibling))
+        urls.append(url)
+    else:
+        for r in repo.get_remotes(with_urls_only=True):
+            url = repo.get_remote_url(r)
+            assert url is not None
+            urls.append(url)
 
     endpoint = endpoint or repo.config.get(
         "datalad_registry.endpoint",
         DEFAULT_ENDPOINT)
     return dict(ds=ds, ds_id=ds_id,
-                sibling=sibling, url=url, url_encoded=url_encode(url),
+                sibling=sibling, urls=urls,
                 endpoint=endpoint)

datalad_registry_client/submit.py

@@ -12,8 +12,10 @@
 from datalad.interface.results import get_status_dict
 from datalad.interface.utils import eval_results
 
+from datalad_registry.utils import url_encode
 from datalad_registry_client import opts
 
+
 lgr = logging.getLogger("datalad.registry.submit")
 
 
@@ -37,36 +39,39 @@ def __call__(
         options = opts.process_args(
             dataset=dataset, sibling=sibling, url=url, endpoint=endpoint)
         ds_id = options["ds_id"]
-        url = options["url"]
-        url_encoded = options['url_encoded']
+        urls = options.pop("urls")  # Don't include in res_base
 
         res_base = get_status_dict(action="registry-submit",
                                    logger=lgr, **options)
 
         base_url = f"{options['endpoint']}/datasets"
 
-        try:
-            r_url = requests.get(
-                f"{base_url}/{ds_id}/urls/{url_encoded}",
-                timeout=1)
-            r_url.raise_for_status()
-        except requests.exceptions.RequestException as exc:
-            yield dict(res_base, status="error",
-                       message=("Check if URL is known failed: %s", exc))
-            return
-        url_info = r_url.json()
-        if url_info.get("status") == "unknown":
-            msg = "Registered URL"
-        else:
-            msg = "Announced update"
+        for u in urls:
+            url_encoded = url_encode(u)
+            try:
+                r_url = requests.get(
+                    f"{base_url}/{ds_id}/urls/{url_encoded}",
+                    timeout=1)
+                r_url.raise_for_status()
+            except requests.exceptions.RequestException as exc:
+                yield dict(res_base, status="error", url=u,
+                           url_encoded=url_encoded,
+                           message=("Check if URL is known failed: %s", exc))
+                return
+            url_info = r_url.json()
+            if url_info.get("status") == "unknown":
+                msg = "Registered URL"
+            else:
+                msg = "Announced update"
 
-        try:
-            r_patch = requests.patch(f"{base_url}/{ds_id}/urls/{url_encoded}",
-                                     timeout=1)
-            r_patch.raise_for_status()
-        except requests.exceptions.RequestException as exc:
-            yield dict(res_base, status="error",
-                       message=("Submitting URL failed: %s", exc))
-            return
-        yield dict(res_base, status="ok",
-                   message=("%s: %s", msg, url))
+            try:
+                r_patch = requests.patch(f"{base_url}/{ds_id}/urls/{url_encoded}",
+                                         timeout=1)
+                r_patch.raise_for_status()
+            except requests.exceptions.RequestException as exc:
+                yield dict(res_base, status="error", url=u,
+                           url_encoded=url_encoded,
+                           message=("Submitting URL failed: %s", exc))
+                return
+            yield dict(res_base, status="ok", url=u, url_encoded=url_encoded,
+                       message=("%s: %s", msg, u))

datalad_registry_client/tests/test_submit.py

@@ -1,3 +1,5 @@
+import os
+import time
 import subprocess as sp
 
 import datalad.api as dl
@@ -29,14 +31,10 @@ def test_submit_via_local(tmp_path):
 
     assert requests.get(query_url).json()["status"] == "unknown"
 
-    # If sibling is not specified, URL is required.
-    with pytest.raises(ValueError):
-        ds.registry_submit()
-
     assert_in_results(
         ds.registry_submit(url=ds.path),
         action="registry-submit", type="dataset",
-        path=ds.path, status="ok")
+        path=ds.path, url=ds.path, status="ok")
 
     assert requests.get(query_url).json()["status"] != "unknown"
 
@@ -45,7 +43,7 @@ def test_submit_via_local(tmp_path):
     assert_in_results(
         res,
         action="registry-submit", type="dataset",
-        path=ds.path, status="ok")
+        path=ds.path, url=ds.path, status="ok")
 
 
 @pytest.mark.slow
@@ -77,11 +75,40 @@ def test_submit_via_sibling(tmp_path):
     assert_in_results(
         ds.registry_submit(sibling="origin"),
         action="registry-submit", type="dataset",
-        path=ds.path, status="ok")
+        path=ds.path, url=ds_sib.path, status="ok")
 
     assert requests.get(query_url).json()["status"] != "unknown"
 
 
+@pytest.mark.devserver
+@pytest.mark.slow
+def test_submit_all_siblings(tmp_path):
+    ds_sib = dl.Dataset(tmp_path / "sib").create()
+    ds = dl.clone(ds_sib.path, str(tmp_path / "clone"))
+
+    pid = os.getpid()
+    ts = time.time()
+    url2 = f"https://www.example.nil/{pid}/{ts}/repo.git"
+    ds.config.set("remote.sibling2.url", url2, where="local")
+
+    ds_id = ds.id
+
+    urls = [ds_sib.path, url2]
+    query_urls = [f"{DEFAULT_ENDPOINT}/datasets/{ds_id}/urls/{url_encode(u)}" for u in urls]
+
+    for qu in query_urls:
+        assert requests.get(qu).json()["status"] == "unknown"
+
+    for u in urls:
+        assert_in_results(
+            ds.registry_submit(),
+            action="registry-submit", type="dataset",
+            path=ds.path, url=u, status="ok")
+
+    for qu in query_urls:
+        assert requests.get(qu).json()["status"] != "unknown"
+
+
 @pytest.mark.devserver
 @pytest.mark.slow
 def test_submit_explicit_endpoint(tmp_path):
@@ -92,7 +119,7 @@ def test_submit_explicit_endpoint(tmp_path):
     assert_in_results(
         ds.registry_submit(url=ds.path, endpoint="abc", on_failure="ignore"),
         action="registry-submit", type="dataset",
-        path=ds.path, status="error")
+        path=ds.path, url=ds.path, status="error")
 
     # Valid, explicit.
     url_encoded = url_encode(ds.path)
@@ -101,6 +128,6 @@ def test_submit_explicit_endpoint(tmp_path):
     assert_in_results(
         ds.registry_submit(url=ds.path, endpoint=DEFAULT_ENDPOINT),
         action="registry-submit", type="dataset",
-        path=ds.path, status="ok")
+        path=ds.path, url=ds.path, status="ok")
 
     assert requests.get(query_url).json()["status"] != "unknown"