Simplify updating repositories

alibuild_helpers/build.py

@@ -65,27 +65,19 @@ def update_git_repos(args, specs, buildOrder):
     def update_repo(package, git_prompt):
         specs[package]["scm"] = Git()
         if specs[package]["is_devel_pkg"]:
-          localCheckout = os.path.join(os.getcwd(), specs[package]["package"])
-          if exists("%s/.sl" % localCheckout):
-            specs[package]["scm"] = Sapling()
+            specs[package]["source"] = os.path.join(os.getcwd(), specs[package]["package"])
+            if exists(os.path.join(specs[package]["source"], ".sl")):
+                specs[package]["scm"] = Sapling()
         updateReferenceRepoSpec(args.referenceSources, package, specs[package],
                                 fetch=args.fetchRepos,
                                 usePartialClone=not args.docker,
                                 allowGitPrompt=git_prompt)
 
         # Retrieve git heads
-        scm = specs[package]["scm"]
-        cmd = scm.listRefsCmd()
-        if specs[package]["is_devel_pkg"]:
-            specs[package]["source"] = \
-                os.path.join(os.getcwd(), specs[package]["package"])
-            cmd.append(specs[package]["source"])
-        else:
-            cmd.append(specs[package].get("reference", specs[package]["source"]))
-
-        output = logged_scm(scm, package, args.referenceSources,
-                            cmd, ".", prompt=git_prompt, logOutput=False)
-        specs[package]["scm_refs"] = scm.parseRefs(output)
+        output = logged_scm(specs[package]["scm"], package, args.referenceSources,
+                            specs[package]["scm"].listRefsCmd(specs[package].get("reference", specs[package]["source"])),
+                            ".", prompt=git_prompt, logOutput=False)
+        specs[package]["scm_refs"] = specs[package]["scm"].parseRefs(output)
 
     progress = ProgressPrint("Updating repositories")
     requires_auth = set()

alibuild_helpers/git.py

@@ -32,8 +32,8 @@ def parseRefs(self, output):
       git_ref: git_hash for git_hash, sep, git_ref
       in (line.partition("\t") for line in output.splitlines()) if sep
     }
-  def listRefsCmd(self):
-    return ["ls-remote", "--heads", "--tags"]
+  def listRefsCmd(self, repository):
+    return ["ls-remote", "--heads", "--tags", repository]
   def cloneCmd(self, source, referenceRepo, usePartialClone):
     cmd = ["clone", "--bare", source, referenceRepo]
     if usePartialClone:

alibuild_helpers/scm.py

@@ -9,7 +9,7 @@ def branchOrRef(self, directory):
     raise NotImplementedError
   def lsRemote(self, remote):
     raise NotImplementedError
-  def listRefsCmd(self):
+  def listRefsCmd(self, repository):
     raise NotImplementedError
   def parseRefs(self, output):
     raise NotImplementedError

alibuild_helpers/sl.py

@@ -27,8 +27,8 @@ def parseRefs(self, output):
       sl_ref: sl_hash for sl_ref, sep, sl_hash
       in (line.partition("\t") for line in output.splitlines()) if sep
     }
-  def listRefsCmd(self):
-    return ["bookmark", "--list", "--remote", "-R"]
+  def listRefsCmd(self, repository):
+    return ["bookmark", "--list", "--remote", "-R", repository]
   def diffCmd(self, directory):
     return "cd %s && sl diff && sl status" % directory
   def checkUntracked(self, line):

alibuild_helpers/workarea.py

@@ -9,6 +9,7 @@
   from ordereddict import OrderedDict
 
 from alibuild_helpers.log import dieOnError, debug, error
+from alibuild_helpers.utilities import call_ignoring_oserrors
 
 FETCH_LOG_NAME = "fetch-log.txt"
 
@@ -92,18 +93,15 @@ def updateReferenceRepo(referenceSources, p, spec,
   @fetch            : whether to fetch updates: if False, only clone if not found
   """
   assert isinstance(spec, OrderedDict)
-  if "source" not in spec:
-    return
+  if spec["is_devel_pkg"] or "source" not in spec:
+    return None
 
   scm = spec["scm"]
 
   debug("Updating references.")
   referenceRepo = os.path.join(os.path.abspath(referenceSources), p.lower())
 
-  try:
-    os.makedirs(os.path.abspath(referenceSources))
-  except:
-    pass
+  call_ignoring_oserrors(os.makedirs, os.path.abspath(referenceSources), exist_ok=True)
 
   if not is_writeable(referenceSources):
     if os.path.exists(referenceRepo):

tests/test_workarea.py

@@ -17,6 +17,7 @@
     ("package", "AliRoot"),
     ("source", "https://github.com/alisw/AliRoot"),
     ("scm", Git()),
+    ("is_devel_pkg", False),
 ))
 
 
@@ -39,7 +40,7 @@ def test_reference_sources_reused(self, mock_git, mock_makedirs, mock_exists):
         updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot",
                                 spec=spec, fetch=True)
         mock_exists.assert_called_with("%s/sw/MIRROR/aliroot" % getcwd())
-        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd())
+        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True)
         mock_git.assert_not_called()
         self.assertEqual(spec.get("reference"), "%s/sw/MIRROR/aliroot" % getcwd())
 
@@ -60,7 +61,7 @@ def test_reference_sources_updated(self, mock_git, mock_open, mock_makedirs, moc
                                 spec=spec, fetch=True)
         mock_exists.assert_called_with("%s/sw/MIRROR/aliroot" % getcwd())
         mock_exists.assert_has_calls([])
-        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd())
+        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True)
         mock_git.assert_called_once_with([
             "fetch", "-f", "--tags", spec["source"], "+refs/heads/*:refs/heads/*",
         ], directory="%s/sw/MIRROR/aliroot" % getcwd(), check=False, prompt=True)
@@ -77,7 +78,7 @@ def test_reference_sources_not_writable(self, mock_git, mock_makedirs, mock_exis
         updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot",
                                 spec=spec, fetch=True)
         mock_exists.assert_called_with("%s/sw/MIRROR/aliroot" % getcwd())
-        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd())
+        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True)
         mock_git.assert_not_called()
         self.assertNotIn("reference", spec,
                          "should delete spec['reference'], as no mirror exists")
@@ -94,7 +95,7 @@ def test_reference_sources_created(self, mock_git, mock_makedirs, mock_exists):
         updateReferenceRepoSpec(referenceSources="sw/MIRROR", p="AliRoot",
                                 spec=spec, fetch=True)
         mock_exists.assert_called_with("%s/sw/MIRROR/aliroot" % getcwd())
-        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd())
+        mock_makedirs.assert_called_with("%s/sw/MIRROR" % getcwd(), exist_ok=True)
         mock_git.assert_called_once_with([
             "clone", "--bare", spec["source"],
             "%s/sw/MIRROR/aliroot" % getcwd(), "--filter=blob:none",