diff --git a/alibuild_helpers/sync.py b/alibuild_helpers/sync.py
index 7d295463..466f9263 100644
--- a/alibuild_helpers/sync.py
+++ b/alibuild_helpers/sync.py
@@ -186,6 +186,15 @@ def fetch_symlinks(self, spec):
     links_path = resolve_links_path(self.architecture, spec["package"])
     os.makedirs(os.path.join(self.workdir, links_path), exist_ok=True)
 
+    # If we already have a symlink we can use, don't update the list. This
+    # speeds up rebuilds significantly.
+    if any(f"/{pkg_hash[:2]}/{pkg_hash}/" in target
+           for target in (os.readlink(os.path.join(self.workdir, links_path, link))
+                          for link in os.listdir(os.path.join(self.workdir, links_path)))
+           for pkg_hash in spec["remote_hashes"]):
+      debug("Found symlink for %s@%s, not updating", spec["package"], spec["version"])
+      return
+
     with requests.Session() as session:
       # Fetch manifest file with initial symlinks. This file is updated
       # regularly; we use it to avoid many small network requests.
diff --git a/tests/test_sync.py b/tests/test_sync.py
index de3d730e..62acdfb8 100644
--- a/tests/test_sync.py
+++ b/tests/test_sync.py
@@ -90,6 +90,7 @@ def mock_get(self, url, *args, **kw):
     @patch("os.path.isfile", new=MagicMock(return_value=False))
     @patch("os.rename", new=MagicMock(return_value=None))
     @patch("os.makedirs", new=MagicMock(return_value=None))
+    @patch("os.listdir", new=MagicMock(return_value=[]))
     @patch("alibuild_helpers.sync.symlink", new=MagicMock(return_value=None))
     @patch("alibuild_helpers.sync.execute", new=MagicMock(return_value=None))
     @patch("alibuild_helpers.sync.debug")