Add retry logic

README.md

@@ -2,15 +2,18 @@
 
 [![build](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml/badge.svg)](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml)
 
-`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another. For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)).
+`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another.
+For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)).
+Synchronization between all source-destination repository pairs in performed concurrently.
 
 `git-synchronizer` will:
 * push all branches and tags from source to destination repository,
 * remove branches and tags from the destination repository which are no longer present in source repository.
 
 ## Installing
 
-Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page. `git-synchronizer` is distributed as a single binary file and does not require any additional system requirements.
+Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page.
+`git-synchronizer` is distributed as a single binary file and does not require any additional system requirements.
 
 ## Usage
 
@@ -35,12 +38,14 @@ defaults:
   source:
     auth:
       method: token
-      # Name of environment variable storing the Personal Access Token with permissions to read source repositories.
+      # Name of environment variable storing the Personal Access Token
+      # with permissions to read source repositories.
       token_name: GITHUB_TOKEN
   destination:
     auth:
       method: token
-      # Name of environment variable storing the Personal Access Token with permissions to push to destination repositories.
+      # Name of environment variable storing the Personal Access Token
+      # with permissions to push to destination repositories.
       token_name: GITLAB_TOKEN
 
 # List of repository pairs to be synchronized.
@@ -91,7 +96,8 @@ This project is built with the [Go programming language](https://go.dev/).
 
 ### Development Environment
 
-It is recommended to use Go 1.21+ for developing this project. This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project.
+It is recommended to use Go 1.21+ for developing this project.
+This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project.
 
 ### Common Commands
 

cmd/mirror.go

@@ -22,8 +22,11 @@ import (
 	"strings"
 	"time"
 
+	backoff "github.com/cenkalti/backoff/v4"
 	git "github.com/go-git/go-git/v5"
 	gitconfig "github.com/go-git/go-git/v5/config"
+	gitplumbing "github.com/go-git/go-git/v5/plumbing"
+	gittransport "github.com/go-git/go-git/v5/plumbing/transport"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
@@ -85,15 +88,33 @@ func ValidateRepositories(repositories []RepositoryPair) {
 	}
 }
 
+func ListRemote(remote *git.Remote, listOptions *git.ListOptions) ([]*gitplumbing.Reference, error) {
+	refList, err := remote.List(listOptions)
+	if err == gittransport.ErrAuthenticationRequired {
+		return nil, backoff.Permanent(err)
+	} else if err != nil {
+		log.Warn("Retrying listing remote...")
+	}
+	return refList, err
+}
+
 // GetBranchesAndTagsFromRemote returns list of branches and tags present in remoteName of repository.
 func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listOptions *git.ListOptions) ([]string, []string, error) {
 	var branchList []string
 	var tagList []string
+	var err error
+
 	remote, err := repository.Remote(remoteName)
 	if err != nil {
 		return branchList, tagList, err
 	}
-	refList, err := remote.List(listOptions)
+
+	listRemoteBackoff := backoff.NewExponentialBackOff()
+	listRemoteBackoff.MaxElapsedTime = time.Minute
+	refList, err := backoff.RetryWithData(
+		func() ([]*gitplumbing.Reference, error) { return ListRemote(remote, listOptions) },
+		listRemoteBackoff,
+	)
 	if err != nil {
 		return branchList, tagList, err
 	}
@@ -207,6 +228,47 @@ func GetDestinationAuth(destAuth Authentication) *githttp.BasicAuth {
 	return destinationAuth
 }
 
+// GitPlainClone clones git repository and is retried in case of error.
+func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions) (*git.Repository, error) {
+	repository, err := git.PlainClone(gitDirectory, false, cloneOptions)
+	if err == gittransport.ErrAuthenticationRequired {
+		// Terminate backoff.
+		return nil, backoff.Permanent(err)
+	} else if err != nil {
+		log.Warn("Retrying cloning repository...")
+	}
+	return repository, err
+}
+
+// GitFetchBranches fetches all branches and is retried in case of error.
+func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication) error {
+	gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication)
+	err := sourceRemote.Fetch(gitFetchOptions)
+	if err == gittransport.ErrAuthenticationRequired {
+		// Terminate backoff.
+		return backoff.Permanent(err)
+	} else if err != nil {
+		log.Warn("Retrying fetching branches...")
+	}
+	return err
+}
+
+// PushRefs pushes refs defined in refSpecString to destination remote and is retried in case of error.
+func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString string) error {
+	err := repository.Push(&git.PushOptions{
+		RemoteName: "destination",
+		RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(refSpecString)},
+		Auth:       auth, Force: true, Atomic: true},
+	)
+	if err == gittransport.ErrAuthenticationRequired || err == git.NoErrAlreadyUpToDate {
+		// Terminate backoff.
+		return backoff.Permanent(err)
+	} else if err != nil {
+		log.Warn("Retrying pushing refs...")
+	}
+	return err
+}
+
 // MirrorRepository mirrors branches and tags from source to destination. Tags and branches
 // no longer present in source are removed from destination.
 func MirrorRepository(messages chan MirrorStatus, source, destination string, sourceAuthentication, destinationAuthentication Authentication) {
@@ -217,7 +279,13 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 	defer os.RemoveAll(gitDirectory)
 	var allErrors []string
 	gitCloneOptions := GetCloneOptions(source, sourceAuthentication)
-	repository, err := git.PlainClone(gitDirectory, false, gitCloneOptions)
+
+	cloneBackoff := backoff.NewExponentialBackOff()
+	cloneBackoff.MaxElapsedTime = 5 * time.Minute
+	repository, err := backoff.RetryWithData(
+		func() (*git.Repository, error) { return GitPlainClone(gitDirectory, gitCloneOptions) },
+		cloneBackoff,
+	)
 	if err != nil {
 		ProcessError(err, "cloning repository from ", source, &allErrors)
 		messages <- MirrorStatus{allErrors, time.Now(), 0, 0}
@@ -242,8 +310,12 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 		return
 	}
 
-	gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication)
-	err = sourceRemote.Fetch(gitFetchOptions)
+	fetchBranchesBackoff := backoff.NewExponentialBackOff()
+	fetchBranchesBackoff.MaxElapsedTime = time.Minute
+	err = backoff.Retry(
+		func() error { return GitFetchBranches(sourceRemote, sourceAuthentication) },
+		fetchBranchesBackoff,
+	)
 	if err != nil {
 		ProcessError(err, "fetching branches from ", source, &allErrors)
 		messages <- MirrorStatus{allErrors, time.Now(), 0, 0}
@@ -275,40 +347,50 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 	log.Info("Pushing all branches from ", source, " to ", destination)
 	for _, branch := range sourceBranchList {
 		log.Debug("Pushing branch ", branch, " to ", destination)
-		err = repository.Push(&git.PushOptions{
-			RemoteName: "destination",
-			RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec("+" + refBranchPrefix + branch + ":" + refBranchPrefix + branch)},
-			Auth:       destinationAuth, Force: true, Atomic: true})
+		pushBranchesBackoff := backoff.NewExponentialBackOff()
+		pushBranchesBackoff.MaxElapsedTime = 3 * time.Minute
+		err = backoff.Retry(
+			func() error {
+				return PushRefs(repository, destinationAuth, "+"+refBranchPrefix+branch+":"+refBranchPrefix+branch)
+			},
+			pushBranchesBackoff,
+		)
 		ProcessError(err, "pushing branch "+branch+" to ", destination, &allErrors)
 	}
 
 	// Remove any branches not present in the source repository anymore.
 	for _, branch := range destinationBranchList {
 		if !stringInSlice(branch, sourceBranchList) {
 			log.Info("Removing branch ", branch, " from ", destination)
-			err = repository.Push(&git.PushOptions{
-				RemoteName: "destination",
-				RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(":" + refBranchPrefix + branch)},
-				Auth:       destinationAuth, Force: true, Atomic: true})
+			removeBranchesBackoff := backoff.NewExponentialBackOff()
+			removeBranchesBackoff.MaxElapsedTime = time.Minute
+			err = backoff.Retry(
+				func() error { return PushRefs(repository, destinationAuth, ":"+refBranchPrefix+branch) },
+				removeBranchesBackoff,
+			)
 			ProcessError(err, "removing branch "+branch+" from ", destination, &allErrors)
 		}
 	}
 
 	log.Info("Pushing all tags from ", source, " to ", destination)
-	err = repository.Push(&git.PushOptions{
-		RemoteName: "destination",
-		RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec("+" + refTagPrefix + "*:" + refTagPrefix + "*")},
-		Auth:       destinationAuth, Force: true, Atomic: true})
+	pushTagsBackoff := backoff.NewExponentialBackOff()
+	pushTagsBackoff.MaxElapsedTime = time.Minute
+	err = backoff.Retry(
+		func() error { return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*") },
+		pushTagsBackoff,
+	)
 	ProcessError(err, "pushing all tags to ", destination, &allErrors)
 
 	// Remove any tags not present in the source repository anymore.
 	for _, tag := range destinationTagList {
 		if !stringInSlice(tag, sourceTagList) {
 			log.Info("Removing tag ", tag, " from ", destination)
-			err := repository.Push(&git.PushOptions{
-				RemoteName: "destination",
-				RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(":" + refTagPrefix + tag)},
-				Auth:       destinationAuth, Force: true, Atomic: true})
+			removeTagsBackoff := backoff.NewExponentialBackOff()
+			removeTagsBackoff.MaxElapsedTime = time.Minute
+			err = backoff.Retry(
+				func() error { return PushRefs(repository, destinationAuth, ":"+refTagPrefix+tag) },
+				removeTagsBackoff,
+			)
 			ProcessError(err, "removing tag "+tag+" from ", destination, &allErrors)
 		}
 	}

cmd/root.go

@@ -49,7 +49,6 @@ type Authentication struct {
 
 // Repository list provided in YAML configuration file.
 var inputRepositories []RepositoryPair
-
 var defaultSettings RepositoryPair
 
 var localTempDirectory string

go.mod

@@ -5,6 +5,7 @@ go 1.21
 toolchain go1.21.6
 
 require (
+	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/jamiealquiza/envy v1.1.0
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -20,6 +20,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=