Add retry logic

README.md

@@ -2,15 +2,18 @@
 
 [![build](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml/badge.svg)](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml)
 
-`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another. For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)).
+`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another.
+For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)).
+Synchronization between all source-destination repository pairs in performed concurrently.
 
 `git-synchronizer` will:
 * push all branches and tags from source to destination repository,
 * remove branches and tags from the destination repository which are no longer present in source repository.
 
 ## Installing
 
-Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page. `git-synchronizer` is distributed as a single binary file and does not require any additional system requirements.
+Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page.
+`git-synchronizer` is distributed as a single binary file and does not require any additional system requirements.
 
 ## Usage
 
@@ -35,12 +38,14 @@ defaults:
   source:
     auth:
       method: token
-      # Name of environment variable storing the Personal Access Token with permissions to read source repositories.
+      # Name of environment variable storing the Personal Access Token
+      # with permissions to read source repositories.
       token_name: GITHUB_TOKEN
   destination:
     auth:
       method: token
-      # Name of environment variable storing the Personal Access Token with permissions to push to destination repositories.
+      # Name of environment variable storing the Personal Access Token
+      # with permissions to push to destination repositories.
       token_name: GITLAB_TOKEN
 
 # List of repository pairs to be synchronized.
@@ -91,7 +96,8 @@ This project is built with the [Go programming language](https://go.dev/).
 
 ### Development Environment
 
-It is recommended to use Go 1.21+ for developing this project. This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project.
+It is recommended to use Go 1.21+ for developing this project.
+This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project.
 
 ### Common Commands
 

cmd/mirror.go

@@ -24,6 +24,8 @@ import (
 
 	git "github.com/go-git/go-git/v5"
 	gitconfig "github.com/go-git/go-git/v5/config"
+	gitplumbing "github.com/go-git/go-git/v5/plumbing"
+	gittransport "github.com/go-git/go-git/v5/plumbing/transport"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
@@ -89,11 +91,23 @@ func ValidateRepositories(repositories []RepositoryPair) {
 func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listOptions *git.ListOptions) ([]string, []string, error) {
 	var branchList []string
 	var tagList []string
+	var err error
+
 	remote, err := repository.Remote(remoteName)
 	if err != nil {
 		return branchList, tagList, err
 	}
-	refList, err := remote.List(listOptions)
+
+	var refList []*gitplumbing.Reference
+	for i := 1; i <= maxRetries; i++ {
+		refList, err = remote.List(listOptions)
+		if err == nil || err == gittransport.ErrAuthenticationRequired {
+			break
+		}
+		if i < maxRetries {
+			log.Warn("Retrying listing remote refs...")
+		}
+	}
 	if err != nil {
 		return branchList, tagList, err
 	}
@@ -207,6 +221,58 @@ func GetDestinationAuth(destAuth Authentication) *githttp.BasicAuth {
 	return destinationAuth
 }
 
+// GitPlainClone clones git repository and retries the process in case of failure.
+func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions) (*git.Repository, error) {
+	var repository *git.Repository
+	var err error
+	for i := 1; i <= maxRetries; i++ {
+		repository, err = git.PlainClone(gitDirectory, false, cloneOptions)
+		if err == nil || err == gittransport.ErrAuthenticationRequired {
+			break
+		}
+		if i < maxRetries {
+			log.Warn("Retrying repository cloning...")
+		}
+	}
+	return repository, err
+}
+
+// GitFetchBranches fetches all branches and retries the process in case of failure.
+func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication) error {
+	gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication)
+	var err error
+	for i := 1; i <= maxRetries; i++ {
+		err = sourceRemote.Fetch(gitFetchOptions)
+		if err == nil || err == gittransport.ErrAuthenticationRequired {
+			break
+		}
+		if i < maxRetries {
+			log.Warn("Retrying fetching branches...")
+		}
+	}
+	return err
+}
+
+// PushRefs pushes refs defined in refSpecString to destination remote and retries the process
+// in case of failure.
+func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString string) error {
+	var err error
+	for i := 1; i <= maxRetries; i++ {
+		err = repository.Push(&git.PushOptions{
+			RemoteName: "destination",
+			RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(refSpecString)},
+			Auth:       auth, Force: true, Atomic: true},
+		)
+		if err == nil || err == gittransport.ErrAuthenticationRequired || err == git.NoErrAlreadyUpToDate {
+			break
+		}
+		if i < maxRetries {
+			log.Warn("Retrying pushing refs...")
+		}
+	}
+	return err
+}
+
 // MirrorRepository mirrors branches and tags from source to destination. Tags and branches
 // no longer present in source are removed from destination.
 func MirrorRepository(messages chan MirrorStatus, source, destination string, sourceAuthentication, destinationAuthentication Authentication) {
@@ -217,13 +283,12 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 	defer os.RemoveAll(gitDirectory)
 	var allErrors []string
 	gitCloneOptions := GetCloneOptions(source, sourceAuthentication)
-	repository, err := git.PlainClone(gitDirectory, false, gitCloneOptions)
+	repository, err := GitPlainClone(gitDirectory, gitCloneOptions)
 	if err != nil {
 		ProcessError(err, "cloning repository from ", source, &allErrors)
 		messages <- MirrorStatus{allErrors, time.Now(), 0, 0}
 		return
 	}
-
 	gitListOptions := GetListOptions(sourceAuthentication)
 	sourceBranchList, sourceTagList, err := GetBranchesAndTagsFromRemote(repository, "origin", gitListOptions)
 	if err != nil {
@@ -242,8 +307,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 		return
 	}
 
-	gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication)
-	err = sourceRemote.Fetch(gitFetchOptions)
+	err = GitFetchBranches(sourceRemote, sourceAuthentication)
 	if err != nil {
 		ProcessError(err, "fetching branches from ", source, &allErrors)
 		messages <- MirrorStatus{allErrors, time.Now(), 0, 0}
@@ -275,40 +339,28 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so
 	log.Info("Pushing all branches from ", source, " to ", destination)
 	for _, branch := range sourceBranchList {
 		log.Debug("Pushing branch ", branch, " to ", destination)
-		err = repository.Push(&git.PushOptions{
-			RemoteName: "destination",
-			RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec("+" + refBranchPrefix + branch + ":" + refBranchPrefix + branch)},
-			Auth:       destinationAuth, Force: true, Atomic: true})
+		err = PushRefs(repository, destinationAuth, "+"+refBranchPrefix+branch+":"+refBranchPrefix+branch)
 		ProcessError(err, "pushing branch "+branch+" to ", destination, &allErrors)
 	}
 
 	// Remove any branches not present in the source repository anymore.
 	for _, branch := range destinationBranchList {
 		if !stringInSlice(branch, sourceBranchList) {
 			log.Info("Removing branch ", branch, " from ", destination)
-			err = repository.Push(&git.PushOptions{
-				RemoteName: "destination",
-				RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(":" + refBranchPrefix + branch)},
-				Auth:       destinationAuth, Force: true, Atomic: true})
+			err = PushRefs(repository, destinationAuth, ":"+refBranchPrefix+branch)
 			ProcessError(err, "removing branch "+branch+" from ", destination, &allErrors)
 		}
 	}
 
 	log.Info("Pushing all tags from ", source, " to ", destination)
-	err = repository.Push(&git.PushOptions{
-		RemoteName: "destination",
-		RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec("+" + refTagPrefix + "*:" + refTagPrefix + "*")},
-		Auth:       destinationAuth, Force: true, Atomic: true})
+	err = PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*")
 	ProcessError(err, "pushing all tags to ", destination, &allErrors)
 
 	// Remove any tags not present in the source repository anymore.
 	for _, tag := range destinationTagList {
 		if !stringInSlice(tag, sourceTagList) {
 			log.Info("Removing tag ", tag, " from ", destination)
-			err := repository.Push(&git.PushOptions{
-				RemoteName: "destination",
-				RefSpecs:   []gitconfig.RefSpec{gitconfig.RefSpec(":" + refTagPrefix + tag)},
-				Auth:       destinationAuth, Force: true, Atomic: true})
+			err = PushRefs(repository, destinationAuth, ":"+refTagPrefix+tag)
 			ProcessError(err, "removing tag "+tag+" from ", destination, &allErrors)
 		}
 	}

cmd/root.go

@@ -31,6 +31,7 @@ import (
 var cfgFile string
 var logLevel string
 var workingDirectory string
+var maxRetries int
 
 type RepositoryPair struct {
 	Source      Repository `mapstructure:"source"`
@@ -49,7 +50,6 @@ type Authentication struct {
 
 // Repository list provided in YAML configuration file.
 var inputRepositories []RepositoryPair
-
 var defaultSettings RepositoryPair
 
 var localTempDirectory string
@@ -122,6 +122,10 @@ func newRootCommand() {
 		"Logging level (trace, debug, info, warn, error). ")
 	rootCmd.PersistentFlags().StringVarP(&workingDirectory, "workingDirectory", "w", "/tmp/git-synchronizer",
 		"Directory where synchronized repositories will be cloned.")
+	rootCmd.PersistentFlags().IntVarP(&maxRetries, "maxRetries", "r", 3,
+		"Maximum number of retries in case a failure happens while: cloning source repository, "+
+			"getting branches and tags from remote, fetching branches from source repository, "+
+			"or pushing refs to destination repository.")
 
 	// Add version command.
 	rootCmd.AddCommand(extension.NewVersionCobraCmd())