From 9ebbaf694c069904d3a5c9e0db9b74a601e8b308 Mon Sep 17 00:00:00 2001 From: AkashRajpurohit Date: Sun, 22 Dec 2024 15:39:03 +0530 Subject: [PATCH] feat: :sparkles: make sync operation retryable with delay --- pkg/config/config.go | 45 ++++++++++++++++++----------- pkg/sync/retry.go | 37 ++++++++++++++++++++++++ pkg/sync/sync.go | 68 +++++++++++++++++++++++++++++++------------- 3 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 pkg/sync/retry.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 08d4e63..4e37a90 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,24 +14,30 @@ type Server struct { Protocol string `mapstructure:"protocol"` } +type RetryConfig struct { + Count int `mapstructure:"count"` + Delay int `mapstructure:"delay"` // in seconds +} + type Config struct { - Username string `mapstructure:"username"` - Token string `mapstructure:"token"` // Deprecated: Use Tokens instead - Tokens []string `mapstructure:"tokens"` // New field for multiple tokens - Platform string `mapstructure:"platform"` - Server Server `mapstructure:"server"` - IncludeRepos []string `mapstructure:"include_repos"` - ExcludeRepos []string `mapstructure:"exclude_repos"` - IncludeOrgs []string `mapstructure:"include_orgs"` - ExcludeOrgs []string `mapstructure:"exclude_orgs"` - IncludeForks bool `mapstructure:"include_forks"` - IncludeWiki bool `mapstructure:"include_wiki"` - BackupDir string `mapstructure:"backup_dir"` - Workspace string `mapstructure:"workspace"` - Cron string `mapstructure:"cron"` - CloneType string `mapstructure:"clone_type"` - RawGitURLs []string `mapstructure:"raw_git_urls"` - Concurrency int `mapstructure:"concurrency"` + Username string `mapstructure:"username"` + Token string `mapstructure:"token"` // Deprecated: Use Tokens instead + Tokens []string `mapstructure:"tokens"` // New field for multiple tokens + Platform string `mapstructure:"platform"` + Server Server `mapstructure:"server"` + IncludeRepos []string `mapstructure:"include_repos"` + ExcludeRepos []string `mapstructure:"exclude_repos"` + IncludeOrgs []string `mapstructure:"include_orgs"` + ExcludeOrgs []string `mapstructure:"exclude_orgs"` + IncludeForks bool `mapstructure:"include_forks"` + IncludeWiki bool `mapstructure:"include_wiki"` + BackupDir string `mapstructure:"backup_dir"` + Workspace string `mapstructure:"workspace"` + Cron string `mapstructure:"cron"` + CloneType string `mapstructure:"clone_type"` + RawGitURLs []string `mapstructure:"raw_git_urls"` + Concurrency int `mapstructure:"concurrency"` + Retry RetryConfig `mapstructure:"retry"` } // PreprocessConfig handles backward compatibility for token field @@ -148,6 +154,7 @@ func SaveConfig(config Config, cfgFile string) error { viper.Set("clone_type", config.CloneType) viper.Set("raw_git_urls", config.RawGitURLs) viper.Set("concurrency", config.Concurrency) + viper.Set("retry", config.Retry) return viper.WriteConfig() } @@ -173,5 +180,9 @@ func GetInitialConfig() Config { CloneType: "bare", RawGitURLs: []string{}, Concurrency: 5, + Retry: RetryConfig{ + Count: 3, + Delay: 5, + }, } } diff --git a/pkg/sync/retry.go b/pkg/sync/retry.go new file mode 100644 index 0000000..8519a07 --- /dev/null +++ b/pkg/sync/retry.go @@ -0,0 +1,37 @@ +package sync + +import ( + "fmt" + "time" + + "github.com/AkashRajpurohit/git-sync/pkg/config" + "github.com/AkashRajpurohit/git-sync/pkg/logger" +) + +func retryOperation(cfg config.Config, operation func() error, operationName string) error { + var lastErr error + + // If retry count is 0 or negative, just execute once without retries + if cfg.Retry.Count <= 0 { + return operation() + } + + for attempt := 1; attempt <= cfg.Retry.Count; attempt++ { + err := operation() + if err == nil { + if attempt > 1 { + logger.Warnf("Operation %s succeeded after %d attempts", operationName, attempt) + } + return nil + } + + lastErr = err + if attempt < cfg.Retry.Count { + logger.Warnf("Attempt %d/%d failed for %s: %v. Retrying in %d seconds...", + attempt, cfg.Retry.Count, operationName, err, cfg.Retry.Delay) + time.Sleep(time.Duration(cfg.Retry.Delay) * time.Second) + } + } + + return fmt.Errorf("operation failed after %d attempts: %v", cfg.Retry.Count, lastErr) +} diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index 6fc472a..a9f213f 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -74,8 +74,12 @@ func CloneOrUpdateRepo(repoOwner, repoName string, config config.Config) { if _, err := os.Stat(repoPath); os.IsNotExist(err) { logger.Info("Cloning repo: ", repoFullName) command := getGitCloneCommand(config.CloneType, repoPath, repoURL) - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + return err + }, fmt.Sprintf("clone %s", repoFullName)) if err != nil { logger.Errorf("Failed to clone repo %s: %v", repoFullName, err) @@ -88,8 +92,12 @@ func CloneOrUpdateRepo(repoOwner, repoName string, config config.Config) { } else { logger.Info("Updating repo: ", repoFullName) command := getGitFetchCommand(config.CloneType, repoPath, repoURL) - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + return err + }, fmt.Sprintf("update %s", repoFullName)) if err != nil { logger.Errorf("Failed to update repo %s: %v", repoFullName, err) @@ -108,8 +116,12 @@ func CloneOrUpdateRawRepo(repoOwner, repoName, repoURL string, config config.Con if _, err := os.Stat(repoPath); os.IsNotExist(err) { logger.Info("Cloning raw repo: ", repoURL) command := getGitCloneCommand(config.CloneType, repoPath, repoURL) - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + return err + }, fmt.Sprintf("clone %s", repoURL)) if err != nil { logger.Errorf("Failed to clone raw repo %s: %v", repoURL, err) @@ -122,8 +134,12 @@ func CloneOrUpdateRawRepo(repoOwner, repoName, repoURL string, config config.Con } else { logger.Info("Updating raw repo: ", repoURL) command := getGitFetchCommand(config.CloneType, repoPath, repoURL) - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + return err + }, fmt.Sprintf("update %s", repoURL)) if err != nil { logger.Errorf("Failed to update raw repo %s: %v", repoURL, err) @@ -154,26 +170,40 @@ func SyncWiki(repoOwner, repoName string, config config.Config) { if _, err := os.Stat(repoWikiPath); os.IsNotExist(err) { logger.Info("Cloning wiki: ", repoFullName) command := exec.Command("git", "clone", repoWikiURL, repoWikiPath) - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) - - if err != nil { - if strings.Contains(string(output), "not found") { - logger.Warnf("The wiki for repository %s does not exist. Please check your repository settings and make sure that either wiki is disabled if it is not being used or create a wiki page to start with.", repoFullName) - return + wikiNotFound := false + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + if err != nil && strings.Contains(string(output), "not found") { + wikiNotFound = true + // Don't retry for non-existent wikis + return nil } + return err + }, fmt.Sprintf("clone wiki %s", repoFullName)) + + if err != nil && !wikiNotFound { logger.Errorf("Failed to clone wiki %s: %v", repoFullName, err) recordWikiFailure(repoFullName, err) return } - logger.Info("Cloned wiki: ", repoFullName) - recordWikiSuccess() + if wikiNotFound { + logger.Warnf("The wiki for repository %s does not exist. Please check your repository settings and make sure that either wiki is disabled if it is not being used or create a wiki page to start with.", repoFullName) + } else { + logger.Info("Cloned wiki: ", repoFullName) + recordWikiSuccess() + } } else { logger.Info("Updating wiki: ", repoFullName) command := exec.Command("git", "-C", repoWikiPath, "pull", "--prune", "origin") - output, err := command.CombinedOutput() - logger.Debugf("Output: %s\n", output) + + err := retryOperation(config, func() error { + output, err := command.CombinedOutput() + logger.Debugf("Output: %s\n", output) + return err + }, fmt.Sprintf("update wiki %s", repoFullName)) if err != nil { logger.Errorf("Failed to update wiki %s: %v", repoFullName, err)