Skip to content

Commit

Permalink
feat: ✨ make sync operation retryable with delay
Browse files Browse the repository at this point in the history
  • Loading branch information
@AkashRajpurohit
AkashRajpurohit committed Dec 22, 2024
1 parent 5075c62 commit 9ebbaf6
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 36 deletions.
45 changes: 28 additions & 17 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,30 @@ type Server struct {
Protocol string `mapstructure:"protocol"`
}

type RetryConfig struct {
Count int `mapstructure:"count"`
Delay int `mapstructure:"delay"` // in seconds
}

type Config struct {
Username string `mapstructure:"username"`
Token string `mapstructure:"token"` // Deprecated: Use Tokens instead
Tokens []string `mapstructure:"tokens"` // New field for multiple tokens
Platform string `mapstructure:"platform"`
Server Server `mapstructure:"server"`
IncludeRepos []string `mapstructure:"include_repos"`
ExcludeRepos []string `mapstructure:"exclude_repos"`
IncludeOrgs []string `mapstructure:"include_orgs"`
ExcludeOrgs []string `mapstructure:"exclude_orgs"`
IncludeForks bool `mapstructure:"include_forks"`
IncludeWiki bool `mapstructure:"include_wiki"`
BackupDir string `mapstructure:"backup_dir"`
Workspace string `mapstructure:"workspace"`
Cron string `mapstructure:"cron"`
CloneType string `mapstructure:"clone_type"`
RawGitURLs []string `mapstructure:"raw_git_urls"`
Concurrency int `mapstructure:"concurrency"`
Username string `mapstructure:"username"`
Token string `mapstructure:"token"` // Deprecated: Use Tokens instead
Tokens []string `mapstructure:"tokens"` // New field for multiple tokens
Platform string `mapstructure:"platform"`
Server Server `mapstructure:"server"`
IncludeRepos []string `mapstructure:"include_repos"`
ExcludeRepos []string `mapstructure:"exclude_repos"`
IncludeOrgs []string `mapstructure:"include_orgs"`
ExcludeOrgs []string `mapstructure:"exclude_orgs"`
IncludeForks bool `mapstructure:"include_forks"`
IncludeWiki bool `mapstructure:"include_wiki"`
BackupDir string `mapstructure:"backup_dir"`
Workspace string `mapstructure:"workspace"`
Cron string `mapstructure:"cron"`
CloneType string `mapstructure:"clone_type"`
RawGitURLs []string `mapstructure:"raw_git_urls"`
Concurrency int `mapstructure:"concurrency"`
Retry RetryConfig `mapstructure:"retry"`
}

// PreprocessConfig handles backward compatibility for token field
Expand Down Expand Up @@ -148,6 +154,7 @@ func SaveConfig(config Config, cfgFile string) error {
viper.Set("clone_type", config.CloneType)
viper.Set("raw_git_urls", config.RawGitURLs)
viper.Set("concurrency", config.Concurrency)
viper.Set("retry", config.Retry)

return viper.WriteConfig()
}
Expand All @@ -173,5 +180,9 @@ func GetInitialConfig() Config {
CloneType: "bare",
RawGitURLs: []string{},
Concurrency: 5,
Retry: RetryConfig{
Count: 3,
Delay: 5,
},
}
}
37 changes: 37 additions & 0 deletions pkg/sync/retry.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package sync

import (
"fmt"
"time"

"github.com/AkashRajpurohit/git-sync/pkg/config"
"github.com/AkashRajpurohit/git-sync/pkg/logger"
)

func retryOperation(cfg config.Config, operation func() error, operationName string) error {
var lastErr error

// If retry count is 0 or negative, just execute once without retries
if cfg.Retry.Count <= 0 {
return operation()
}

for attempt := 1; attempt <= cfg.Retry.Count; attempt++ {
err := operation()
if err == nil {
if attempt > 1 {
logger.Warnf("Operation %s succeeded after %d attempts", operationName, attempt)
}
return nil
}

lastErr = err
if attempt < cfg.Retry.Count {
logger.Warnf("Attempt %d/%d failed for %s: %v. Retrying in %d seconds...",
attempt, cfg.Retry.Count, operationName, err, cfg.Retry.Delay)
time.Sleep(time.Duration(cfg.Retry.Delay) * time.Second)
}
}

return fmt.Errorf("operation failed after %d attempts: %v", cfg.Retry.Count, lastErr)
}
68 changes: 49 additions & 19 deletions pkg/sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,12 @@ func CloneOrUpdateRepo(repoOwner, repoName string, config config.Config) {
if _, err := os.Stat(repoPath); os.IsNotExist(err) {
logger.Info("Cloning repo: ", repoFullName)
command := getGitCloneCommand(config.CloneType, repoPath, repoURL)
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
return err
}, fmt.Sprintf("clone %s", repoFullName))

if err != nil {
logger.Errorf("Failed to clone repo %s: %v", repoFullName, err)
Expand All @@ -88,8 +92,12 @@ func CloneOrUpdateRepo(repoOwner, repoName string, config config.Config) {
} else {
logger.Info("Updating repo: ", repoFullName)
command := getGitFetchCommand(config.CloneType, repoPath, repoURL)
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
return err
}, fmt.Sprintf("update %s", repoFullName))

if err != nil {
logger.Errorf("Failed to update repo %s: %v", repoFullName, err)
Expand All @@ -108,8 +116,12 @@ func CloneOrUpdateRawRepo(repoOwner, repoName, repoURL string, config config.Con
if _, err := os.Stat(repoPath); os.IsNotExist(err) {
logger.Info("Cloning raw repo: ", repoURL)
command := getGitCloneCommand(config.CloneType, repoPath, repoURL)
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
return err
}, fmt.Sprintf("clone %s", repoURL))

if err != nil {
logger.Errorf("Failed to clone raw repo %s: %v", repoURL, err)
Expand All @@ -122,8 +134,12 @@ func CloneOrUpdateRawRepo(repoOwner, repoName, repoURL string, config config.Con
} else {
logger.Info("Updating raw repo: ", repoURL)
command := getGitFetchCommand(config.CloneType, repoPath, repoURL)
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
return err
}, fmt.Sprintf("update %s", repoURL))

if err != nil {
logger.Errorf("Failed to update raw repo %s: %v", repoURL, err)
Expand Down Expand Up @@ -154,26 +170,40 @@ func SyncWiki(repoOwner, repoName string, config config.Config) {
if _, err := os.Stat(repoWikiPath); os.IsNotExist(err) {
logger.Info("Cloning wiki: ", repoFullName)
command := exec.Command("git", "clone", repoWikiURL, repoWikiPath)
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

if err != nil {
if strings.Contains(string(output), "not found") {
logger.Warnf("The wiki for repository %s does not exist. Please check your repository settings and make sure that either wiki is disabled if it is not being used or create a wiki page to start with.", repoFullName)
return
wikiNotFound := false

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
if err != nil && strings.Contains(string(output), "not found") {
wikiNotFound = true
// Don't retry for non-existent wikis
return nil
}
return err
}, fmt.Sprintf("clone wiki %s", repoFullName))

if err != nil && !wikiNotFound {
logger.Errorf("Failed to clone wiki %s: %v", repoFullName, err)
recordWikiFailure(repoFullName, err)
return
}

logger.Info("Cloned wiki: ", repoFullName)
recordWikiSuccess()
if wikiNotFound {
logger.Warnf("The wiki for repository %s does not exist. Please check your repository settings and make sure that either wiki is disabled if it is not being used or create a wiki page to start with.", repoFullName)
} else {
logger.Info("Cloned wiki: ", repoFullName)
recordWikiSuccess()
}
} else {
logger.Info("Updating wiki: ", repoFullName)
command := exec.Command("git", "-C", repoWikiPath, "pull", "--prune", "origin")
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)

err := retryOperation(config, func() error {
output, err := command.CombinedOutput()
logger.Debugf("Output: %s\n", output)
return err
}, fmt.Sprintf("update wiki %s", repoFullName))

if err != nil {
logger.Errorf("Failed to update wiki %s: %v", repoFullName, err)
Expand Down

0 comments on commit 9ebbaf6

Please sign in to comment.