Skip to content

Commit

Permalink
feat: Add support for gitlab, azure dev and bitbucket
Browse files Browse the repository at this point in the history
  • Loading branch information
NoUseFreak committed Feb 2, 2024
1 parent 8d64e45 commit 9471226
Show file tree
Hide file tree
Showing 9 changed files with 1,898 additions and 97 deletions.
16 changes: 14 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@ require (
github.com/erikgeiser/promptkit v0.9.0
github.com/google/go-github/v57 v57.0.0
github.com/ktr0731/go-fuzzyfinder v0.8.0
github.com/ktrysmt/go-bitbucket v0.9.74
github.com/microsoft/azure-devops-go-api/azuredevops/v7 v7.1.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/whilp/git-urls v1.0.0
github.com/xanzy/go-gitlab v0.96.0
golang.org/x/term v0.16.0
)

require (
Expand All @@ -23,7 +27,11 @@ require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gdamore/tcell/v2 v2.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ktr0731/go-ansisgr v0.1.0 // indirect
Expand Down Expand Up @@ -51,10 +59,14 @@ require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.20.0 // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
1,552 changes: 1,546 additions & 6 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/pkg/command/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func cloneRepo(repo string) (string, error) {
}
}

fmt.Fprintf(CmdOutput, "(echo \"\\033[0;32m*\\033[0m Cloning %s into %s\" && git clone %s %s); \n", gitURL, targetDir, gitURL, targetDir)
fmt.Fprintf(CmdOutput, "(echo \"\\033[0;32m*\\033[0m Cloning %s into %s\" && git clone -q %s %s); \n", gitURL, targetDir, gitURL, targetDir)

return targetDir, nil
}
Expand Down
141 changes: 53 additions & 88 deletions internal/pkg/command/org.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package command

import (
"context"
"errors"
"fmt"
"os"
"strings"

"github.com/google/go-github/v57/github"
"github.com/nousefreak/projecthelper/internal/pkg/org"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand All @@ -18,6 +17,21 @@ func getOrgCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "org ORANIZATION_URL",
Short: "Clone repositories from an organization into the project structure",
Long: `Clone repositories from an organization into the project structure.
The ORGANIZATION_URL is the URL of the organization on the platform. For example:
- github.com/nousefreak
- gitlab.com/nousefreak
- dev.azure.com/nousefreak
- bitbucket.org/nousefreak
In addidtion to the ORGANIZATION_URL you should also specify credentials.
The credentials are read from the environment variables:
- GITHUB_TOKEN
- GITLAB_TOKEN
- AZURE_TOKEN
- BITBUCKET_USERNAME, BITBUCKET_PASSWORD
`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
parts := strings.Split(args[0], "/")
Expand All @@ -26,103 +40,54 @@ func getOrgCmd() *cobra.Command {
logrus.Fatal("Invalid repo")
}

var provider org.OrgProvider
switch parts[0] {
case "github.com":
if err := cloneGithubOrg(parts[1]); err != nil {
logrus.Fatal(err)
provider = &org.GithubProvider{
Org: parts[1],
}

case "gitlab.com":
provider = &org.GitlabProvider{
User: strings.Join(parts[1:], "/"),
}
case "dev.azure.com":
provider = &org.AzureProvider{
Org: parts[1],
}
case "bitbucket.org":
provider = &org.BitbucketProvider{
Org: parts[1],
}
default:
logrus.Fatal("Unsupported platform")
}

},
}

cmd.Flags().BoolVarP(&cloneForks, "forks", "f", false, "Clone forks")

return cmd
}
repos, found, err := provider.GetRepos()
if err != nil {
logrus.Fatal(fmt.Errorf("failed to get repos: %w", err))
}

func getRepoFunc(org string) func() ([]*github.Repository, bool, error) {
client := github.NewClient(nil)
if os.Getenv("GITHUB_TOKEN") == "" {
opts := &github.RepositoryListByUserOptions{
Type: "all",
ListOptions: github.ListOptions{
PerPage: 100,
},
}
return func() ([]*github.Repository, bool, error) {
repos, resp, err := client.Repositories.ListByUser(context.Background(), org, opts)
opts.Page = resp.NextPage
return repos, resp.NextPage == 0, err
}
}
if !found {
logrus.Warn("No repos found")
}

logrus.Info("Using GITHUB_TOKEN")
client = client.WithAuthToken(os.Getenv("GITHUB_TOKEN"))
opts := &github.RepositoryListByAuthenticatedUserOptions{
Type: "all",
Sort: "updated",
ListOptions: github.ListOptions{
PerPage: 100,
logrus.Infof("Found %d repos", len(repos))

for _, repo := range repos {
if _, err := cloneRepo(repo.SSHURL); err != nil {
if err == ErrDirectoryAlreadyExists || errors.Unwrap(err) == ErrDirectoryAlreadyExists {
logrus.Debugf("Skipping existing repo: %s", repo.Name)
} else {
logrus.Warn(err)
}
} else {
fmt.Fprint(os.Stdout, " ; ")
}
}
},
}
return func() ([]*github.Repository, bool, error) {
repos, resp, err := client.Repositories.ListByAuthenticatedUser(context.Background(), opts)
opts.Page = resp.NextPage
return repos, resp.NextPage == 0, err
}
}

func cloneGithubOrg(org string) error {
logrus.Infof("Cloning repos from github.com/%s", org)

logrus.Info("Fetch repos")
repos := []*github.Repository{}

repoFunc := getRepoFunc(org)
for {
reposPage, done, err := repoFunc()
if err != nil {
return fmt.Errorf("Error fetching repos: %w", err)
}
repos = append(repos, reposPage...)

if done {
break
}
}

orgRepos := []*github.Repository{}
for _, repo := range repos {
if strings.EqualFold(repo.Owner.GetLogin(), org) {
orgRepos = append(orgRepos, repo)
}
}

logrus.Infof("Found %d/%d repos", len(orgRepos), len(repos))

for _, repo := range orgRepos {
if *repo.Archived {
logrus.Debugf("Skipping archived repo: %s", *repo.Name)
continue
}
if cloneForks && *repo.Fork {
logrus.Debugf("Skipping fork: %s", *repo.Name)
continue
}
if _, err := cloneRepo(repo.GetSSHURL()); err != nil {
if err == ErrDirectoryAlreadyExists || errors.Unwrap(err) == ErrDirectoryAlreadyExists {
logrus.Debugf("Skipping existing repo: %s", *repo.Name)
} else {
logrus.Warn(err)
}
} else {
fmt.Fprint(os.Stdout, " ; ")
}
}
cmd.Flags().BoolVarP(&cloneForks, "forks", "f", false, "Clone forks")

return nil
return cmd
}
86 changes: 86 additions & 0 deletions internal/pkg/org/azure.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package org

import (
"context"
"fmt"
"os"
"strconv"

"github.com/microsoft/azure-devops-go-api/azuredevops/v7"
"github.com/microsoft/azure-devops-go-api/azuredevops/v7/core"
"github.com/microsoft/azure-devops-go-api/azuredevops/v7/git"
)

type AzureProvider struct {
Org string
}

func (a *AzureProvider) GetRepos() ([]*Repo, bool, error) {

connection := azuredevops.NewPatConnection("https://dev.azure.com/"+a.Org, os.Getenv("AZURE_TOKEN"))
ctx := context.Background()
gitClient, err := git.NewClient(ctx, connection)
if err != nil {
return nil, false, fmt.Errorf("failed to create core client: %w", err)
}
coreClient, err := core.NewClient(ctx, connection)
if err != nil {
return nil, false, fmt.Errorf("failed to create core client: %w", err)
}

projects, err := getAzureProjects(ctx, coreClient)
if err != nil {
return nil, false, fmt.Errorf("failed to list projects: %w", err)
}

var repos []*Repo
for _, project := range projects {
r, err := gitClient.GetRepositories(ctx, git.GetRepositoriesArgs{
Project: &project,
})
if err != nil {
return nil, false, fmt.Errorf("failed to list repos: %w", err)
}

for _, repo := range *r {
repos = append(repos, &Repo{
Name: *repo.Name,
URL: *repo.RemoteUrl,
CloneURL: *repo.RemoteUrl,
SSHURL: *repo.SshUrl,
})
}
}

return repos, false, nil
}

func getAzureProjects(ctx context.Context, client core.Client) ([]string, error) {
resp, err := client.GetProjects(ctx, core.GetProjectsArgs{})
if err != nil {
return nil, fmt.Errorf("failed to list projects: %w", err)
}

var projects []string
if resp != nil {
for _, project := range (*resp).Value {
projects = append(projects, *project.Name)
}
if resp.ContinuationToken != "" {
ct, err := strconv.Atoi(resp.ContinuationToken)
if err != nil {
return nil, fmt.Errorf("failed to parse continuation token: %w", err)
}
resp, err = client.GetProjects(ctx, core.GetProjectsArgs{
ContinuationToken: &ct,
})
if err != nil {
return nil, fmt.Errorf("failed to list projects: %w", err)
}
} else {
resp = nil
}
}

return projects, nil
}
57 changes: 57 additions & 0 deletions internal/pkg/org/bitbucket.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package org

import (
"fmt"
"os"

"github.com/ktrysmt/go-bitbucket"
)

type BitbucketProvider struct {
Org string
}

func (b *BitbucketProvider) GetRepos() ([]*Repo, bool, error) {
client := bitbucket.NewBasicAuth(os.Getenv("BITBUCKET_USERNAME"), os.Getenv("BITBUCKET_PASSWORD"))
repos, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{
Owner: b.Org,
})
if err != nil {
return nil, false, fmt.Errorf("failed to list repositories: %w", err)
}

var result []*Repo
for _, repo := range repos.Items {
result = append(result, &Repo{
Name: repo.Name,
URL: getBitbucketLink(repo.Links, "html"),
CloneURL: getBitbucketLink(repo.Links, "clone", "https"),
SSHURL: getBitbucketLink(repo.Links, "clone", "ssh"),
})
}

return result, false, nil
}

func getBitbucketLink(links map[string]interface{}, p ...string) string {
for name, link := range links {
if name != p[0] {
continue
}
if l, ok := link.(map[string]interface{}); ok {
return l["href"].(string)
}
if l, ok := link.([]interface{}); ok {
for _, ll := range l {
if llm, ok := ll.(map[string]interface{}); ok {
if llm["name"].(string) != p[1] {
continue
}
return llm["href"].(string)
}
}
}
}

return ""
}
Loading

0 comments on commit 9471226

Please sign in to comment.