From c0675048c491f1f4f013f5b5432c0ffed9ac8dc7 Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Sat, 11 Jan 2025 10:02:29 -0500 Subject: [PATCH] feat(github): add graphql client --- go.mod | 2 + go.sum | 4 ++ pkg/sources/github/connector.go | 63 ++++++++++++++++--- pkg/sources/github/connector_app.go | 15 ++++- pkg/sources/github/connector_basicauth.go | 29 ++++++--- pkg/sources/github/connector_token.go | 33 +++++++--- .../github/connector_unauthenticated.go | 21 +++++-- pkg/sources/github/github.go | 12 +--- 8 files changed, 138 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 95ed0397065b..227a3635d700 100644 --- a/go.mod +++ b/go.mod @@ -270,6 +270,8 @@ require ( github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 // indirect + github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.3.0 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect diff --git a/go.sum b/go.sum index 7bd253534729..11c72c36acc9 100644 --- a/go.sum +++ b/go.sum @@ -709,6 +709,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY= github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg= +github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= +github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index d0a64472138e..7a163f733b40 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -2,43 +2,90 @@ package github import ( "fmt" + "net/http" + "net/url" + "strings" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" - "github.com/trufflesecurity/trufflehog/v3/pkg/log" + "github.com/shurcooL/githubv4" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/log" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" ) -const cloudEndpoint = "https://api.github.com" +const ( + cloudV3Endpoint = "https://api.github.com" + cloudGraphqlEndpoint = "https://api.github.com/graphql" // https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#the-graphql-endpoint +) type connector interface { // APIClient returns a configured GitHub client that can be used for GitHub API operations. APIClient() *github.Client + // GraphQLClient returns a client that can be used for GraphQL operations. + GraphQLClient() *githubv4.Client // Clone clones a repository using the configured authentication information. Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) } -func newConnector(source *Source) (connector, error) { +func newConnector(ctx context.Context, source *Source) (connector, error) { + // Construct the URLs. apiEndpoint := source.conn.Endpoint if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) { - apiEndpoint = cloudEndpoint + apiEndpoint = cloudV3Endpoint } switch cred := source.conn.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: log.RedactGlobally(cred.GithubApp.GetPrivateKey()) - return newAppConnector(apiEndpoint, cred.GithubApp) + return newAppConnector(ctx, apiEndpoint, cred.GithubApp) case *sourcespb.GitHub_BasicAuth: log.RedactGlobally(cred.BasicAuth.GetPassword()) - return newBasicAuthConnector(apiEndpoint, cred.BasicAuth) + return newBasicAuthConnector(ctx, apiEndpoint, cred.BasicAuth) case *sourcespb.GitHub_Token: log.RedactGlobally(cred.Token) - return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit) + return newTokenConnector(ctx, apiEndpoint, cred.Token, source.handleRateLimit) case *sourcespb.GitHub_Unauthenticated: - return newUnauthenticatedConnector(apiEndpoint) + return newUnauthenticatedConnector(ctx, apiEndpoint) default: return nil, fmt.Errorf("unknown connection type") } } + +func createAPIClient(ctx context.Context, httpClient *http.Client, apiEndpoint string) (*github.Client, error) { + ctx.Logger().WithName("github").V(2). + Info("Creating API client", "url", apiEndpoint) + + // If we're using public GitHub, make a regular client. + // Otherwise, make an enterprise client. + if strings.EqualFold(apiEndpoint, cloudV3Endpoint) { + return github.NewClient(httpClient), nil + } + + return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) +} + +func createGraphqlClient(ctx context.Context, client *http.Client, apiEndpoint string) (*githubv4.Client, error) { + var graphqlEndpoint string + if apiEndpoint == cloudV3Endpoint { + graphqlEndpoint = cloudGraphqlEndpoint + } else { + // Use the root endpoint for the host. + // https://docs.github.com/en/enterprise-server@3.11/graphql/guides/introduction-to-graphql + parsedURL, err := url.Parse(apiEndpoint) + if err != nil { + return nil, fmt.Errorf("could not create GraphQL client: %w", err) + } + + // GitHub Enterprise uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958) + // Swap it, and anything before `/api`, with GraphQL. + before, _ := strings.CutSuffix(parsedURL.Path, "/api/v3") + parsedURL.Path = before + "/api/graphql" + graphqlEndpoint = parsedURL.String() + } + ctx.Logger().WithName("github").V(2). + Info("Creating GraphQL client", "url", graphqlEndpoint) + + return githubv4.NewEnterpriseClient(graphqlEndpoint, client), nil +} diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 859e2aa9ce04..49cfd882c810 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -7,6 +7,8 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" @@ -15,13 +17,14 @@ import ( type appConnector struct { apiClient *github.Client + graphqlClient *githubv4.Client installationClient *github.Client installationID int64 } var _ connector = (*appConnector)(nil) -func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { +func newAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) if err != nil { return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err) @@ -67,8 +70,14 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn return nil, fmt.Errorf("could not create API client: %w", err) } + graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return &appConnector{ apiClient: apiClient, + graphqlClient: graphqlClient, installationClient: installationClient, installationID: installationID, }, nil @@ -78,6 +87,10 @@ func (c *appConnector) APIClient() *github.Client { return c.apiClient } +func (c *appConnector) GraphQLClient() *githubv4.Client { + return c.graphqlClient +} + func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { // TODO: Check rate limit for this call. token, _, err := c.installationClient.Apps.CreateInstallationToken( diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 510b6160d23d..7aaaf4239e34 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -5,6 +5,8 @@ import ( gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" @@ -12,14 +14,15 @@ import ( ) type basicAuthConnector struct { - apiClient *github.Client - username string - password string + apiClient *github.Client + graphqlClient *githubv4.Client + username string + password string } var _ connector = (*basicAuthConnector)(nil) -func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { +func newBasicAuthConnector(ctx context.Context, apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { const httpTimeoutSeconds = 60 httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) httpClient.Transport = &github.BasicAuthTransport{ @@ -27,15 +30,21 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* Password: cred.Password, } - apiClient, err := createGitHubClient(httpClient, apiEndpoint) + apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint) if err != nil { return nil, fmt.Errorf("could not create API client: %w", err) } + graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return &basicAuthConnector{ - apiClient: apiClient, - username: cred.Username, - password: cred.Password, + apiClient: apiClient, + graphqlClient: graphqlClient, + username: cred.Username, + password: cred.Password, }, nil } @@ -43,6 +52,10 @@ func (c *basicAuthConnector) APIClient() *github.Client { return c.apiClient } +func (c *basicAuthConnector) GraphQLClient() *githubv4.Client { + return c.graphqlClient +} + func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username) } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 88ef8f8f1ec0..70386e13ce42 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -7,24 +7,29 @@ import ( gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" + "golang.org/x/oauth2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" - "golang.org/x/oauth2" ) type tokenConnector struct { - apiClient *github.Client - token string + token string + apiClient *github.Client + graphqlClient *githubv4.Client + isGitHubEnterprise bool - handleRateLimit func(context.Context, error) bool - user string - userMu sync.Mutex + + handleRateLimit func(context.Context, error) bool + user string + userMu sync.Mutex } var _ connector = (*tokenConnector)(nil) -func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(context.Context, error) bool) (*tokenConnector, error) { +func newTokenConnector(ctx context.Context, apiEndpoint string, token string, handleRateLimit func(context.Context, error) bool) (*tokenConnector, error) { const httpTimeoutSeconds = 60 httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) @@ -33,15 +38,21 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(co Source: tokenSource, } - apiClient, err := createGitHubClient(httpClient, apiEndpoint) + apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint) if err != nil { return nil, fmt.Errorf("could not create API client: %w", err) } + graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return &tokenConnector{ apiClient: apiClient, + graphqlClient: graphqlClient, token: token, - isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), + isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudV3Endpoint), handleRateLimit: handleRateLimit, }, nil } @@ -50,6 +61,10 @@ func (c *tokenConnector) APIClient() *github.Client { return c.apiClient } +func (c *tokenConnector) GraphQLClient() *githubv4.Client { + return c.graphqlClient +} + func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { if err := c.setUserIfUnset(ctx); err != nil { return "", nil, err diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 53b77b10893a..28fcab5de351 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -5,6 +5,7 @@ import ( gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v67/github" + "github.com/shurcooL/githubv4" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -12,20 +13,28 @@ import ( ) type unauthenticatedConnector struct { - apiClient *github.Client + apiClient *github.Client + graphqlClient *githubv4.Client } var _ connector = (*unauthenticatedConnector)(nil) -func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { +func newUnauthenticatedConnector(ctx context.Context, apiEndpoint string) (*unauthenticatedConnector, error) { const httpTimeoutSeconds = 60 httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) - apiClient, err := createGitHubClient(httpClient, apiEndpoint) + apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint) if err != nil { return nil, fmt.Errorf("could not create API client: %w", err) } + + graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return &unauthenticatedConnector{ - apiClient: apiClient, + apiClient: apiClient, + graphqlClient: graphqlClient, }, nil } @@ -33,6 +42,10 @@ func (c *unauthenticatedConnector) APIClient() *github.Client { return c.apiClient } +func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client { + return c.graphqlClient +} + func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingUnauthenticated(ctx, repoURL) } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 4dce437fba47..563897a00114 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -225,7 +225,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so } s.conn = &conn - connector, err := newConnector(s) + connector, err := newConnector(aCtx, s) if err != nil { return fmt.Errorf("could not create connector: %w", err) } @@ -609,16 +609,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, installationClient *githu return nil } -func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { - // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. - if strings.EqualFold(apiEndpoint, cloudEndpoint) { - return github.NewClient(httpClient), nil - } - - return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) -} - func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error { var scannedCount uint64 = 1