[GitHub] Add a GraphQL client to the connector

go.mod

@@ -270,6 +270,8 @@ require (
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 // indirect
+	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.3.0 // indirect
 	github.com/sorairolake/lzip-go v0.3.5 // indirect

go.sum

@@ -709,6 +709,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
 github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
 github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
 github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M=
+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

pkg/sources/github/connector.go

@@ -2,43 +2,90 @@ package github
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v67/github"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
+	"github.com/shurcooL/githubv4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
-const cloudEndpoint = "https://api.github.com"
+const (
+	cloudV3Endpoint      = "https://api.github.com"
+	cloudGraphqlEndpoint = "https://api.github.com/graphql" // https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#the-graphql-endpoint
+)
 
 type connector interface {
 	// APIClient returns a configured GitHub client that can be used for GitHub API operations.
 	APIClient() *github.Client
+	// GraphQLClient returns a client that can be used for GraphQL operations.
+	GraphQLClient() *githubv4.Client
 	// Clone clones a repository using the configured authentication information.
 	Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error)
 }
 
-func newConnector(source *Source) (connector, error) {
+func newConnector(ctx context.Context, source *Source) (connector, error) {
+	// Construct the URLs.
 	apiEndpoint := source.conn.Endpoint
 	if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {
-		apiEndpoint = cloudEndpoint
+		apiEndpoint = cloudV3Endpoint
 	}
 
 	switch cred := source.conn.GetCredential().(type) {
 	case *sourcespb.GitHub_GithubApp:
 		log.RedactGlobally(cred.GithubApp.GetPrivateKey())
-		return newAppConnector(apiEndpoint, cred.GithubApp)
+		return newAppConnector(ctx, apiEndpoint, cred.GithubApp)
 	case *sourcespb.GitHub_BasicAuth:
 		log.RedactGlobally(cred.BasicAuth.GetPassword())
-		return newBasicAuthConnector(apiEndpoint, cred.BasicAuth)
+		return newBasicAuthConnector(ctx, apiEndpoint, cred.BasicAuth)
 	case *sourcespb.GitHub_Token:
 		log.RedactGlobally(cred.Token)
-		return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit)
+		return newTokenConnector(ctx, apiEndpoint, cred.Token, source.handleRateLimit)
 	case *sourcespb.GitHub_Unauthenticated:
-		return newUnauthenticatedConnector(apiEndpoint)
+		return newUnauthenticatedConnector(ctx, apiEndpoint)
 	default:
 		return nil, fmt.Errorf("unknown connection type")
 	}
 }
+
+func createAPIClient(ctx context.Context, httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
+	ctx.Logger().WithName("github").V(2).
+		Info("Creating API client", "url", apiEndpoint)
+
+	// If we're using public GitHub, make a regular client.
+	// Otherwise, make an enterprise client.
+	if strings.EqualFold(apiEndpoint, cloudV3Endpoint) {
+		return github.NewClient(httpClient), nil
+	}
+
+	return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
+}
+
+func createGraphqlClient(ctx context.Context, client *http.Client, apiEndpoint string) (*githubv4.Client, error) {
+	var graphqlEndpoint string
+	if apiEndpoint == cloudV3Endpoint {
+		graphqlEndpoint = cloudGraphqlEndpoint
+	} else {
+		// Use the root endpoint for the host.
+		// https://docs.github.com/en/enterprise-server@3.11/graphql/guides/introduction-to-graphql
+		parsedURL, err := url.Parse(apiEndpoint)
+		if err != nil {
+			return nil, fmt.Errorf("could not create GraphQL client: %w", err)
+		}
+
+		// GitHub Enterprise uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958)
+		// Swap it, and anything before `/api`, with GraphQL.
+		before, _ := strings.CutSuffix(parsedURL.Path, "/api/v3")
+		parsedURL.Path = before + "/api/graphql"
+		graphqlEndpoint = parsedURL.String()
+	}
+	ctx.Logger().WithName("github").V(2).
+		Info("Creating GraphQL client", "url", graphqlEndpoint)
+
+	return githubv4.NewEnterpriseClient(graphqlEndpoint, client), nil
+}

pkg/sources/github/connector_app.go

@@ -7,6 +7,8 @@ import (
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v67/github"
+	"github.com/shurcooL/githubv4"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
@@ -15,13 +17,14 @@ import (
 
 type appConnector struct {
 	apiClient          *github.Client
+	graphqlClient      *githubv4.Client
 	installationClient *github.Client
 	installationID     int64
 }
 
 var _ connector = (*appConnector)(nil)
 
-func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) {
+func newAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) {
 	installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err)
@@ -67,8 +70,14 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn
 		return nil, fmt.Errorf("could not create API client: %w", err)
 	}
 
+	graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return &appConnector{
 		apiClient:          apiClient,
+		graphqlClient:      graphqlClient,
 		installationClient: installationClient,
 		installationID:     installationID,
 	}, nil
@@ -78,6 +87,10 @@ func (c *appConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *appConnector) GraphQLClient() *githubv4.Client {
+	return c.graphqlClient
+}
+
 func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	// TODO: Check rate limit for this call.
 	token, _, err := c.installationClient.Apps.CreateInstallationToken(

pkg/sources/github/connector_basicauth.go

@@ -5,44 +5,57 @@ import (
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v67/github"
+	"github.com/shurcooL/githubv4"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
 type basicAuthConnector struct {
-	apiClient *github.Client
-	username  string
-	password  string
+	apiClient     *github.Client
+	graphqlClient *githubv4.Client
+	username      string
+	password      string
 }
 
 var _ connector = (*basicAuthConnector)(nil)
 
-func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) {
+func newBasicAuthConnector(ctx context.Context, apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) {
 	const httpTimeoutSeconds = 60
 	httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
 	httpClient.Transport = &github.BasicAuthTransport{
 		Username: cred.Username,
 		Password: cred.Password,
 	}
 
-	apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+	apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
 	if err != nil {
 		return nil, fmt.Errorf("could not create API client: %w", err)
 	}
 
+	graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return &basicAuthConnector{
-		apiClient: apiClient,
-		username:  cred.Username,
-		password:  cred.Password,
+		apiClient:     apiClient,
+		graphqlClient: graphqlClient,
+		username:      cred.Username,
+		password:      cred.Password,
 	}, nil
 }
 
 func (c *basicAuthConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *basicAuthConnector) GraphQLClient() *githubv4.Client {
+	return c.graphqlClient
+}
+
 func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username)
 }

pkg/sources/github/connector_token.go

@@ -7,24 +7,29 @@ import (
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v67/github"
+	"github.com/shurcooL/githubv4"
+	"golang.org/x/oauth2"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
-	"golang.org/x/oauth2"
 )
 
 type tokenConnector struct {
-	apiClient          *github.Client
-	token              string
+	token         string
+	apiClient     *github.Client
+	graphqlClient *githubv4.Client
+
 	isGitHubEnterprise bool
-	handleRateLimit    func(context.Context, error) bool
-	user               string
-	userMu             sync.Mutex
+
+	handleRateLimit func(context.Context, error) bool
+	user            string
+	userMu          sync.Mutex
 }
 
 var _ connector = (*tokenConnector)(nil)
 
-func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(context.Context, error) bool) (*tokenConnector, error) {
+func newTokenConnector(ctx context.Context, apiEndpoint string, token string, handleRateLimit func(context.Context, error) bool) (*tokenConnector, error) {
 	const httpTimeoutSeconds = 60
 	httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
 	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
@@ -33,15 +38,21 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(co
 		Source: tokenSource,
 	}
 
-	apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+	apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
 	if err != nil {
 		return nil, fmt.Errorf("could not create API client: %w", err)
 	}
 
+	graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return &tokenConnector{
 		apiClient:          apiClient,
+		graphqlClient:      graphqlClient,
 		token:              token,
-		isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint),
+		isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudV3Endpoint),
 		handleRateLimit:    handleRateLimit,
 	}, nil
 }
@@ -50,6 +61,10 @@ func (c *tokenConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *tokenConnector) GraphQLClient() *githubv4.Client {
+	return c.graphqlClient
+}
+
 func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	if err := c.setUserIfUnset(ctx); err != nil {
 		return "", nil, err

pkg/sources/github/connector_unauthenticated.go

@@ -5,34 +5,47 @@ import (
 
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v67/github"
+	"github.com/shurcooL/githubv4"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
 type unauthenticatedConnector struct {
-	apiClient *github.Client
+	apiClient     *github.Client
+	graphqlClient *githubv4.Client
 }
 
 var _ connector = (*unauthenticatedConnector)(nil)
 
-func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) {
+func newUnauthenticatedConnector(ctx context.Context, apiEndpoint string) (*unauthenticatedConnector, error) {
 	const httpTimeoutSeconds = 60
 	httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
-	apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+	apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
 	if err != nil {
 		return nil, fmt.Errorf("could not create API client: %w", err)
 	}
+
+	graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return &unauthenticatedConnector{
-		apiClient: apiClient,
+		apiClient:     apiClient,
+		graphqlClient: graphqlClient,
 	}, nil
 }
 
 func (c *unauthenticatedConnector) APIClient() *github.Client {
 	return c.apiClient
 }
 
+func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client {
+	return c.graphqlClient
+}
+
 func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
 	return git.CloneRepoUsingUnauthenticated(ctx, repoURL)
 }

pkg/sources/github/github.go

@@ -225,7 +225,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 	}
 	s.conn = &conn
 
-	connector, err := newConnector(s)
+	connector, err := newConnector(aCtx, s)
 	if err != nil {
 		return fmt.Errorf("could not create connector: %w", err)
 	}
@@ -609,16 +609,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, installationClient *githu
 	return nil
 }
 
-func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
-	// If we're using public GitHub, make a regular client.
-	// Otherwise, make an enterprise client.
-	if strings.EqualFold(apiEndpoint, cloudEndpoint) {
-		return github.NewClient(httpClient), nil
-	}
-
-	return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
-}
-
 func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {
 	var scannedCount uint64 = 1