Skip to content

Commit

Permalink
Private Repo Support (#386)
Browse files Browse the repository at this point in the history 
* Private Repo Support

Signed-off-by: Kim Tsao <ktsao@redhat.com>

* address review comments

Signed-off-by: Kim Tsao <ktsao@redhat.com>

* address review comments - add role.yaml

Signed-off-by: Kim Tsao <ktsao@redhat.com>

* Address review comments - update docs, fix numbering and remove namespace from example

Signed-off-by: Kim Tsao <ktsao@redhat.com>

---------

Signed-off-by: Kim Tsao <ktsao@redhat.com>
  • Loading branch information
@kim-tsao
kim-tsao authored Sep 15, 2023
1 parent dab86f8 commit 61526b7
Show file tree
Hide file tree
Showing 36 changed files with 844 additions and 652 deletions.
5 changes: 3 additions & 2 deletions cdq-analysis/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ go 1.19

require (
github.com/devfile/api/v2 v2.2.1-alpha.0.20230413012049-a6c32fca0dbd
github.com/devfile/library/v2 v2.2.1-0.20230418160146-e75481b7eebd
github.com/devfile/library/v2 v2.2.1-0.20230821212346-99a3776c0b1e
github.com/devfile/registry-support/index/generator v0.0.0-20221018203505-df96d34d4273
github.com/devfile/registry-support/registry-library v0.0.0-20221018213054-47b3ffaeadba
github.com/go-logr/logr v1.2.3
github.com/hashicorp/go-multierror v1.1.1
github.com/pkg/errors v0.9.1
github.com/redhat-developer/alizer/go v0.0.0-20230516215932-135a2bb3fb90
github.com/spf13/afero v1.8.0
github.com/stretchr/testify v1.8.1
Expand All @@ -33,6 +34,7 @@ require (
github.com/containerd/typeurl v1.0.2 // indirect
github.com/creack/pty v1.1.17 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 // indirect
github.com/docker/cli v20.10.13+incompatible // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.11+incompatible // indirect
Expand Down Expand Up @@ -91,7 +93,6 @@ require (
github.com/openshift/api v0.0.0-20210503193030-25175d9d392d // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pjbgf/sha1cd v0.2.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions cdq-analysis/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,8 @@ github.com/devfile/api/v2 v2.2.1-alpha.0.20230413012049-a6c32fca0dbd/go.mod h1:q
github.com/devfile/library v1.2.1-0.20211104222135-49d635cb492f/go.mod h1:uFZZdTuRqA68FVe/JoJHP92CgINyQkyWnM2Qyiim+50=
github.com/devfile/library v1.2.1-0.20220308191614-f0f7e11b17de/go.mod h1:GSPfJaBg0+bBjBHbwBE5aerJLH6tWGQu2q2rHYd9czM=
github.com/devfile/library/v2 v2.0.1/go.mod h1:paJ0PARAVy0br13VpBEQ4fO3rZVDxWtooQ29+23PNBk=
github.com/devfile/library/v2 v2.2.1-0.20230418160146-e75481b7eebd h1:YHSwUdfWsG9Qk7Vn+NfafELv6+G6a43RRE/NjS0TfK0=
github.com/devfile/library/v2 v2.2.1-0.20230418160146-e75481b7eebd/go.mod h1:jXIVOBkEqh7YJddFcZ+vak47rlbCW//f+qqG5hUozhM=
github.com/devfile/library/v2 v2.2.1-0.20230821212346-99a3776c0b1e h1:ozHGZOXlBvtf4YqZRrMiSNH6q6gjw6XaUisMhWG4Vwg=
github.com/devfile/library/v2 v2.2.1-0.20230821212346-99a3776c0b1e/go.mod h1:7oEhkC6GW6OKmAP8HbxbaQ+nFbnACQuU7anYhJroltQ=
github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e/go.mod h1:iRPBxs+ZjfLEduVXpCCIOzdD2588Zv9OCs/CcXMcCCY=
github.com/devfile/registry-support/index/generator v0.0.0-20220527155645-8328a8a883be/go.mod h1:1fyDJL+fPHtcrYA6yjSVWeLmXmjCNth0d5Rq1rvtryc=
github.com/devfile/registry-support/index/generator v0.0.0-20221018203505-df96d34d4273 h1:DXENQSRTEDsk9com38njPg5511DD12HPIgzyFUErnpM=
Expand Down
14 changes: 10 additions & 4 deletions cdq-analysis/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,11 @@ func main() {
gitToken := os.Getenv("GITHUB_TOKEN")

// Parse all of the possible command-line flags for the tool
var contextPath, URL, name, devfilePath, dockerfilePath, Revision, namespace, DevfileRegistryURL string
var contextPath, URL, name, Revision, namespace, DevfileRegistryURL string
var isDevfilePresent, isDockerfilePresent, createK8sJob bool
flag.StringVar(&name, "name", "", "The ComponentDetectionQuery name")
flag.StringVar(&contextPath, "contextPath", "./", "The context path for the cdq analysis")
flag.StringVar(&URL, "URL", "", "The URL for the git repository")
flag.StringVar(&devfilePath, "devfilePath", "", "The devfile path if the devfile present")
flag.StringVar(&dockerfilePath, "dockerfilePath", "", "The dockerfile path if the dockerfile present")
flag.StringVar(&Revision, "revision", "", "The revision of the git repo to run cdq analysis against with")
flag.StringVar(&DevfileRegistryURL, "devfileRegistryURL", pkg.DevfileRegistryEndpoint, "The devfile registry URL")
flag.StringVar(&namespace, "namespace", "", "The namespace from which to fetch resources")
Expand Down Expand Up @@ -80,7 +78,15 @@ func main() {
Log: log,
CreateK8sJob: createK8sJob,
}
pkg.CloneAndAnalyze(k8sInfoClient, gitToken, namespace, name, contextPath, devfilePath, dockerfilePath, URL, Revision, DevfileRegistryURL, isDevfilePresent, isDockerfilePresent)

cdqInfo := &pkg.CDQInfoClient{
DevfileRegistryURL: DevfileRegistryURL,
GitURL: pkg.GitURL{RepoURL: URL, Revision: Revision, Token: gitToken},
}

/* #nosec G104 -- the main.go is triggerred by docker image, and the result as well as the error will be send by the k8s job*/
pkg.CloneAndAnalyze(k8sInfoClient, namespace, name, contextPath, cdqInfo)

}

// validateVariables ensures that all of the necessary variables passed in are set to valid values
Expand Down
136 changes: 109 additions & 27 deletions cdq-analysis/pkg/componentdetectionquery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"fmt"
"path"

"github.com/spf13/afero"

"github.com/go-logr/logr"
"github.com/redhat-developer/alizer/go/pkg/apis/model"
corev1 "k8s.io/api/core/v1"
Expand All @@ -36,61 +38,141 @@ type K8sInfoClient struct {
CreateK8sJob bool
}

// CDQ analyzer
// return values are for testing purpose
func CloneAndAnalyze(k K8sInfoClient, gitToken, namespace, name, context, devfilePath, dockerfilePath, URL, Revision, DevfileRegistryURL string, isDevfilePresent, isDockerfilePresent bool) (map[string][]byte, map[string]string, map[string]string, map[string][]int, string, error) {
type ClonedInfo struct {
ClonedPath string // For locally cloned git repos
ComponentPath string // For locally cloned git repos
Fs afero.Afero // For locally cloned git repos
}

// CDQInfoClient is a struct that contains the relevant information to 1) clone and search a given git repo for the presence of a devfile or dockerfile and 2) search for matching samples in the devfile
// registry if no devfile or dockerfiles are found.

type CDQInfoClient struct {
DevfileRegistryURL string
GitURL GitURL
ClonedRepo ClonedInfo
devfilePath string // A resolved devfile; one of DevfileName, HiddenDevfileName, HiddenDevfileDir or HiddenDirHiddenDevfile
dockerfilePath string // A resolved dockerfile
}

func GetDevfileAndDockerFilePaths(client CDQInfoClient) (string, string) {
return client.devfilePath, client.dockerfilePath
}

func (cdqInfo *CDQInfoClient) clone(k K8sInfoClient, namespace, name, context string) error {
log := k.Log
var clonePath, componentPath string
alizerClient := AlizerClient{}
var err error
devfilesMap := make(map[string][]byte)
devfilesURLMap := make(map[string]string)
dockerfileContextMap := make(map[string]string)
componentPortsMap := make(map[string][]int)
Fs := NewFilesystem()
var err error

var components []model.Component
if context == "" {
context = "./"
}

cdqInfo.ClonedRepo.Fs = Fs
clonePath, err = CreateTempPath(name, Fs)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to create a temp path %s for cloning %v", clonePath, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
return err
}

err = CloneRepo(clonePath, URL, Revision, gitToken)
revision := cdqInfo.GitURL.Revision
repoURL := cdqInfo.GitURL.RepoURL
gitToken := cdqInfo.GitURL.Token

err = CloneRepo(clonePath, GitURL{RepoURL: repoURL, Revision: revision, Token: gitToken})
if err != nil {
log.Error(err, fmt.Sprintf("Unable to clone repo %s to path %s, exiting reconcile loop %v", URL, clonePath, namespace))
log.Error(err, fmt.Sprintf("Unable to clone repo %s to path %s, exiting reconcile loop %v", repoURL, clonePath, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
return err
}
log.Info(fmt.Sprintf("cloned from %s to path %s... %v", URL, clonePath, namespace))
log.Info(fmt.Sprintf("cloned from %s to path %s... %v", repoURL, clonePath, namespace))
componentPath = clonePath
if context != "" {
componentPath = path.Join(clonePath, context)
}

if Revision == "" {
Revision, err = GetBranchFromRepo(componentPath)
if revision == "" {
revision, err = GetBranchFromRepo(componentPath)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to clone repo %s to path %s, exiting reconcile loop %v", URL, clonePath, namespace))
log.Error(err, fmt.Sprintf("Unable to get branch from cloned repo for component path %s, exiting reconcile loop %v", componentPath, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
return err
}
}

cdqInfo.ClonedRepo.ClonedPath = clonePath
cdqInfo.ClonedRepo.ComponentPath = componentPath
cdqInfo.GitURL.Revision = revision
return nil

}

// CDQ analyzer
// return values are for testing purpose
func CloneAndAnalyze(k K8sInfoClient, namespace, name, context string, cdqInfo *CDQInfoClient) (map[string][]byte, map[string]string, map[string]string, map[string][]int, string, error) {
log := k.Log
var clonePath, componentPath string
alizerClient := AlizerClient{}
devfilesMap := make(map[string][]byte)
devfilesURLMap := make(map[string]string)
dockerfileContextMap := make(map[string]string)
componentPortsMap := make(map[string][]int)
var Fs afero.Afero
var err error

var components []model.Component

repoURL := cdqInfo.GitURL.RepoURL
gitToken := cdqInfo.GitURL.Token
registryURL := cdqInfo.DevfileRegistryURL

err = cdqInfo.clone(k, namespace, name, context)
if err != nil {
return nil, nil, nil, nil, "", err
}

// search the cloned repo for valid devfile locations
devfileBytes, err := FindValidDevfiles(cdqInfo)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to find from any known devfile locations from %s ", cdqInfo.GitURL))
}

// search the cloned repo for valid dockerfile locations
dockerfileBytes, err := FindValidDockerfile(cdqInfo)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to find from any known devfile locations from %s ", cdqInfo.GitURL))
}

isDevfilePresent := len(devfileBytes) != 0
isDockerfilePresent := len(dockerfileBytes) != 0

// the following values come from the previous step when the repo was cloned
Fs = cdqInfo.ClonedRepo.Fs
componentPath = cdqInfo.ClonedRepo.ComponentPath
clonePath = cdqInfo.ClonedRepo.ClonedPath
revision := cdqInfo.GitURL.Revision
devfilePath := cdqInfo.devfilePath
dockerfilePath := cdqInfo.dockerfilePath

if context == "" {
context = "./"
}

//search for devfiles

isMultiComponent := false
if isDevfilePresent {
updatedLink, err := UpdateGitLink(URL, Revision, path.Join(context, devfilePath))
// devfilePath is the resolved, valid devfile location set in FindValidDevfiles
updatedLink, err := UpdateGitLink(repoURL, revision, path.Join(context, devfilePath))
log.Info(fmt.Sprintf("updatedLink %s ", updatedLink))
if err != nil {
log.Error(err, fmt.Sprintf("Unable to update the devfile link for CDQ %v... %v", name, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
}
shouldIgnoreDevfile, devfileBytes, err := ValidateDevfile(log, updatedLink)

shouldIgnoreDevfile, devfileBytes, err := ValidateDevfile(log, updatedLink, gitToken)
if err != nil {
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
Expand All @@ -115,7 +197,7 @@ func CloneAndAnalyze(k K8sInfoClient, gitToken, namespace, name, context, devfil
if !isDevfilePresent {
components, err = alizerClient.DetectComponents(componentPath)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to detect components using Alizer for repo %v, under path %v... %v ", URL, componentPath, namespace))
log.Error(err, fmt.Sprintf("Unable to detect components using Alizer for repo %v, under path %v... %v ", repoURL, componentPath, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
}
Expand All @@ -132,17 +214,17 @@ func CloneAndAnalyze(k K8sInfoClient, gitToken, namespace, name, context, devfil
// Logic to read multiple components in from git
if isMultiComponent {
log.Info(fmt.Sprintf("Since this is a multi-component, attempt will be made to read only level 1 dir for devfiles... %v", namespace))
devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, err = ScanRepo(log, alizerClient, componentPath, DevfileRegistryURL, URL, Revision, context)
devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, err = ScanRepo(log, alizerClient, componentPath, context, *cdqInfo)
if err != nil {
if _, ok := err.(*NoDevfileFound); !ok {
log.Error(err, fmt.Sprintf("Unable to find devfile(s) in repo %s due to an error %s, exiting reconcile loop %v", URL, err.Error(), namespace))
log.Error(err, fmt.Sprintf("Unable to find devfile(s) in repo %s due to an error %s, exiting reconcile loop %v", repoURL, err.Error(), namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
return nil, nil, nil, nil, "", err
}
}
} else {
log.Info(fmt.Sprintf("Since this is not a multi-component, attempt will be made to read devfile at the root dir... %v", namespace))
err := AnalyzePath(log, alizerClient, componentPath, context, DevfileRegistryURL, devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, isDevfilePresent, isDockerfilePresent)
err := AnalyzePath(log, alizerClient, componentPath, context, registryURL, devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, isDevfilePresent, isDockerfilePresent, gitToken)
if err != nil {
log.Error(err, fmt.Sprintf("Unable to analyze path %s for a devfile, Dockerfile or Containerfile %v", componentPath, namespace))
k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
Expand All @@ -159,7 +241,7 @@ func CloneAndAnalyze(k K8sInfoClient, gitToken, namespace, name, context, devfil
}

k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, nil)
return devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, Revision, nil
return devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, nil
}

func (k K8sInfoClient) SendBackDetectionResult(devfilesMap map[string][]byte, devfilesURLMap map[string]string, dockerfileContextMap map[string]string, componentPortsMap map[string][]int, name, namespace string, completeError error) {
Expand Down
Loading

0 comments on commit 61526b7

Please sign in to comment.