[DEVHAS183] create kube jobs for CDQ analysis (#388)

* add k8s job Signed-off-by: Stephanie <yangcao@redhat.com> * set env variables Signed-off-by: Stephanie <yangcao@redhat.com> * fix some bugs Signed-off-by: Stephanie <yangcao@redhat.com> * add tests Signed-off-by: Stephanie <yangcao@redhat.com> * add role Signed-off-by: Stephanie <yangcao@redhat.com> * fix unit test failure Signed-off-by: Stephanie <yangcao@redhat.com> * increase config map waiting timeout to 5mins Signed-off-by: Stephanie <yangcao@redhat.com> * rebase Signed-off-by: Stephanie <yangcao@redhat.com> * address review comments Signed-off-by: Stephanie <yangcao@redhat.com> * fix job bug Signed-off-by: Stephanie <yangcao@redhat.com> * run go mod tidy Signed-off-by: Stephanie <yangcao@redhat.com> --------- Signed-off-by: Stephanie <yangcao@redhat.com>
redhat-appstudio · Sep 29, 2023 · ea77d60 · ea77d60
1 parent c2f7c29
commit ea77d60
Show file tree

Hide file tree

Showing 13 changed files with 1,939 additions and 73 deletions.
diff --git a/.tekton/push-cdq-analysis.yaml b/.tekton/push-cdq-analysis.yaml
@@ -16,9 +16,6 @@ spec:
       value: "quay.io/redhat-appstudio/cdq-analysis:{{revision}}"
     - name: path-context
       value: "cdq-analysis"
-    - name: infra-deployment-update-script
-      value: |
-        sed -i -e 's|\(https://github.com/redhat-appstudio/application-service/.*?ref=\)\(.*\)|\1{{ revision }}|' -e 's/\(newTag: \).*/\1{{ revision }}/' components/has/cdq-analysis/kustomization.yaml  
   pipelineRef:
     name: docker-build
     bundle: quay.io/redhat-appstudio-tekton-catalog/pipeline-core-services-docker-build:latest

diff --git a/cdq-analysis/entrypoint.sh b/cdq-analysis/entrypoint.sh
@@ -1,7 +1,6 @@
 #!/bin/sh
 set -eux
 
-./cdq-analysis --name "$NAME" --namespace "$NAMESPACE" -- contextPath "$CONTEXT_PATH" \
-            --revision "$REVISION" --URL "$URL" --DevfileRegistryURL "$DEVFILE_REGISTRY_URL" \
-            --devfilePath "$DEVFILE_PATH" --dockerfilePath "$DOCKERFILE_PATH" --isDevfilePresent $IS_DEVFILE_PRESENT \
-            --isDockerfilePresent $IS_DOCKERFILE_PRESENT --createK8sJob $CREATE_K8S_Job
+./cdq-analysis --name "$NAME" --namespace "$NAMESPACE" --contextPath "$CONTEXT_PATH" \
+            --revision "$REVISION" --URL "$URL" --devfileRegistryURL "$DEVFILE_REGISTRY_URL" \
+            --createK8sJob $CREATE_K8S_Job
diff --git a/cdq-analysis/main.go b/cdq-analysis/main.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 
 	"go.uber.org/zap/zapcore"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -30,26 +31,27 @@ import (
 )
 
 func main() {
-	if os.Getenv("GITHUB_TOKEN") == "" {
-		log.Fatal("GITHUB_TOKEN must be set as an environment variable")
-	}
 	gitToken := os.Getenv("GITHUB_TOKEN")
 
 	// Parse all of the possible command-line flags for the tool
-	var contextPath, URL, name, Revision, namespace, DevfileRegistryURL string
-	var isDevfilePresent, isDockerfilePresent, createK8sJob bool
+	var contextPath, URL, name, Revision, namespace, DevfileRegistryURL, createK8sJobStr string
+	var createK8sJob bool
 	flag.StringVar(&name, "name", "", "The ComponentDetectionQuery name")
 	flag.StringVar(&contextPath, "contextPath", "./", "The context path for the cdq analysis")
 	flag.StringVar(&URL, "URL", "", "The URL for the git repository")
 	flag.StringVar(&Revision, "revision", "", "The revision of the git repo to run cdq analysis against with")
 	flag.StringVar(&DevfileRegistryURL, "devfileRegistryURL", pkg.DevfileRegistryEndpoint, "The devfile registry URL")
 	flag.StringVar(&namespace, "namespace", "", "The namespace from which to fetch resources")
-	flag.BoolVar(&isDevfilePresent, "isDevfilePresent", false, "If the devfile present in the root of the repository")
-	flag.BoolVar(&isDockerfilePresent, "isDockerfilePresent", false, "If the dockerfile present in the root of the repository")
-	flag.BoolVar(&createK8sJob, "createK8sJob", false, "If a kubernetes job need to be created to send back the result")
+	flag.StringVar(&createK8sJobStr, "createK8sJob", "false", "If a kubernetes job need to be created to send back the result")
 	flag.Parse()
 
-	if err := validateVariables(name, URL, namespace, Revision); err != nil {
+	createK8sJob, err := strconv.ParseBool(createK8sJobStr)
+	if err != nil {
+		log.Fatal(fmt.Errorf("Error parse createK8sJob: %v", err))
+		createK8sJob = false
+	}
+
+	if err := validateVariables(name, URL, namespace); err != nil {
 		log.Fatal(err)
 	}
 
@@ -90,7 +92,7 @@ func main() {
 }
 
 // validateVariables ensures that all of the necessary variables passed in are set to valid values
-func validateVariables(name, URL, namespace, revision string) error {
+func validateVariables(name, URL, namespace string) error {
 
 	// The namespace flag must be passed in
 	if namespace == "" {
@@ -107,10 +109,5 @@ func validateVariables(name, URL, namespace, revision string) error {
 		return fmt.Errorf("usage: --name <cdq-name> must be passed in as a flag")
 	}
 
-	// The revision flag must be passed in
-	if revision == "" {
-		return fmt.Errorf("usage: --revision <revision> must be passed in as a flag")
-	}
-
 	return nil
 }
diff --git a/cdq-analysis/pkg/componentdetectionquery.go b/cdq-analysis/pkg/componentdetectionquery.go
@@ -66,23 +66,22 @@ func (cdqInfo *CDQInfoClient) clone(k K8sInfoClient, namespace, name, context st
 	devfilesURLMap := make(map[string]string)
 	dockerfileContextMap := make(map[string]string)
 	componentPortsMap := make(map[string][]int)
+	revision := cdqInfo.GitURL.Revision
+	repoURL := cdqInfo.GitURL.RepoURL
+	gitToken := cdqInfo.GitURL.Token
 	Fs := NewFilesystem()
 	cdqInfo.ClonedRepo.Fs = Fs
 	clonePath, err = CreateTempPath(name, Fs)
 	if err != nil {
 		log.Error(err, fmt.Sprintf("Unable to create a temp path %s for cloning %v", clonePath, namespace))
-		k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+		k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 		return err
 	}
 
-	revision := cdqInfo.GitURL.Revision
-	repoURL := cdqInfo.GitURL.RepoURL
-	gitToken := cdqInfo.GitURL.Token
-
 	err = CloneRepo(clonePath, GitURL{RepoURL: repoURL, Revision: revision, Token: gitToken})
 	if err != nil {
 		log.Error(err, fmt.Sprintf("Unable to clone repo %s to path %s, exiting reconcile loop %v", repoURL, clonePath, namespace))
-		k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+		k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 		return err
 	}
 	log.Info(fmt.Sprintf("cloned from %s to path %s... %v", repoURL, clonePath, namespace))
@@ -95,7 +94,7 @@ func (cdqInfo *CDQInfoClient) clone(k K8sInfoClient, namespace, name, context st
 		revision, err = GetBranchFromRepo(componentPath)
 		if err != nil {
 			log.Error(err, fmt.Sprintf("Unable to get branch from cloned repo for component path %s, exiting reconcile loop %v", componentPath, namespace))
-			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 			return err
 		}
 	}
@@ -167,13 +166,13 @@ func CloneAndAnalyze(k K8sInfoClient, namespace, name, context string, cdqInfo *
 		log.Info(fmt.Sprintf("updatedLink %s ", updatedLink))
 		if err != nil {
 			log.Error(err, fmt.Sprintf("Unable to update the devfile link for CDQ %v... %v", name, namespace))
-			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 			return nil, nil, nil, nil, "", err
 		}
 
 		shouldIgnoreDevfile, devfileBytes, err := ValidateDevfile(log, updatedLink, gitToken)
 		if err != nil {
-			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 			return nil, nil, nil, nil, "", err
 		}
 		if shouldIgnoreDevfile {
@@ -197,7 +196,7 @@ func CloneAndAnalyze(k K8sInfoClient, namespace, name, context string, cdqInfo *
 			components, err = alizerClient.DetectComponents(componentPath)
 			if err != nil {
 				log.Error(err, fmt.Sprintf("Unable to detect components using Alizer for repo %v, under path %v... %v ", repoURL, componentPath, namespace))
-				k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+				k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 				return nil, nil, nil, nil, "", err
 			}
 			log.Info(fmt.Sprintf("components detected %v... %v", components, namespace))
@@ -217,7 +216,7 @@ func CloneAndAnalyze(k K8sInfoClient, namespace, name, context string, cdqInfo *
 		if err != nil {
 			if _, ok := err.(*NoDevfileFound); !ok {
 				log.Error(err, fmt.Sprintf("Unable to find devfile(s) in repo %s due to an error %s, exiting reconcile loop %v", repoURL, err.Error(), namespace))
-				k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+				k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 				return nil, nil, nil, nil, "", err
 			}
 		}
@@ -226,30 +225,30 @@ func CloneAndAnalyze(k K8sInfoClient, namespace, name, context string, cdqInfo *
 		err := AnalyzePath(log, alizerClient, componentPath, context, registryURL, devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, isDevfilePresent, isDockerfilePresent, gitToken)
 		if err != nil {
 			log.Error(err, fmt.Sprintf("Unable to analyze path %s for a devfile, Dockerfile or Containerfile %v", componentPath, namespace))
-			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 			return nil, nil, nil, nil, "", err
 		}
 	}
 
 	if isExist, _ := IsExisting(Fs, clonePath); isExist {
 		if err := Fs.RemoveAll(clonePath); err != nil {
 			log.Error(err, fmt.Sprintf("Unable to remove the clonepath %s %v", clonePath, namespace))
-			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, err)
+			k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, err)
 			return nil, nil, nil, nil, "", err
 		}
 	}
 
-	k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, name, namespace, nil)
+	k.SendBackDetectionResult(devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, name, namespace, nil)
 	return devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, revision, nil
 }
 
-func (k K8sInfoClient) SendBackDetectionResult(devfilesMap map[string][]byte, devfilesURLMap map[string]string, dockerfileContextMap map[string]string, componentPortsMap map[string][]int, name, namespace string, completeError error) {
+func (k K8sInfoClient) SendBackDetectionResult(devfilesMap map[string][]byte, devfilesURLMap map[string]string, dockerfileContextMap map[string]string, componentPortsMap map[string][]int, revision, name, namespace string, completeError error) {
 	log := k.Log
 	if !k.CreateK8sJob {
 		log.Info("Skip creating the job...")
 		return
 	}
-	log.Info(fmt.Sprintf("Sending back result, devfilesMap %v,devfilesURLMap %v, dockerfileContextMap %v , error %v ... %v", devfilesMap, devfilesURLMap, dockerfileContextMap, completeError, namespace))
+	log.Info(fmt.Sprintf("Sending back result, devfilesMap %v,devfilesURLMap %v, dockerfileContextMap %v, componentPortsMap %v, error %v ... %v", devfilesMap, devfilesURLMap, dockerfileContextMap, componentPortsMap, completeError, namespace))
 
 	configMapBinaryData := make(map[string][]byte)
 	if devfilesMap != nil {
@@ -265,6 +264,14 @@ func (k K8sInfoClient) SendBackDetectionResult(devfilesMap map[string][]byte, de
 		dockerfileContextMapbytes, _ := json.Marshal(dockerfileContextMap)
 		configMapBinaryData["dockerfileContextMap"] = dockerfileContextMapbytes
 	}
+
+	if componentPortsMap != nil {
+		componentPortsMapbytes, _ := json.Marshal(componentPortsMap)
+		configMapBinaryData["componentPortsMap"] = componentPortsMapbytes
+	}
+
+	configMapBinaryData["revision"] = []byte(revision)
+
 	if completeError != nil {
 		errorMap := make(map[string]string)
 		switch completeError.(type) {
@@ -292,7 +299,6 @@ func (k K8sInfoClient) SendBackDetectionResult(devfilesMap map[string][]byte, de
 	}
 	_, err := k.Clientset.CoreV1().ConfigMaps(namespace).Create(k.Ctx, &configMap, metav1.CreateOptions{})
 	if err != nil {
-		log.Error(err, fmt.Sprintf("Error creating configmap"))
+		log.Error(err, "Error creating configmap")
 	}
-	return
 }
diff --git a/cdq-analysis/pkg/componentdetectionquery_test.go b/cdq-analysis/pkg/componentdetectionquery_test.go
@@ -285,6 +285,7 @@ func TestSendBackDetectionResult(t *testing.T) {
 
 	compName := "testComponent"
 	namespaceName := "testNamespace"
+	revision := "main"
 
 	springDevfileContext := `
 schemaVersion: 2.2.0
@@ -299,7 +300,7 @@ metadata:
 		"./": []byte(springDevfileContext),
 	}
 	devfilesURLMap := map[string]string{
-		"./": "https://registry.devfile.io/devfiles/java-springboot-basic",
+		"./": "https://raw.githubusercontent.com/devfile-samples/devfile-sample-java-springboot-basic/main/devfile.yaml",
 	}
 	dockerfileContextMap := map[string]string{
 		"./": "https://raw.githubusercontent.com/devfile-samples/devfile-sample-java-springboot-basic/main/docker/Dockerfile",
@@ -313,26 +314,31 @@ metadata:
 	dockerfileContextMapbytes, _ := json.Marshal(dockerfileContextMap)
 	configMapBinaryData["dockerfileContextMap"] = dockerfileContextMapbytes
 
+	configMapBinaryData["revision"] = []byte(revision)
+
 	internalErrBinaryData := make(map[string][]byte)
 	internalErr := fmt.Errorf("dummy internal error")
 	internalErrMap := make(map[string]string)
 	internalErrMap["InternalError"] = fmt.Sprintf("%v", internalErr)
 	errorMapbytes, _ := json.Marshal(internalErrMap)
 	internalErrBinaryData["errorMap"] = errorMapbytes
+	internalErrBinaryData["revision"] = []byte(revision)
 
 	devfileNotFoundBinaryData := make(map[string][]byte)
 	devfileNotFoundErr := NoDevfileFound{"dummy location", fmt.Errorf("dummy NoDevfileFound error")}
 	devfileNotFoundErrMap := make(map[string]string)
 	devfileNotFoundErrMap["NoDevfileFound"] = fmt.Sprintf("%v", &devfileNotFoundErr)
 	devfileNotFoundErrorMapbytes, _ := json.Marshal(devfileNotFoundErrMap)
 	devfileNotFoundBinaryData["errorMap"] = devfileNotFoundErrorMapbytes
+	devfileNotFoundBinaryData["revision"] = []byte(revision)
 
 	dockerfileNotFoundBinaryData := make(map[string][]byte)
 	dockerfileNotFoundErr := NoDockerfileFound{"dummy location", fmt.Errorf("dummy NoDockerfileFound error")}
 	dockerfileNotFoundErrMap := make(map[string]string)
 	dockerfileNotFoundErrMap["NoDockerfileFound"] = fmt.Sprintf("%v", &dockerfileNotFoundErr)
 	dockerfileNotFoundErrorMapbytes, _ := json.Marshal(dockerfileNotFoundErrMap)
 	dockerfileNotFoundBinaryData["errorMap"] = dockerfileNotFoundErrorMapbytes
+	dockerfileNotFoundBinaryData["revision"] = []byte(revision)
 
 	tests := []struct {
 		testCase             string
@@ -368,7 +374,7 @@ metadata:
 	}
 	for _, tt := range tests {
 		t.Run(tt.testCase, func(t *testing.T) {
-			k8sClient.SendBackDetectionResult(tt.devfilesMap, tt.devfilesURLMap, tt.dockerfileContextMap, tt.componentPortsMap, compName, namespaceName, tt.err)
+			k8sClient.SendBackDetectionResult(tt.devfilesMap, tt.devfilesURLMap, tt.dockerfileContextMap, tt.componentPortsMap, revision, compName, namespaceName, tt.err)
 			configMap, err := clientset.CoreV1().ConfigMaps(namespaceName).Get(k8sClient.Ctx, compName, metav1.GetOptions{})
 			if err != nil {
 				t.Errorf("got unexpected error %v", err)

diff --git a/cdq-analysis/pkg/devfile.go b/cdq-analysis/pkg/devfile.go
@@ -70,6 +70,9 @@ const (
 
 	// DevfileStageRegistryEndpoint is the endpoint of the staging devfile registry
 	DevfileStageRegistryEndpoint = "https://registry.stage.devfile.io"
+
+	// CDQAnalysisImage is the image of the go module, to spwan the k8s job
+	CDQAnalysisImage = "quay.io/redhat-appstudio/cdq-analysis:latest"
 )
 
 var ValidDevfileLocations = []string{Devfile, HiddenDevfile, HiddenDirDevfile, HiddenDirHiddenDevfile}

diff --git a/cdq-analysis/pkg/errors.go b/cdq-analysis/pkg/errors.go
@@ -44,3 +44,13 @@ func (e *NoDockerfileFound) Error() string {
 	}
 	return errMsg
 }
+
+// InternalError returns cdq errors other than user error
+type InternalError struct {
+	Err error
+}
+
+func (e *InternalError) Error() string {
+	errMsg := fmt.Sprintf("internal error: %v", e.Err)
+	return errMsg
+}
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -145,13 +145,27 @@ rules:
   - spifilecontentrequests/status
   verbs:
   - get
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
   - configmaps
   verbs:
+  - create
+  - delete
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""