fix: panic handlings and argocd app delete stuck in partial stage (#5770

)

* fix: panic handlings

* fix: false positive matrics on gitOps failures

* fix: for GetConfigForHelmApps err: pg no row

pkg/appStore/installedApp/repository/InstalledAppRepository.go

@@ -714,7 +714,7 @@ func (impl InstalledAppRepositoryImpl) GetDeploymentSuccessfulStatusCountForTele
 func (impl InstalledAppRepositoryImpl) GetGitOpsInstalledAppsWhereArgoAppDeletedIsTrue(installedAppId int, envId int) (InstalledApps, error) {
 	var installedApps InstalledApps
 	err := impl.dbConnection.Model(&installedApps).
-		Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name").
+		Column("installed_apps.*", "App.id", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name").
 		Where("deployment_app_delete_request = ?", true).
 		Where("installed_apps.active = ?", true).
 		Where("installed_apps.id = ?", installedAppId).

pkg/appStore/installedApp/service/AppStoreDeploymentService.go

@@ -877,7 +877,7 @@ func (impl *AppStoreDeploymentServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoA
 		apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo"
 		return apiError
 	}
-	deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedAppId, envId)
+	deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedApp.App.Id, envId)
 	if err != nil {
 		impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", installedAppId, "envId", envId, "err", err)
 		apiError.HttpStatusCode = http.StatusInternalServerError

pkg/deployment/gitOps/git/GitServiceGithub.go

@@ -19,7 +19,9 @@ package git
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/runTime"
 	bean2 "github.com/devtron-labs/devtron/api/bean/gitOps"
 	globalUtil "github.com/devtron-labs/devtron/util"
 	"github.com/devtron-labs/devtron/util/retryFunc"
@@ -91,6 +93,15 @@ func (impl GitHubClient) DeleteRepository(config *bean2.GitOpsConfigDto) error {
 	return nil
 }
 
+func IsRepoNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	var responseErr *github.ErrorResponse
+	ok := errors.As(err, &responseErr)
+	return ok && responseErr.Response.StatusCode == 404
+}
+
 func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.GitOpsConfigDto) (url string, isNew bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) {
 	var err error
 	start := time.Now()
@@ -100,15 +111,14 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git
 
 	detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error)
 	repoExists := true
-	url, err = impl.GetRepoUrl(config)
+	url, err = impl.getRepoUrl(ctx, config, IsRepoNotFound)
 	if err != nil {
-		responseErr, ok := err.(*github.ErrorResponse)
-		if !ok || responseErr.Response.StatusCode != 404 {
+		if IsRepoNotFound(err) {
+			repoExists = false
+		} else {
 			impl.logger.Errorw("error in creating github repo", "err", err)
 			detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err
 			return "", false, detailedErrorGitOpsConfigActions
-		} else {
-			repoExists = false
 		}
 	}
 	if repoExists {
@@ -251,12 +261,21 @@ func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig,
 }
 
 func (impl GitHubClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) {
+	ctx := context.Background()
+	return impl.getRepoUrl(ctx, config, globalUtil.AllPublishableError())
+}
+
+func (impl GitHubClient) getRepoUrl(ctx context.Context, config *bean2.GitOpsConfigDto,
+	isNonPublishableError globalUtil.EvalIsNonPublishableErr) (repoUrl string, err error) {
 	start := time.Now()
 	defer func() {
+		if isNonPublishableError(err) {
+			impl.logger.Debugw("found non publishable error. skipping metrics publish!", "caller method", runTime.GetCallerFunctionName(), "err", err)
+			return
+		}
 		globalUtil.TriggerGitOpsMetrics("GetRepoUrl", "GitHubClient", start, err)
 	}()
 
-	ctx := context.Background()
 	repo, _, err := impl.client.Repositories.Get(ctx, impl.org, config.GitRepoName)
 	if err != nil {
 		impl.logger.Errorw("error in getting repo url by repo name", "org", impl.org, "gitRepoName", config.GitRepoName, "err", err)

pkg/pipeline/CdHandler.go

@@ -475,11 +475,15 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC
 		if !cdWorkflow.BlobStorageEnabled {
 			return nil, nil, errors.New("logs-not-stored-in-repository")
 		} else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == executors.WorkflowCancel {
-			impl.Logger.Debugw("pod is not live ", "err", err)
+			impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err)
 			return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv)
 		}
-		impl.Logger.Errorw("err on fetch workflow logs", "err", err)
-		return nil, nil, err
+		if err != nil {
+			impl.Logger.Errorw("err on fetch workflow logs", "err", err)
+			return nil, nil, err
+		} else if logStream == nil {
+			return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName)
+		}
 	}
 	logReader := bufio.NewReader(logStream)
 	return logReader, cleanUp, err

pkg/pipeline/CiHandler.go

@@ -798,11 +798,15 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC
 		if !ciWorkflow.BlobStorageEnabled {
 			return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"}
 		} else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == executors.WorkflowCancel {
-			impl.Logger.Errorw("err", "err", err)
+			impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err)
 			return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt)
 		}
-		impl.Logger.Errorw("err", "err", err)
-		return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()}
+		if err != nil {
+			impl.Logger.Errorw("err on fetch workflow logs", "err", err)
+			return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()}
+		} else if logStream == nil {
+			return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName)
+		}
 	}
 	logReader := bufio.NewReader(logStream)
 	return logReader, cleanUp, err

pkg/pipeline/CiLogService.go

@@ -68,9 +68,12 @@ func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest types.BuildL
 	}
 	req := impl.k8sUtil.GetLogsForAPod(kubeClient, ciLogRequest.Namespace, ciLogRequest.PodName, CiPipeline.Main, true)
 	podLogs, err := req.Stream(context.Background())
-	if podLogs == nil || err != nil {
-		impl.logger.Errorw("error in opening stream", "name", ciLogRequest.PodName)
+	if err != nil {
+		impl.logger.Errorw("error in opening stream", "name", ciLogRequest.PodName, "err", err)
 		return nil, nil, err
+	} else if podLogs == nil {
+		impl.logger.Warnw("no stream reader found", "name", ciLogRequest.PodName)
+		return nil, func() error { return nil }, err
 	}
 	cleanUpFunc := func() error {
 		impl.logger.Info("closing running pod log stream")

util/helper.go

@@ -278,6 +278,14 @@ func TriggerGitOpsMetrics(operation string, method string, startTime time.Time,
 	middleware.GitOpsDuration.WithLabelValues(operation, method, status).Observe(time.Since(startTime).Seconds())
 }
 
+type EvalIsNonPublishableErr func(err error) bool
+
+func AllPublishableError() EvalIsNonPublishableErr {
+	return func(err error) bool {
+		return false
+	}
+}
+
 func InterfaceToString(resp interface{}) string {
 	var dat string
 	b, err := json.Marshal(resp)