Skip to content

Commit

Permalink
feat: Added support for templates in external secrets (#5874)
Browse files Browse the repository at this point in the history 
* feat: Added support for templates in external secrets

* renamed dataFrom to esoDataFrom

* deployment history data fixed

* deployment history data diff view fixed

* added validations and const

* renamed EsoData to ESOData

* chore: updated []json.RawMessage to json.RawMessage

* feat: updated validations
  • Loading branch information
@Ash-exp
Ash-exp authored Sep 27, 2024
1 parent 3060edb commit c54fc03
Show file tree
Hide file tree
Showing 10 changed files with 123 additions and 59 deletions.
7 changes: 4 additions & 3 deletions api/appbean/AppDetail.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,9 +179,10 @@ type Secret struct {
}

type ConfigMapSecretDataVolumeUsageConfig struct {
MountPath string `json:"mountPath"`
SubPath bool `json:"subPath"`
FilePermission string `json:"filePermission"`
MountPath string `json:"mountPath"`
SubPath bool `json:"subPath"`
FilePermission string `json:"filePermission"`
ESOSubPath []string `json:"esoSubPath"`
}

type ExternalSecret struct {
Expand Down
3 changes: 2 additions & 1 deletion api/bean/ConfigMapAndSecret.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type ConfigSecretMap struct {
RoleARN string `json:"roleARN"`
SecretData json.RawMessage `json:"secretData,omitempty"`
SubPath bool `json:"subPath"`
ESOSubPath []string `json:"esoSubPath"`
FilePermission string `json:"filePermission"`
}

Expand All @@ -70,7 +71,7 @@ func (configSecretJson *ConfigSecretJson) SetReferencedSecrets(secrets []ConfigS
configSecretJson.Secrets = util.GetReferencedArray(secrets)
}

func (ConfigSecretRootJson) GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
func GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
secretsJson := ConfigSecretRootJson{}
err := json.Unmarshal([]byte(data), &secretsJson)
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion api/restHandler/CoreAppRestHandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -891,7 +891,7 @@ func (handler CoreAppRestHandlerImpl) buildAppConfigMaps(appId int, envId int, c
}
var dataObj map[string]interface{}
if data != nil {
err := json.Unmarshal([]byte(data), &dataObj)
err := json.Unmarshal(data, &dataObj)
if err != nil {
handler.logger.Errorw("service err, un-marshaling of data fail in config map", "err", err, "appId", appId)
return nil, err, http.StatusInternalServerError
Expand Down Expand Up @@ -1041,6 +1041,7 @@ func (handler CoreAppRestHandlerImpl) buildAppSecrets(appId int, envId int, secr
globalSecret.DataVolumeUsageConfig = &appBean.ConfigMapSecretDataVolumeUsageConfig{
SubPath: secret.SubPath,
FilePermission: secret.FilePermission,
ESOSubPath: secret.ESOSubPath,
}
considerGlobalDefaultData := envId > 0 && secret.Data == nil
if considerGlobalDefaultData {
Expand Down Expand Up @@ -1486,6 +1487,7 @@ func (handler CoreAppRestHandlerImpl) createGlobalSecrets(appId int, userId int3
secretData.MountPath = dataVolumeUsageConfig.MountPath
secretData.SubPath = dataVolumeUsageConfig.SubPath
secretData.FilePermission = dataVolumeUsageConfig.FilePermission
secretData.ESOSubPath = dataVolumeUsageConfig.ESOSubPath
}

if secret.IsExternal {
Expand Down Expand Up @@ -1989,6 +1991,7 @@ func (handler CoreAppRestHandlerImpl) createEnvSecret(appId int, userId int32, e
secretData.MountPath = secretOverrideDataVolumeUsageConfig.MountPath
secretData.SubPath = secretOverrideDataVolumeUsageConfig.SubPath
secretData.FilePermission = secretOverrideDataVolumeUsageConfig.FilePermission
secretData.ESOSubPath = secretOverrideDataVolumeUsageConfig.ESOSubPath
}
var secretDataRequest []*bean2.ConfigData
secretDataRequest = append(secretDataRequest, secretData)
Expand Down
1 change: 1 addition & 0 deletions pkg/appClone/AppCloneService.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,7 @@ func (impl *AppCloneServiceImpl) configDataClone(cfData []*bean3.ConfigData) []*
ExternalSecretType: refdata.ExternalSecretType,
FilePermission: refdata.FilePermission,
SubPath: refdata.SubPath,
ESOSubPath: refdata.ESOSubPath,
}
copiedData = append(copiedData, data)
}
Expand Down
13 changes: 11 additions & 2 deletions pkg/bean/configSecretData.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package bean
import (
"encoding/json"
"github.com/devtron-labs/devtron/util"
"strings"
)

type ConfigList struct {
Expand All @@ -29,6 +30,7 @@ type SecretList struct {
ConfigData []*ConfigData `json:"secrets"`
}

// TODO refactoring: duplicate struct of ConfigData in ConfigMapBean.go
type ConfigData struct {
Name string `json:"name"`
Type string `json:"type"`
Expand All @@ -45,9 +47,14 @@ type ConfigData struct {
DefaultESOSecretData ESOSecretData `json:"defaultESOSecretData,omitempty"`
RoleARN string `json:"roleARN"`
SubPath bool `json:"subPath"`
ESOSubPath []string `json:"esoSubPath"`
FilePermission string `json:"filePermission"`
}

func (c *ConfigData) IsESOExternalSecretType() bool {
return strings.HasPrefix(c.ExternalSecretType, "ESO")
}

type ExternalSecret struct {
Key string `json:"key"`
Name string `json:"name"`
Expand All @@ -58,8 +65,10 @@ type ExternalSecret struct {
type ESOSecretData struct {
SecretStore json.RawMessage `json:"secretStore,omitempty"`
SecretStoreRef json.RawMessage `json:"secretStoreRef,omitempty"`
EsoData []ESOData `json:"esoData"`
ESOData []ESOData `json:"esoData"`
RefreshInterval string `json:"refreshInterval,omitempty"`
ESODataFrom json.RawMessage `json:"esoDataFrom,omitempty"`
Template json.RawMessage `json:"template,omitempty"`
}

type ESOData struct {
Expand All @@ -68,7 +77,7 @@ type ESOData struct {
Property string `json:"property,omitempty"`
}

func (ConfigData) GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
func GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
secretDataMap := make(map[string]*ConfigData)
err := json.Unmarshal([]byte(data), &secretDataMap)
if err != nil {
Expand Down
29 changes: 23 additions & 6 deletions pkg/pipeline/ConfigMapService.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,8 +487,6 @@ func (impl ConfigMapServiceImpl) CMEnvironmentFetch(appId int, envId int) (*bean
item.DefaultMountPath = item.MountPath
item.Data = nil
item.MountPath = ""
item.SubPath = item.SubPath
item.FilePermission = item.FilePermission
configDataRequest.ConfigData = append(configDataRequest.ConfigData, item)
}
}
Expand Down Expand Up @@ -580,6 +578,7 @@ func (impl ConfigMapServiceImpl) CSGlobalAddUpdate(configMapRequest *bean.Config
item.ExternalSecret = configData.ExternalSecret
item.RoleARN = configData.RoleARN
item.SubPath = configData.SubPath
item.ESOSubPath = configData.ESOSubPath
item.FilePermission = configData.FilePermission
}
configs = append(configs, item)
Expand Down Expand Up @@ -736,6 +735,7 @@ func (impl ConfigMapServiceImpl) CSEnvironmentAddUpdate(configMapRequest *bean.C
item.ExternalSecret = configData.ExternalSecret
item.RoleARN = configData.RoleARN
item.SubPath = configData.SubPath
item.ESOSubPath = configData.ESOSubPath
item.FilePermission = configData.FilePermission
found = true
}
Expand Down Expand Up @@ -873,16 +873,16 @@ func (impl ConfigMapServiceImpl) CSEnvironmentFetch(appId int, envId int) (*bean
item.DefaultExternalSecret = item.ExternalSecret
}
item.DefaultESOSecretData = item.ESOSecretData
item.ESOSecretData.EsoData = nil
item.ESOSecretData.ESOData = nil
item.ESOSecretData.SecretStore = nil
item.ESOSecretData.ESODataFrom = nil
item.ESOSecretData.Template = nil
item.ESOSecretData.SecretStoreRef = nil
item.ESOSecretData.RefreshInterval = ""
item.DefaultMountPath = item.MountPath
item.Data = nil
item.ExternalSecret = nil
item.MountPath = ""
item.SubPath = item.SubPath
item.FilePermission = item.FilePermission
configDataRequest.ConfigData = append(configDataRequest.ConfigData, item)
}
}
Expand Down Expand Up @@ -1455,7 +1455,24 @@ func (impl ConfigMapServiceImpl) validateConfigDataForSecretsOnly(configData *be
if err != nil {
impl.logger.Errorw("error in decoding secret data", "error", err)
return false, util.NewApiError().WithHttpStatusCode(http.StatusUnprocessableEntity).WithCode(strconv.Itoa(http.StatusUnprocessableEntity)).
WithUserMessage("error in decoding data, make sure the secret data is encoded properly")
WithUserMessage("error in decoding data, make sure the secret data is encoded properly").
WithInternalMessage("error in decoding data, make sure the secret data is encoded properly")
}
}
if configData.IsESOExternalSecretType() {
if !configData.External {
return false, util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).
WithUserMessage(fmt.Sprintf("external flag should be true for '%s' secret type", configData.ExternalSecretType)).
WithInternalMessage(fmt.Sprintf("external flag should be true for '%s' secret type", configData.ExternalSecretType))
}
if configData.ESOSecretData.ESODataFrom == nil && configData.ESOSecretData.ESOData == nil {
return false, util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).
WithUserMessage("both esoSecretData.esoDataFrom and esoSecretData.esoData can't be empty").
WithInternalMessage("both esoSecretData.esoDataFrom and esoSecretData.esoData can't be empty")
} else if configData.ESOSecretData.SecretStore == nil && configData.ESOSecretData.SecretStoreRef == nil {
return false, util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).
WithUserMessage("both esoSecretData.secretStore and esoSecretData.secretStoreRef can't be empty").
WithInternalMessage("both esoSecretData.secretStore and esoSecretData.secretStoreRef can't be empty")
}
}
return true, nil
Expand Down
11 changes: 10 additions & 1 deletion pkg/pipeline/bean/ConfigMapBean.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package bean

import (
"encoding/json"
"strings"
)

type ConfigDataRequest struct {
Expand All @@ -31,8 +32,10 @@ type ConfigDataRequest struct {
type ESOSecretData struct {
SecretStore json.RawMessage `json:"secretStore,omitempty"`
SecretStoreRef json.RawMessage `json:"secretStoreRef,omitempty"`
EsoData []ESOData `json:"esoData,omitempty"`
ESOData []ESOData `json:"esoData,omitempty"`
RefreshInterval string `json:"refreshInterval,omitempty"`
ESODataFrom json.RawMessage `json:"esoDataFrom,omitempty"`
Template json.RawMessage `json:"template,omitempty"`
}

type ESOData struct {
Expand All @@ -57,9 +60,15 @@ type ConfigData struct {
DefaultExternalSecret []ExternalSecret `json:"defaultSecretData,omitempty"`
RoleARN string `json:"roleARN"`
SubPath bool `json:"subPath"`
ESOSubPath []string `json:"esoSubPath"`
FilePermission string `json:"filePermission"`
Overridden bool `json:"overridden"`
}

func (c *ConfigData) IsESOExternalSecretType() bool {
return strings.HasPrefix(c.ExternalSecretType, "ESO")
}

type ExternalSecret struct {
Key string `json:"key"`
Name string `json:"name"`
Expand Down
39 changes: 29 additions & 10 deletions pkg/pipeline/history/ConfigMapHistoryService.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"context"
"encoding/json"
"errors"
"strings"
globalUtil "github.com/devtron-labs/devtron/util"
"time"

"github.com/devtron-labs/devtron/internal/sql/repository/chartConfig"
Expand Down Expand Up @@ -490,7 +490,7 @@ func (impl ConfigMapHistoryServiceImpl) GetHistoryForDeployedCMCSById(ctx contex
SubPath: &config.SubPath,
FilePermission: config.FilePermission,
CodeEditorValue: &HistoryDetailConfig{
DisplayName: "Data",
DisplayName: DataDisplayName,
Value: string(config.Data),
VariableSnapshot: variableSnapshotMap,
ResolvedValue: resolvedTemplate,
Expand Down Expand Up @@ -521,13 +521,27 @@ func (impl ConfigMapHistoryServiceImpl) GetHistoryForDeployedCMCSById(ctx contex
}
historyDto.ExternalSecretType = config.ExternalSecretType
historyDto.RoleARN = config.RoleARN
historyDto.ESOSubPath = config.ESOSubPath
if config.External {
externalSecretData, err := json.Marshal(config.ExternalSecret)
if err != nil {
impl.logger.Errorw("error in marshaling external secret data", "err", err)
}
if len(externalSecretData) > 0 {
historyDto.CodeEditorValue.Value = string(externalSecretData)
if config.ExternalSecretType == globalUtil.KubernetesSecret {
externalSecretData, err := json.Marshal(config.ExternalSecret)
if err != nil {
impl.logger.Errorw("error in marshaling external secret data", "err", err)
}
if len(externalSecretData) > 0 {
historyDto.CodeEditorValue.DisplayName = ExternalSecretDisplayName
historyDto.CodeEditorValue.Value = string(externalSecretData)
}
} else if config.IsESOExternalSecretType() {
externalSecretDataBytes, jErr := json.Marshal(config.ESOSecretData)
if jErr != nil {
impl.logger.Errorw("error in marshaling eso secret data", "esoSecretData", config.ESOSecretData, "err", jErr)
return nil, jErr
}
if len(externalSecretDataBytes) > 0 {
historyDto.CodeEditorValue.DisplayName = ESOSecretDataDisplayName
historyDto.CodeEditorValue.Value = string(externalSecretDataBytes)
}
}
}
}
Expand Down Expand Up @@ -593,7 +607,7 @@ func (impl ConfigMapHistoryServiceImpl) ConvertConfigDataToComponentLevelDto(con
SubPath: &config.SubPath,
FilePermission: config.FilePermission,
CodeEditorValue: &HistoryDetailConfig{
DisplayName: "Data",
DisplayName: DataDisplayName,
Value: string(config.Data),
},
}
Expand Down Expand Up @@ -623,22 +637,27 @@ func (impl ConfigMapHistoryServiceImpl) ConvertConfigDataToComponentLevelDto(con
}
historyDto.ExternalSecretType = config.ExternalSecretType
historyDto.RoleARN = config.RoleARN
historyDto.ESOSubPath = config.ESOSubPath
if config.External {
var externalSecretData []byte
if strings.HasPrefix(config.ExternalSecretType, "ESO") {
displayName := historyDto.CodeEditorValue.DisplayName
if config.IsESOExternalSecretType() {
displayName = ESOSecretDataDisplayName
externalSecretData, err = json.Marshal(config.ESOSecretData)
if err != nil {
impl.logger.Errorw("error in marshaling external secret data", "err", err)
return nil, err
}
} else {
displayName = ExternalSecretDisplayName
externalSecretData, err = json.Marshal(config.ExternalSecret)
if err != nil {
impl.logger.Errorw("error in marshaling external secret data", "err", err)
return nil, err
}
}
if len(externalSecretData) > 0 {
historyDto.CodeEditorValue.DisplayName = displayName
historyDto.CodeEditorValue.Value = string(externalSecretData)
}
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/pipeline/history/bean.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ type HistoryDetailDto struct {
ExternalSecretType string `json:"externalType,omitempty"`
RoleARN string `json:"roleARN,omitempty"`
SubPath *bool `json:"subPath,omitempty"`
ESOSubPath []string `json:"esoSubPath,omitempty"`
FilePermission string `json:"filePermission,omitempty"`
CodeEditorValue *HistoryDetailConfig `json:"codeEditorValue"`
SecretViewAccess bool `json:"secretViewAccess"` // this is being used to check whether a user can see obscured secret values or not.
Expand All @@ -84,6 +85,12 @@ type HistoryDetailConfig struct {
ResolvedValue string `json:"resolvedValue"`
}

const (
DataDisplayName = "Data"
ESOSecretDataDisplayName = "ESO Secret Data"
ExternalSecretDisplayName = "External Secret Data"
)

//history components(deployment template, configMaps, secrets, pipeline strategy) components below

type ConfigMapAndSecretHistoryDto struct {
Expand Down
Loading

0 comments on commit c54fc03

Please sign in to comment.