APIGOV-27135 - mulesoft DA changes to parse RAML (#197)

* APIGOV-27135 - minimal changes for raml support

* APIGOV-27135 - add env var for raml + config tests

* APIGOV-27135 - fix client test

* APIGOV-27135 - MR issues

* APIGOV-27135 - refactor GetExchangeFileContent + add tag if converted

* APIGOV-27135 MR issues

* APIGOV-27135 MR issues

.gitignore

@@ -6,6 +6,8 @@
 *.test
 *.out
 
+file
+__debug_bin**
 dist/
 coverage/
 **/local-values.yaml

deployment/discovery-deployment.yaml

@@ -61,6 +61,8 @@ spec:
               value:
             - name: MULESOFT_POLLINTERVAL
               value: 20s
+            - name: MULESOFT_DISCOVERORIGINALRAML
+              value: false
             - name: STATUS_PORT
               value: "8989"
           image: <Path_To_Image>

deployment/docker-compose.yml

@@ -23,6 +23,7 @@ services:
       MULESOFT_POLLINTERVAL: 20s
       MULESOFT_AUTH_USERNAME: <USERNAME>
       MULESOFT_AUTH_PASSWORD: <PASSWORD>
+      MULESOFT_DISCOVERORIGINALRAML: false
   traceability:
     image: mulesoft-traceability
     # ports:

go.mod

@@ -7,7 +7,7 @@ toolchain go1.21.3
 //replace github.com/Axway/agent-sdk => /home/ubuntu/go/src/github.com/Axway/agent-sdk
 
 require (
-	github.com/Axway/agent-sdk v1.1.73
+	github.com/Axway/agent-sdk v1.1.74-0.20240131154839-8ccbdd686492
 	github.com/elastic/beats/v7 v7.17.17
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Axway/agent-sdk v1.1.73 h1:aLtVRDeWNz/bvlZrhDyYL7lWelWV0TsMqzHmsG2BUAY=
 github.com/Axway/agent-sdk v1.1.73/go.mod h1:CMuNRWCU4UswYhaR65bzLKEDT0xcViQwHJUGFCnH2yk=
+github.com/Axway/agent-sdk v1.1.74-0.20240131154839-8ccbdd686492 h1:MtI5ZD3jSR3uYez39KNaUGILbmHUSPdIx8N0Llu4yqk=
+github.com/Axway/agent-sdk v1.1.74-0.20240131154839-8ccbdd686492/go.mod h1:CMuNRWCU4UswYhaR65bzLKEDT0xcViQwHJUGFCnH2yk=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v12.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=

pkg/anypoint/client.go

@@ -6,8 +6,9 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -38,7 +39,7 @@ type Client interface {
 	GetEnvironmentByName(name string) (*Environment, error)
 	GetExchangeAsset(groupID, assetID, assetVersion string) (*ExchangeAsset, error)
 	GetExchangeAssetIcon(icon string) (string, string, error)
-	GetExchangeFileContent(link, packaging, mainFile string) ([]byte, error)
+	GetExchangeFileContent(link, packaging, mainFile string, useOriginalRaml bool) ([]byte, bool, error)
 	GetPolicies(apiID int64) ([]Policy, error)
 	GetSLATiers(int642 int64) (*Tiers, error)
 	ListAssets(page *Page) ([]Asset, error)
@@ -296,11 +297,20 @@ func (c *AnypointClient) GetAPI(id string) (*API, error) {
 
 // GetPolicies lists the API policies.
 func (c *AnypointClient) GetPolicies(apiID int64) ([]Policy, error) {
-	policies := []Policy{}
+	policies := Policies{}
 	url := fmt.Sprintf("%s/apimanager/api/v1/organizations/%s/environments/%s/apis/%d/policies", c.baseURL, c.auth.GetOrgID(), c.environment.ID, apiID)
 	err := c.invokeJSONGet(url, nil, &policies, nil)
-
-	return policies, err
+	// Older versions of mulesoft may return []Policy JSON format instead.
+	if err != nil && strings.HasPrefix(err.Error(), "json: cannot unmarshal") {
+		err = c.invokeJSONGet(url, nil, &(policies.Policies), nil)
+	}
+	// Same issue, but with ConfigurationData and Configuration
+	for i, pCfg := range policies.Policies {
+		if pCfg.ConfigurationData != nil {
+			policies.Policies[i].Configuration = pCfg.Configuration
+		}
+	}
+	return policies.Policies, err
 }
 
 // GetExchangeAsset creates the AssetDetail form the Asset API.
@@ -333,33 +343,39 @@ func (c *AnypointClient) GetExchangeAssetIcon(icon string) (string, string, erro
 }
 
 // GetExchangeFileContent download the file from the ExternalLink reference. If the file is a zip file
-// and thre is a MainFile set then the content of the MainFile is returned.
-func (c *AnypointClient) GetExchangeFileContent(link, packaging, mainFile string) (fileContent []byte, err error) {
-	fileContent, _, err = c.invokeGet(link)
+// and there is a MainFile set then the content of the MainFile is returned.
+func (c *AnypointClient) GetExchangeFileContent(link, packaging, mainFile string, useOriginalRaml bool) ([]byte, bool, error) {
+	wasConverted := false
+	fileContent, _, err := c.invokeGet(link)
+	if packaging != "zip" {
+		return fileContent, wasConverted, err
+	}
+	zipReader, err := zip.NewReader(bytes.NewReader(fileContent), int64(len(fileContent)))
+	if err != nil {
+		return nil, wasConverted, err
+	}
 
-	if packaging == "zip" && mainFile != "" {
-		zipReader, err := zip.NewReader(bytes.NewReader(fileContent), int64(len(fileContent)))
+	for _, f := range zipReader.File {
+		// In case of RAML spec, this gets automatically converted and is renamed to api.json
+		if f.Name != mainFile && f.Name != "api.json" {
+			continue
+		}
+		content, err := f.Open()
 		if err != nil {
-			return nil, err
+			return nil, wasConverted, err
 		}
 
-		for _, f := range zipReader.File {
-			if f.Name == mainFile {
-				content, err := f.Open()
-				if err != nil {
-					return nil, err
-				}
-
-				fileContent, err = ioutil.ReadAll(content)
-				content.Close()
-				if err != nil {
-					return nil, err
-				}
-				break
-			}
+		fileContent, err = io.ReadAll(content)
+		content.Close()
+		if err != nil {
+			return nil, wasConverted, err
+		}
+		if !useOriginalRaml && f.Name == "api.json" {
+			return fileContent, true, err
 		}
+		break
 	}
-	return fileContent, err
+	return fileContent, wasConverted, err
 }
 
 // GetAnalyticsWindow lists the managed assets in Mulesoft: https://docs.qax.mulesoft.com/api-manager/2.x/analytics-event-api

pkg/anypoint/mocks.go

@@ -92,10 +92,10 @@ func (m *MockAnypointClient) GetExchangeAssetIcon(_ string) (string, string, err
 	return icon, contentType, args.Error(2)
 }
 
-func (m *MockAnypointClient) GetExchangeFileContent(_, _, _ string) ([]byte, error) {
+func (m *MockAnypointClient) GetExchangeFileContent(_, _, _ string, shouldConvert bool) ([]byte, bool, error) {
 	args := m.Called()
 	result := args.Get(0)
-	return result.([]byte), args.Error(1)
+	return result.([]byte), shouldConvert, args.Error(2)
 }
 
 func (m *MockAnypointClient) GetAnalyticsWindow() ([]AnalyticsEvent, error) {

pkg/anypoint/types.go

@@ -98,7 +98,8 @@ type API struct {
 type Policy struct {
 	// APIID                int64       `json:"apiId,omitempty"`
 	// Audit                Audit       `json:"audit,omitempty"`
-	Configuration interface{} `json:"configurationData,omitempty"`
+	Configuration     interface{} `json:"configuration,omitempty"`
+	ConfigurationData interface{} `json:"configurationData,omitempty"`
 	// ID                   int64       `json:"id,omitempty"`
 	// MasterOrganizationID string      `json:"masterOrganizationId,omitempty"`
 	// Order                int         `json:"order,omitempty"`

pkg/config/config.go

@@ -11,6 +11,17 @@ import (
 	corecfg "github.com/Axway/agent-sdk/pkg/config"
 )
 
+type props interface {
+	AddStringProperty(name string, defaultVal string, description string)
+	AddStringSliceProperty(name string, defaultVal []string, description string)
+	AddBoolProperty(name string, defaultVal bool, description string)
+	AddDurationProperty(name string, defaultVal time.Duration, description string, opts ...properties.DurationOpt)
+	StringPropertyValue(name string) string
+	StringSlicePropertyValue(name string) []string
+	BoolPropertyValue(name string) bool
+	DurationPropertyValue(name string) time.Duration
+}
+
 var config *AgentConfig
 
 const (
@@ -32,6 +43,19 @@ const (
 	pathPollInterval          = "mulesoft.pollInterval"
 	pathProxyURL              = "mulesoft.proxyUrl"
 	pathCachePath             = "mulesoft.cachePath"
+	pathDiscoverOriginalRaml  = "mulesoft.discoverOriginalRaml"
+)
+
+const (
+	anypointExchangeUrlErr = "invalid mulesoft configuration: anypointExchangeUrl is not configured"
+	usernameOrClientIDErr  = "invalid mulesoft configuration: username or client id must be configured"
+	usernameAndClientIDErr = "invalid mulesoft configuration: cannot use both non-empty username and client id"
+	passwordErr            = "invalid mulesoft configuration: password is not configured"
+	clientSecretErr        = "invalid mulesoft configuration: client secret is not configured"
+	envErr                 = "invalid mulesoft configuration: environment is not configured"
+	orgNameErr             = "invalid mulesoft configuration: OrgName is not configured"
+	pollIntervalErr        = "invalid mulesoft configuration: pollInterval is invalid"
+	cachePathErr           = "invalid mulesoft cache path: path does not exist: "
 )
 
 // SetConfig sets the global AgentConfig reference.
@@ -53,109 +77,112 @@ type AgentConfig struct {
 // MulesoftConfig - represents the config for the Mulesoft gateway
 type MulesoftConfig struct {
 	corecfg.IConfigValidator
-	AnypointExchangeURL string            `config:"anypointExchangeUrl"`
-	CachePath           string            `config:"cachePath"`
-	DiscoveryIgnoreTags string            `config:"discoveryIgnoreTags"`
-	DiscoveryTags       string            `config:"discoveryTags"`
-	Environment         string            `config:"environment"`
-	OrgName             string            `config:"orgname"`
-	Password            string            `config:"auth.password"`
-	PollInterval        time.Duration     `config:"pollInterval"`
-	ProxyURL            string            `config:"proxyUrl"`
-	SessionLifetime     time.Duration     `config:"auth.lifetime"`
-	TLS                 corecfg.TLSConfig `config:"ssl"`
-	Username            string            `config:"auth.username"`
-	ClientID            string            `config:"auth.clientID"`
-	ClientSecret        string            `config:"auth.clientSecret"`
+	AnypointExchangeURL  string            `config:"anypointExchangeUrl"`
+	CachePath            string            `config:"cachePath"`
+	DiscoveryIgnoreTags  string            `config:"discoveryIgnoreTags"`
+	DiscoveryTags        string            `config:"discoveryTags"`
+	Environment          string            `config:"environment"`
+	OrgName              string            `config:"orgname"`
+	Password             string            `config:"auth.password"`
+	PollInterval         time.Duration     `config:"pollInterval"`
+	ProxyURL             string            `config:"proxyUrl"`
+	SessionLifetime      time.Duration     `config:"auth.lifetime"`
+	TLS                  corecfg.TLSConfig `config:"ssl"`
+	Username             string            `config:"auth.username"`
+	ClientID             string            `config:"auth.clientID"`
+	ClientSecret         string            `config:"auth.clientSecret"`
+	DiscoverOriginalRaml bool              `config:"discoverOriginalRaml"`
 }
 
 // ValidateCfg - Validates the gateway config
 func (c *MulesoftConfig) ValidateCfg() (err error) {
 	if c.AnypointExchangeURL == "" {
-		return errors.New("invalid mulesoft configuration: anypointExchangeUrl is not configured")
+		return errors.New(anypointExchangeUrlErr)
 	}
 
 	if c.Username == "" && c.ClientID == "" {
-		return errors.New("invalid mulesoft configuration: username or client id must be configured")
+		return errors.New(usernameOrClientIDErr)
 	}
 
 	if c.Username != "" && c.ClientID != "" {
-		return errors.New("invalid mulesoft configuration: both username or client id can not be configured")
+		return errors.New(usernameAndClientIDErr)
 	}
 
 	if c.Username != "" && c.Password == "" {
-		return errors.New("invalid mulesoft configuration: password is not configured")
+		return errors.New(passwordErr)
 	}
 
 	if c.ClientID != "" && c.ClientSecret == "" {
-		return errors.New("invalid mulesoft configuration: client secret is not configured")
+		return errors.New(clientSecretErr)
 	}
 
 	if c.Environment == "" {
-		return errors.New("invalid mulesoft configuration: environment is not configured")
+		return errors.New(envErr)
 	}
 
 	if c.OrgName == "" {
-		return errors.New("invalid mulesoft configuration: OrgName is not configured")
+		return errors.New(orgNameErr)
 	}
 
 	if c.PollInterval == 0 {
-		return errors.New("invalid mulesoft configuration: pollInterval is invalid")
+		return errors.New(pollIntervalErr)
 	}
 
 	if _, err := os.Stat(c.CachePath); os.IsNotExist(err) {
-		return fmt.Errorf("invalid mulesoft cache path: path does not exist: %s", c.CachePath)
+		return fmt.Errorf(cachePathErr + c.CachePath)
 	}
 	c.CachePath = filepath.Clean(c.CachePath)
 	return
 }
 
 // AddConfigProperties - Adds the command properties needed for Mulesoft
-func AddConfigProperties(props properties.Properties) {
-	props.AddStringProperty(pathAnypointExchangeURL, "https://anypoint.mulesoft.com", "Mulesoft Anypoint Exchange URL.")
-	props.AddStringProperty(pathEnvironment, "", "Mulesoft Anypoint environment.")
-	props.AddStringProperty(pathOrgName, "", "Mulesoft Anypoint Business Group.")
-	props.AddStringProperty(pathAuthUsername, "", "Mulesoft username.")
-	props.AddStringProperty(pathAuthPassword, "", "Mulesoft password.")
-	props.AddStringProperty(pathAuthClientID, "", "Mulesoft client id.")
-	props.AddStringProperty(pathAuthClientSecret, "", "Mulesoft client secret.")
-	props.AddDurationProperty(pathAuthLifetime, 60*time.Minute, "Mulesoft session lifetime.")
-	props.AddStringProperty(pathDiscoveryTags, "", "APIs containing any of these tags are selected for discovery.")
-	props.AddStringProperty(pathDiscoveryIgnoreTags, "", "APIs containing any of these tags are ignored. Takes precedence over "+pathDiscoveryIgnoreTags+".")
-	props.AddStringProperty(pathCachePath, "/tmp", "Mulesoft Cache Path")
-	props.AddDurationProperty(pathPollInterval, 20*time.Second, "The interval at which Mulesoft is checked for updates.",
-		properties.WithLowerLimit(20*time.Second))
+func AddConfigProperties(rootProps props) {
+	rootProps.AddStringProperty(pathAnypointExchangeURL, "https://anypoint.mulesoft.com", "Mulesoft Anypoint Exchange URL.")
+	rootProps.AddStringProperty(pathEnvironment, "", "Mulesoft Anypoint environment.")
+	rootProps.AddStringProperty(pathOrgName, "", "Mulesoft Anypoint Business Group.")
+	rootProps.AddStringProperty(pathAuthUsername, "", "Mulesoft username.")
+	rootProps.AddStringProperty(pathAuthPassword, "", "Mulesoft password.")
+	rootProps.AddStringProperty(pathAuthClientID, "", "Mulesoft client id.")
+	rootProps.AddStringProperty(pathAuthClientSecret, "", "Mulesoft client secret.")
+	rootProps.AddDurationProperty(pathAuthLifetime, 60*time.Minute, "Mulesoft session lifetime.")
+	rootProps.AddStringProperty(pathDiscoveryTags, "", "APIs containing any of these tags are selected for discovery.")
+	rootProps.AddStringProperty(pathDiscoveryIgnoreTags, "", "APIs containing any of these tags are ignored. Takes precedence over "+pathDiscoveryIgnoreTags+".")
+	rootProps.AddStringProperty(pathCachePath, "/tmp", "Mulesoft Cache Path")
+	rootProps.AddDurationProperty(pathPollInterval, 20*time.Second, "The interval at which Mulesoft is checked for updates.", properties.WithLowerLimit(20*time.Second))
+	rootProps.AddStringProperty(pathProxyURL, "", "Proxy URL")
 
 	// ssl properties and command flags
-	props.AddStringSliceProperty(pathSSLNextProtos, []string{}, "List of supported application level protocols, comma separated.")
-	props.AddBoolProperty(pathSSLInsecureSkipVerify, false, "Controls whether a client verifies the server's certificate chain and host name.")
-	props.AddStringSliceProperty(pathSSLCipherSuites, corecfg.TLSDefaultCipherSuitesStringSlice(), "List of supported cipher suites, comma separated.")
-	props.AddStringProperty(pathSSLMinVersion, corecfg.TLSDefaultMinVersionString(), "Minimum acceptable SSL/TLS protocol version.")
-	props.AddStringProperty(pathSSLMaxVersion, "0", "Maximum acceptable SSL/TLS protocol version.")
+	rootProps.AddStringSliceProperty(pathSSLNextProtos, []string{}, "List of supported application level protocols, comma separated.")
+	rootProps.AddBoolProperty(pathSSLInsecureSkipVerify, false, "Controls whether a client verifies the server's certificate chain and host name.")
+	rootProps.AddStringSliceProperty(pathSSLCipherSuites, corecfg.TLSDefaultCipherSuitesStringSlice(), "List of supported cipher suites, comma separated.")
+	rootProps.AddStringProperty(pathSSLMinVersion, corecfg.TLSDefaultMinVersionString(), "Minimum acceptable SSL/TLS protocol version.")
+	rootProps.AddStringProperty(pathSSLMaxVersion, "0", "Maximum acceptable SSL/TLS protocol version.")
+	rootProps.AddBoolProperty(pathDiscoverOriginalRaml, false, "If RAML API specs are discovered as RAML and not converted to OAS")
 }
 
 // NewMulesoftConfig - parse the props and create an Mulesoft Configuration structure
-func NewMulesoftConfig(props properties.Properties) *MulesoftConfig {
+func NewMulesoftConfig(rootProps props) *MulesoftConfig {
 	return &MulesoftConfig{
-		AnypointExchangeURL: props.StringPropertyValue(pathAnypointExchangeURL),
-		CachePath:           props.StringPropertyValue(pathCachePath),
-		DiscoveryIgnoreTags: props.StringPropertyValue(pathDiscoveryIgnoreTags),
-		DiscoveryTags:       props.StringPropertyValue(pathDiscoveryTags),
-		Environment:         props.StringPropertyValue(pathEnvironment),
-		OrgName:             props.StringPropertyValue(pathOrgName),
-		Password:            props.StringPropertyValue(pathAuthPassword),
-		PollInterval:        props.DurationPropertyValue(pathPollInterval),
-		ProxyURL:            props.StringPropertyValue(pathProxyURL),
-		SessionLifetime:     props.DurationPropertyValue(pathAuthLifetime),
-		Username:            props.StringPropertyValue(pathAuthUsername),
-		ClientID:            props.StringPropertyValue(pathAuthClientID),
-		ClientSecret:        props.StringPropertyValue(pathAuthClientSecret),
+		AnypointExchangeURL: rootProps.StringPropertyValue(pathAnypointExchangeURL),
+		CachePath:           rootProps.StringPropertyValue(pathCachePath),
+		DiscoveryIgnoreTags: rootProps.StringPropertyValue(pathDiscoveryIgnoreTags),
+		DiscoveryTags:       rootProps.StringPropertyValue(pathDiscoveryTags),
+		Environment:         rootProps.StringPropertyValue(pathEnvironment),
+		OrgName:             rootProps.StringPropertyValue(pathOrgName),
+		Password:            rootProps.StringPropertyValue(pathAuthPassword),
+		PollInterval:        rootProps.DurationPropertyValue(pathPollInterval),
+		ProxyURL:            rootProps.StringPropertyValue(pathProxyURL),
+		SessionLifetime:     rootProps.DurationPropertyValue(pathAuthLifetime),
+		Username:            rootProps.StringPropertyValue(pathAuthUsername),
+		ClientID:            rootProps.StringPropertyValue(pathAuthClientID),
+		ClientSecret:        rootProps.StringPropertyValue(pathAuthClientSecret),
 		TLS: &corecfg.TLSConfiguration{
-			NextProtos:         props.StringSlicePropertyValue(pathSSLNextProtos),
-			InsecureSkipVerify: props.BoolPropertyValue(pathSSLInsecureSkipVerify),
-			CipherSuites:       corecfg.NewCipherArray(props.StringSlicePropertyValue(pathSSLCipherSuites)),
-			MinVersion:         corecfg.TLSVersionAsValue(props.StringPropertyValue(pathSSLMinVersion)),
-			MaxVersion:         corecfg.TLSVersionAsValue(props.StringPropertyValue(pathSSLMaxVersion)),
+			NextProtos:         rootProps.StringSlicePropertyValue(pathSSLNextProtos),
+			InsecureSkipVerify: rootProps.BoolPropertyValue(pathSSLInsecureSkipVerify),
+			CipherSuites:       corecfg.NewCipherArray(rootProps.StringSlicePropertyValue(pathSSLCipherSuites)),
+			MinVersion:         corecfg.TLSVersionAsValue(rootProps.StringPropertyValue(pathSSLMinVersion)),
+			MaxVersion:         corecfg.TLSVersionAsValue(rootProps.StringPropertyValue(pathSSLMaxVersion)),
 		},
+		DiscoverOriginalRaml: rootProps.BoolPropertyValue(pathDiscoverOriginalRaml),
 	}
 }