Support custom headers during plugin index request (#1381)

* Support custom headers during plugin index request

* Add ability to get JSON schema for index entry

* Use forked go-getter to fix the problem with additional query params

* Update go-getter pkg

* Add comments in the `values.yaml` file

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/gookit/color v1.5.2
 	github.com/gorilla/mux v1.8.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
-	github.com/hashicorp/go-getter v1.7.1
+	github.com/hashicorp/go-getter v1.7.3
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-plugin v1.4.10
@@ -280,3 +280,5 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace github.com/hashicorp/go-getter v1.7.3 => github.com/kubeshop/go-getter v0.0.0-20240219121353-061e752b1b28

go.sum

@@ -694,8 +694,6 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY=
-github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
@@ -819,6 +817,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kubeshop/go-getter v0.0.0-20240219121353-061e752b1b28 h1:cIyWkSZbkn/7VU7/T375t9n7esO8/bruaJ82FIk4Gs4=
+github.com/kubeshop/go-getter v0.0.0-20240219121353-061e752b1b28/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=

helm/botkube/README.md

@@ -270,17 +270,17 @@ Controller for the Botkube Slack app which helps you monitor your Kubernetes clu
 | [configWatcher.inCluster.informerResyncPeriod](./values.yaml#L1201) | string | `"10m"` | Resync period for the Config Watcher informers. |
 | [plugins](./values.yaml#L1204) | object | `{"cacheDir":"/tmp","healthCheckInterval":"10s","incomingWebhook":{"enabled":true,"port":2115,"targetPort":2115},"repositories":{"botkube":{"url":"https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml"}},"restartPolicy":{"threshold":10,"type":"DeactivatePlugin"}}` | Configuration for Botkube executors and sources plugins. |
 | [plugins.cacheDir](./values.yaml#L1206) | string | `"/tmp"` | Directory, where downloaded plugins are cached. |
-| [plugins.repositories](./values.yaml#L1208) | object | `{"botkube":{"url":"https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml"}}` | List of plugins repositories. |
+| [plugins.repositories](./values.yaml#L1208) | object | `{"botkube":{"url":"https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml"}}` | List of plugins repositories. Each repository defines the URL and optional `headers` |
 | [plugins.repositories.botkube](./values.yaml#L1210) | object | `{"url":"https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml"}` | This repository serves officially supported Botkube plugins. |
-| [plugins.incomingWebhook](./values.yaml#L1213) | object | `{"enabled":true,"port":2115,"targetPort":2115}` | Configure Incoming webhook for source plugins. |
-| [plugins.restartPolicy](./values.yaml#L1218) | object | `{"threshold":10,"type":"DeactivatePlugin"}` | Botkube Restart Policy on plugin failure. |
-| [plugins.restartPolicy.type](./values.yaml#L1220) | string | `"DeactivatePlugin"` | Restart policy type. Allowed values: "RestartAgent", "DeactivatePlugin". |
-| [plugins.restartPolicy.threshold](./values.yaml#L1222) | int | `10` | Number of restarts before policy takes into effect. |
-| [config](./values.yaml#L1226) | object | `{"provider":{"apiKey":"","endpoint":"https://api.botkube.io/graphql","identifier":""}}` | Configuration for synchronizing Botkube configuration. |
-| [config.provider](./values.yaml#L1228) | object | `{"apiKey":"","endpoint":"https://api.botkube.io/graphql","identifier":""}` | Base provider definition. |
-| [config.provider.identifier](./values.yaml#L1231) | string | `""` | Unique identifier for remote Botkube settings. If set to an empty string, Botkube won't fetch remote configuration. |
-| [config.provider.endpoint](./values.yaml#L1233) | string | `"https://api.botkube.io/graphql"` | Endpoint to fetch Botkube settings from. |
-| [config.provider.apiKey](./values.yaml#L1235) | string | `""` | Key passed as a `X-API-Key` header to the provider's endpoint. |
+| [plugins.incomingWebhook](./values.yaml#L1215) | object | `{"enabled":true,"port":2115,"targetPort":2115}` | Configure Incoming webhook for source plugins. |
+| [plugins.restartPolicy](./values.yaml#L1220) | object | `{"threshold":10,"type":"DeactivatePlugin"}` | Botkube Restart Policy on plugin failure. |
+| [plugins.restartPolicy.type](./values.yaml#L1222) | string | `"DeactivatePlugin"` | Restart policy type. Allowed values: "RestartAgent", "DeactivatePlugin". |
+| [plugins.restartPolicy.threshold](./values.yaml#L1224) | int | `10` | Number of restarts before policy takes into effect. |
+| [config](./values.yaml#L1228) | object | `{"provider":{"apiKey":"","endpoint":"https://api.botkube.io/graphql","identifier":""}}` | Configuration for synchronizing Botkube configuration. |
+| [config.provider](./values.yaml#L1230) | object | `{"apiKey":"","endpoint":"https://api.botkube.io/graphql","identifier":""}` | Base provider definition. |
+| [config.provider.identifier](./values.yaml#L1233) | string | `""` | Unique identifier for remote Botkube settings. If set to an empty string, Botkube won't fetch remote configuration. |
+| [config.provider.endpoint](./values.yaml#L1235) | string | `"https://api.botkube.io/graphql"` | Endpoint to fetch Botkube settings from. |
+| [config.provider.apiKey](./values.yaml#L1237) | string | `""` | Key passed as a `X-API-Key` header to the provider's endpoint. |
 
 ### AWS IRSA on EKS support
 

helm/botkube/values.yaml

@@ -1204,11 +1204,13 @@ configWatcher:
 plugins:
   # -- Directory, where downloaded plugins are cached.
   cacheDir: "/tmp"
-  # -- List of plugins repositories.
+  # -- List of plugins repositories. Each repository defines the URL and optional `headers`
   repositories:
     # -- This repository serves officially supported Botkube plugins.
     botkube:
       url: https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml
+      # headers: {} # optional headers for plugins repository.
+
   # -- Configure Incoming webhook for source plugins.
   incomingWebhook:
     enabled: true

internal/cli/migrate/converter.go

@@ -112,7 +112,8 @@ func (c *Converter) convertExecutors(executors map[string]bkconfig.Executors) ([
 
 			rawCfg, err := json.Marshal(p.Config)
 			if err != nil {
-				return nil, err
+				errs = multierror.Append(errs, fmt.Errorf("while marshalling config for executor %q: %w", name, err))
+				continue
 			}
 			displayName := conf.DisplayName
 			if displayName == "" {
@@ -148,7 +149,8 @@ func (c *Converter) convertSources(sources map[string]bkconfig.Sources) ([]*gqlM
 			}
 			rawCfg, err := json.Marshal(p.Config)
 			if err != nil {
-				return nil, err
+				errs = multierror.Append(errs, fmt.Errorf("while marshalling config for source %q: %w", name, err))
+				continue
 			}
 			displayName := conf.DisplayName
 			if displayName == "" {

pkg/config/config.go

@@ -143,11 +143,11 @@ type Config struct {
 
 // PluginManagement holds Botkube plugin management related configuration.
 type PluginManagement struct {
-	CacheDir            string                         `yaml:"cacheDir"`
-	Repositories        map[string]PluginsRepositories `yaml:"repositories"`
-	IncomingWebhook     IncomingWebhook                `yaml:"incomingWebhook"`
-	RestartPolicy       PluginRestartPolicy            `yaml:"restartPolicy"`
-	HealthCheckInterval time.Duration                  `yaml:"healthCheckInterval"`
+	CacheDir            string                       `yaml:"cacheDir"`
+	Repositories        map[string]PluginsRepository `yaml:"repositories"`
+	IncomingWebhook     IncomingWebhook              `yaml:"incomingWebhook"`
+	RestartPolicy       PluginRestartPolicy          `yaml:"restartPolicy"`
+	HealthCheckInterval time.Duration                `yaml:"healthCheckInterval"`
 }
 
 type PluginRestartPolicy struct {
@@ -166,9 +166,10 @@ func (p PluginRestartPolicyType) ToLower() string {
 	return strings.ToLower(string(p))
 }
 
-// PluginsRepositories holds the Plugin repository information.
-type PluginsRepositories struct {
-	URL string `yaml:"url"`
+// PluginsRepository holds the Plugin repository information.
+type PluginsRepository struct {
+	URL     string `yaml:"url"`
+	Headers map[string]string
 }
 
 // IncomingWebhook contains configuration for incoming source webhook.

pkg/config/testdata/TestLoadConfigSuccess/config.golden.yaml

@@ -178,6 +178,7 @@ plugins:
     repositories:
         botkube:
             url: http://localhost:3000/botkube.yaml
+            headers: {}
     incomingWebhook:
         enabled: false
         port: 0

pkg/plugin/downloader.go

@@ -3,6 +3,7 @@ package plugin
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -16,7 +17,7 @@ var allowedExt = map[string]struct{}{
 }
 
 // downloadBinary downloads binary into specific destination.
-func downloadBinary(ctx context.Context, destPath string, url URL, autoDetectFilename bool) error {
+func downloadBinary(ctx context.Context, destPath string, binaryURL URL, autoDetectFilename bool) error {
 	dir, filename := filepath.Split(destPath)
 	err := os.MkdirAll(dir, dirPerms)
 	if err != nil {
@@ -35,10 +36,18 @@ func downloadBinary(ctx context.Context, destPath string, url URL, autoDetectFil
 		}
 	}
 
-	urlWithGoGetterMagicParams := fmt.Sprintf("%s?filename=%s", url.URL, filename)
-	if url.Checksum != "" {
-		urlWithGoGetterMagicParams = fmt.Sprintf("%s&checksum=%s", urlWithGoGetterMagicParams, url.Checksum)
+	parsedURL, err := url.Parse(binaryURL.URL)
+	if err != nil {
+		return fmt.Errorf("while parsing URL %q: %w", binaryURL.URL, err)
+	}
+	// Add go-getter magic params
+	queryParams := parsedURL.Query()
+	queryParams.Set("filename", filename)
+	if binaryURL.Checksum != "" {
+		queryParams.Set("checksum", binaryURL.Checksum)
 	}
+	parsedURL.RawQuery = queryParams.Encode()
+	urlWithGoGetterMagicParams := parsedURL.String()
 
 	getterCli := &getter.Client{
 		Ctx:  ctx,
@@ -50,11 +59,11 @@ func downloadBinary(ctx context.Context, destPath string, url URL, autoDetectFil
 
 	err = getterCli.Get()
 	if err != nil {
-		return fmt.Errorf("while downloading binary from URL %q: %w", url, err)
+		return fmt.Errorf("while downloading binary with go-getter via url %s: %w", urlWithGoGetterMagicParams, err)
 	}
 
 	if stat, err := os.Stat(tmpDestPath); err == nil && stat.IsDir() {
-		if autoDetectFilename && hasArchiveExtension(url.URL) {
+		if autoDetectFilename && hasArchiveExtension(parsedURL.Path) {
 			filename, err = getFirstFileInDirectory(tmpDestPath)
 			if err != nil {
 				return fmt.Errorf("while getting binary name")

pkg/plugin/index.go

@@ -1,8 +1,11 @@
 package plugin
 
 import (
+	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 
 	"github.com/dustin/go-humanize"
@@ -154,3 +157,32 @@ func (in Index) Validate() error {
 
 	return issues.ErrorOrNil()
 }
+
+// Get returns the JSON schema.
+// When RefURL is not empty, Get tries to fetch the schema from the URL.
+func (s *JSONSchema) Get(ctx context.Context, httpCli *http.Client) (json.RawMessage, error) {
+	if s.Value != "" {
+		return json.RawMessage(s.Value), nil
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.RefURL, http.NoBody)
+	if err != nil {
+		return nil, fmt.Errorf("while creating request: %w", err)
+	}
+
+	res, err := httpCli.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("while fetching JSON schema by RefURL %q: %w", s.RefURL, err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("while getting JSON schema: got status code: %d", res.StatusCode)
+	}
+
+	var out json.RawMessage
+	if err := json.NewDecoder(res.Body).Decode(&out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}

pkg/plugin/manager.go

@@ -16,14 +16,17 @@ import (
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/sirupsen/logrus"
+	stringutil "k8s.io/utils/strings"
 
+	"github.com/kubeshop/botkube/internal/config/remote"
 	"github.com/kubeshop/botkube/pkg/api"
 	"github.com/kubeshop/botkube/pkg/api/executor"
 	"github.com/kubeshop/botkube/pkg/api/source"
 	"github.com/kubeshop/botkube/pkg/config"
 	"github.com/kubeshop/botkube/pkg/formatx"
 	"github.com/kubeshop/botkube/pkg/httpx"
 	"github.com/kubeshop/botkube/pkg/multierror"
+	"github.com/kubeshop/botkube/pkg/templatex"
 )
 
 const (
@@ -35,6 +38,7 @@ const (
 	DependencyDirEnvName = "PLUGIN_DEPENDENCY_DIR"
 
 	defaultHealthCheckInterval = 10 * time.Second
+	printHeaderValueCharCount  = 3
 )
 
 // pluginMap is the map of plugins we can dispense.
@@ -45,13 +49,19 @@ var pluginMap = map[string]plugin.Plugin{
 	TypeExecutor.String(): &executor.Plugin{},
 }
 
+// IndexRenderData returns plugin index render data.
+type IndexRenderData struct {
+	Remote remote.Config `yaml:"remote"`
+}
+
 // Manager provides functionality for managing executor and source plugins.
 type Manager struct {
-	isStarted  atomic.Bool
-	log        logrus.FieldLogger
-	logConfig  config.Logger
-	cfg        config.PluginManagement
-	httpClient *http.Client
+	isStarted       atomic.Bool
+	log             logrus.FieldLogger
+	logConfig       config.Logger
+	cfg             config.PluginManagement
+	httpClient      *http.Client
+	indexRenderData IndexRenderData
 
 	sourceSupervisorChan   chan pluginMetadata
 	executorSupervisorChan chan pluginMetadata
@@ -79,9 +89,15 @@ func NewManager(logger logrus.FieldLogger, logCfg config.Logger, cfg config.Plug
 	executorsStore := newStore[executor.Executor]()
 	sourcesStore := newStore[source.Source]()
 
+	remoteCfg, _ := remote.GetConfig()
+	indexRenderData := IndexRenderData{
+		Remote: remoteCfg,
+	}
+
 	return &Manager{
 		cfg:                    cfg,
 		httpClient:             httpx.NewHTTPClient(),
+		indexRenderData:        indexRenderData,
 		sourceSupervisorChan:   sourceSupervisorChan,
 		executorSupervisorChan: executorSupervisorChan,
 		schedulerChan:          schedulerChan,
@@ -310,7 +326,7 @@ func (m *Manager) loadRepositoriesMetadata(ctx context.Context, forceUpdate bool
 				"forceUpdate": forceUpdate,
 			}).Info("Downloading repository index")
 
-			err := m.fetchIndex(ctx, path, entry.URL)
+			err := m.fetchIndex(ctx, path, entry)
 			if err != nil {
 				return fmt.Errorf("while fetching index for %q repository with URL %q: %w", repo, entry.URL, err)
 			}
@@ -334,12 +350,28 @@ func (m *Manager) loadRepositoriesMetadata(ctx context.Context, forceUpdate bool
 	return nil
 }
 
-func (m *Manager) fetchIndex(ctx context.Context, path, url string) error {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
+func (m *Manager) fetchIndex(ctx context.Context, path string, repo config.PluginsRepository) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, repo.URL, http.NoBody)
 	if err != nil {
 		return fmt.Errorf("while creating request: %w", err)
 	}
 
+	headers, err := m.renderPluginIndexHeaders(repo.Headers)
+	if err != nil {
+		return fmt.Errorf("while rendering plugin index header: %w", err)
+	}
+
+	var strBuilder strings.Builder
+	for key, value := range headers {
+		strBuilder.WriteString(fmt.Sprintf("%s=%s\n", key, stringutil.ShortenString(value, printHeaderValueCharCount)))
+		req.Header.Set(key, value)
+	}
+
+	m.log.WithFields(logrus.Fields{
+		"headers": strBuilder.String(),
+		"url":     repo.URL,
+	}).Debug("Fetching index via GET request...")
+
 	res, err := m.httpClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("while executing request: %w", err)
@@ -367,6 +399,23 @@ func (m *Manager) fetchIndex(ctx context.Context, path, url string) error {
 	return nil
 }
 
+func (m *Manager) renderPluginIndexHeaders(headers map[string]string) (map[string]string, error) {
+	out := make(map[string]string)
+
+	errs := multierror.New()
+	for key, value := range headers {
+		renderedValue, err := templatex.RenderStringIfTemplate(value, m.indexRenderData)
+		if err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("while rendering header %q: %w", key, err))
+			continue
+		}
+
+		out[key] = renderedValue
+	}
+
+	return out, errs.ErrorOrNil()
+}
+
 func createGRPCClients[C any](ctx context.Context, logger logrus.FieldLogger, logConfig config.Logger, pluginMeta map[string]pluginMetadata, pluginType Type, supervisorChan chan pluginMetadata, healthCheckInterval time.Duration) (*storePlugins[C], error) {
 	out := map[string]enabledPlugins[C]{}
 	for key, pm := range pluginMeta {
@@ -496,7 +545,7 @@ func (m *Manager) ensurePluginDownloaded(ctx context.Context, binPath string, in
 
 		err = downloadBinary(ctx, binPath, url, true)
 		if err != nil {
-			return fmt.Errorf("while downloading dependency from URL %q: %w", url, err)
+			return fmt.Errorf("while downloading dependency from URL %q (checksum: %q): %w", url.URL, url.Checksum, err)
 		}
 	}