feat(integration): identify supported OAuth integrations through glob…

…al secrets (#791)

Because

- We want to let CE users set up OAuth configuration for components.
- Until now, OAuth was considered as supported based on the component
  definition.

This commit

- Consider OAuth supported when both the component definition and the
  component global secrets have a complete OAuth configuration.

.env.component

@@ -2,18 +2,24 @@
 
 # Provide your API key for the AI vendors so that you can set the components up
 # with default credentials.
-CFG_CONNECTOR_SECRETS_OPENAI_APIKEY=
-CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY=
-CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY=
-CFG_CONNECTOR_SECRETS_COHERE_APIKEY=
-CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY=
-CFG_CONNECTOR_SECRETS_GROQ_APIKEY=
-CFG_CONNECTOR_SECRETS_FIREWORKSAI_APIKEY=
+CFG_COMPONENT_SECRETS_OPENAI_APIKEY=
+CFG_COMPONENT_SECRETS_STABILITYAI_APIKEY=
+CFG_COMPONENT_SECRETS_ANTHROPIC_APIKEY=
+CFG_COMPONENT_SECRETS_COHERE_APIKEY=
+CFG_COMPONENT_SECRETS_MISTRALAI_APIKEY=
+CFG_COMPONENT_SECRETS_GROQ_APIKEY=
+CFG_COMPONENT_SECRETS_FIREWORKSAI_APIKEY=
 
 # Numbers Protocol API key.
-CFG_CONNECTOR_SECRETS_NUMBERS_XAPIKEY=
+CFG_COMPONENT_SECRETS_NUMBERS_XAPIKEY=
 
 # OAuth secrets. When these are filled, the specified component will support
 # OAuth integrations.
-CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTID=
-CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTSECRET=
+CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTID=
+CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTSECRET=
+CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTID=
+CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTSECRET=
+# Dummy values are added here to check OAuth connection creation on integration
+# tests.
+CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTID=dummy-id
+CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTSECRET=dummy-secret

.github/workflows/integration-test.yml

@@ -78,7 +78,10 @@ jobs:
           envFile: .env
 
       - name: Launch Instill Core (latest)
+        # CFG_COMPONENT_SECRETS_GITHUB* variables are injected to test OAuth connection
+        # creation
         run: |
+          sed -i 's/\(\w\+GITHUB\w\+\)=/\1=foo/' .env.component
           COMPOSE_PROFILES=all \
           EDITION=local-ce:test \
           RAY_LATEST_TAG=latest \

cmd/main/main.go

@@ -256,7 +256,7 @@ func main() {
 		logger.Fatal("failed to create minio client", zap.Error(err))
 	}
 	workerUID, _ := uuid.NewV4()
-	compStore := componentstore.Init(logger, config.Config.Connector.Secrets, nil)
+	compStore := componentstore.Init(logger, config.Config.Component.Secrets, nil)
 
 	service := service.NewService(
 		repo,

config/config.go

@@ -24,7 +24,7 @@ var Config AppConfig
 // AppConfig defines
 type AppConfig struct {
 	Server          ServerConfig          `koanf:"server"`
-	Connector       ConnectorConfig       `koanf:"connector"`
+	Component       ComponentConfig       `koanf:"component"`
 	Database        DatabaseConfig        `koanf:"database"`
 	InfluxDB        InfluxDBConfig        `koanf:"influxdb"`
 	Temporal        TemporalConfig        `koanf:"temporal"`
@@ -83,8 +83,11 @@ type ServerConfig struct {
 	InstillCoreHost    string `koanf:"instillcorehost"`
 }
 
-// ConnectorConfig defines the connector configurations
-type ConnectorConfig struct {
+// ComponentConfig contains the configuration of different components. Global
+// secrets may be defined here by component, allowing them to have e.g. a
+// default API key when no setup is specified, or to connect with a 3rd party
+// vendor via OAuth.
+type ComponentConfig struct {
 	Secrets componentstore.ComponentSecrets
 }
 

config/config.yaml

@@ -19,7 +19,7 @@ server:
   instanceid: "pipeline-backend"
   datachanbuffersize: 100
   instillcorehost: http://localhost:8080
-connector:
+component:
 database:
   username: postgres
   password: password

integration-test/pipeline/rest-integration.js

@@ -131,6 +131,11 @@ export function CheckConnections(data) {
       [`POST ${path} (dictionary) has a creation time`]: (r) => new Date(r.json().connection.createTime).getTime() > new Date().setTime(0),
     });
 
+    // Besides an OAuth configuration on the component definition, OAuth
+    // support requires the client ID and secret to be defined in the config
+    // (as environment variables). Make sure .env.component contains a client
+    // secret and ID for GitHub and that it doesn't for Slack.
+
     // Successful creation: OAuth
     var oAuthReq = http.request(
       "POST",
@@ -157,6 +162,30 @@ export function CheckConnections(data) {
       [`POST ${path} (OAuth) has a creation time`]: (r) => new Date(r.json().connection.createTime).getTime() > new Date().setTime(0),
     });
 
+    // Check OAuth support.
+    var unsupportedOAuthReq = http.request(
+      "POST",
+      pipelinePublicHost + path,
+      JSON.stringify({
+        id: "unsupported-oauth",
+        integrationId: "slack",
+        method: "METHOD_OAUTH",
+        setup: setup,
+        scopes: ["foo", "bar"],
+        identity: identity,
+        oAuthAccessDetails: {
+          access_token: "one2THREE",
+          scope: "foo,bar",
+          token_type: "bearer",
+        }
+      }),
+      data.header
+    );
+    check(unsupportedOAuthReq, {
+      [`POST ${path} response status is 400 when component lacks client ID and secret`]: (r) => r.status === 400,
+    });
+
+
     // Check ID format
     var invalidID = dbIDPrefix + "This-Is-Invalid";
     var invalidIDReq = http.request(
@@ -180,7 +209,7 @@ export function CheckConnections(data) {
       "POST",
       pipelinePublicHost + path,
       JSON.stringify({
-        id: invalidID,
+        id: "invalid-setup",
         integrationId: integrationID,
         method: "METHOD_OAUTH",
         setup: {"token": 234},
@@ -197,7 +226,7 @@ export function CheckConnections(data) {
       "POST",
       pipelinePublicHost + path,
       JSON.stringify({
-        id: invalidID,
+        id: "invalid-method",
         integrationId: integrationID,
         method: "METHOD_DICTIONARY",
         setup: {"token": 234},
@@ -371,7 +400,7 @@ component:
   });
 
   group("Integration API: Update connection", () => {
-    var path = resourcePath;
+    var path = resourcePath + "-oauth";
     var originalConn = http.request(
       "GET",
       pipelinePublicHost + path,

pkg/component/application/github/v0/component_test.go

@@ -837,3 +837,35 @@ func taskTesting[inType any, outType any](testcases []TaskCase[inType, outType],
 		})
 	}
 }
+
+func TestComponent_WithOAuthConfig(t *testing.T) {
+	c := qt.New(t)
+
+	bc := base.Component{}
+
+	test := func(name string, conf map[string]any, check qt.Checker) {
+		c.Run(name, func(c *qt.C) {
+			cmp := Init(bc)
+			cmp.WithOAuthConfig(conf)
+			c.Check(cmp.SupportsOAuth(), check)
+		})
+	}
+
+	newConf := func(clientID, clientSecret string) map[string]any {
+		conf := map[string]any{}
+		if clientID != "" {
+			conf["oauthclientid"] = clientID
+		}
+
+		if clientSecret != "" {
+			conf["oauthclientsecret"] = clientSecret
+		}
+
+		return conf
+	}
+
+	test("ok - with OAuth details", newConf("foo", "bar"), qt.IsTrue)
+	test("ok - without OAuth secret", newConf("foo", ""), qt.IsFalse)
+	test("ok - without OAuth ID", newConf("", "bar"), qt.IsFalse)
+	test("ok - without OAuth ID", newConf("", ""), qt.IsFalse)
+}

pkg/component/application/github/v0/main.go

@@ -39,6 +39,7 @@ var (
 
 type component struct {
 	base.Component
+	base.OAuthConnector
 }
 
 type execution struct {
@@ -139,3 +140,8 @@ func (c *component) ParseEvent(ctx context.Context, req *structpb.Struct, setup
 	// TODO: parse and validate event
 	return req, nil
 }
+
+// SupportsOAuth checks whether the component is configured to support OAuth.
+func (c *component) SupportsOAuth() bool {
+	return c.OAuthConnector.SupportsOAuth()
+}

pkg/component/application/slack/v0/main.go

@@ -46,6 +46,7 @@ type SlackClient interface {
 
 type component struct {
 	base.Component
+	base.OAuthConnector
 }
 
 type execution struct {
@@ -138,3 +139,8 @@ func (c *component) ParseEvent(ctx context.Context, req *structpb.Struct, setup
 	// TODO: parse and validate event
 	return req, nil
 }
+
+// SupportsOAuth checks whether the component is configured to support OAuth.
+func (c *component) SupportsOAuth() bool {
+	return c.OAuthConnector.SupportsOAuth()
+}

pkg/component/base/component.go

@@ -84,6 +84,7 @@ type IComponent interface {
 	Test(sysVars map[string]any, config *structpb.Struct) error
 
 	IsSecretField(target string) bool
+	SupportsOAuth() bool
 
 	// Note: These two functions are for the pipeline run-on-event feature,
 	// which is still experimental and may change at any time.
@@ -831,6 +832,12 @@ func (c *Component) traverseSecretField(input *structpb.Value, prefix string, se
 	return secretFields
 }
 
+// SupportsOAuth is false by default. To support OAuth, component
+// implementations must be composed with `OAuthComponent`.
+func (c *Component) SupportsOAuth() bool {
+	return false
+}
+
 func (c *Component) ListInputAcceptFormatsFields() (map[string]map[string][]string, error) {
 	return c.inputAcceptFormatsFields, nil
 }

pkg/component/base/oauth.go

@@ -0,0 +1,30 @@
+package base
+
+// OAuthConnector contains the OAuth configuration that a comopnent can use to
+// support OAuth 2.0 connections. Such components must have an
+// `instillOAuthConfig` object in their setup definition.
+type OAuthConnector struct {
+	oAuthClientID     string
+	oAuthClientSecret string
+}
+
+const (
+	cfgOAuthClientID     = "oauth-client-id"
+	cfgOAuthClientSecret = "oauth-client-secret"
+)
+
+// WithOAuthConfig loads the OAuth 2.0 connection details into the connector,
+// which can be used to determine if the Instill AI deployment supports OAuth
+// connections for a given component.
+// TODO jvallesm: this is a prerequisite for supporting refresh token when the
+// component execution uses an OAuth connection.
+func (c *OAuthConnector) WithOAuthConfig(s map[string]any) {
+	c.oAuthClientID = ReadFromGlobalConfig(cfgOAuthClientID, s)
+	c.oAuthClientSecret = ReadFromGlobalConfig(cfgOAuthClientSecret, s)
+
+}
+
+// SupportsOAuth checks whether the connector is configured to support OAuth.
+func (c *OAuthConnector) SupportsOAuth() bool {
+	return c.oAuthClientID != "" && c.oAuthClientSecret != ""
+}

pkg/component/store/store.go

@@ -106,7 +106,6 @@ func Init(
 		compStore.Import(audio.Init(baseComp))
 		compStore.Import(video.Init(baseComp))
 
-		compStore.Import(github.Init(baseComp))
 		{
 			// StabilityAI
 			conn := stabilityai.Init(baseComp)
@@ -190,7 +189,18 @@ func Init(
 		compStore.Import(restapi.Init(baseComp))
 		compStore.Import(collection.Init(baseComp))
 		compStore.Import(web.Init(baseComp))
-		compStore.Import(slack.Init(baseComp))
+		{
+			// GitHub
+			conn := github.Init(baseComp)
+			conn.WithOAuthConfig(secrets[conn.GetDefinitionID()])
+			compStore.Import(conn)
+		}
+		{
+			// Slack
+			conn := slack.Init(baseComp)
+			conn.WithOAuthConfig(secrets[conn.GetDefinitionID()])
+			compStore.Import(conn)
+		}
 		compStore.Import(email.Init(baseComp))
 		compStore.Import(jira.Init(baseComp))
 		compStore.Import(ollama.Init(baseComp))
@@ -298,11 +308,25 @@ func (s *Store) ListDefinitions(sysVars map[string]any, returnTombstone bool) []
 	return defs
 }
 
+// IsSecretField checks whether a property in a component definition is a
+// secret field.
 func (s *Store) IsSecretField(defUID uuid.UUID, target string) (bool, error) {
-	if c, ok := s.componentUIDMap[defUID]; ok {
-		return c.comp.IsSecretField(target), nil
+	c, ok := s.componentUIDMap[defUID]
+	if !ok {
+		return false, ErrComponentDefinitionNotFound
 	}
-	return false, ErrComponentDefinitionNotFound
+
+	return c.comp.IsSecretField(target), nil
+}
+
+// SupportsOAuth checks whether a component supports OAuth connections.
+func (s *Store) SupportsOAuth(defUID uuid.UUID) (bool, error) {
+	c, ok := s.componentUIDMap[defUID]
+	if !ok {
+		return false, ErrComponentDefinitionNotFound
+	}
+
+	return c.comp.SupportsOAuth(), nil
 }
 
 // ErrComponentDefinitionNotFound is returned when trying to access an