diff --git a/cmd/resolvers/main.go b/cmd/resolvers/main.go index 641c407785e..d4f445896be 100644 --- a/cmd/resolvers/main.go +++ b/cmd/resolvers/main.go @@ -25,6 +25,7 @@ import ( "github.com/tektoncd/pipeline/pkg/resolution/resolver/cluster" "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" "github.com/tektoncd/pipeline/pkg/resolution/resolver/git" + "github.com/tektoncd/pipeline/pkg/resolution/resolver/http" "github.com/tektoncd/pipeline/pkg/resolution/resolver/hub" filteredinformerfactory "knative.dev/pkg/client/injection/kube/informers/factory/filtered" "knative.dev/pkg/injection/sharedmain" @@ -40,7 +41,8 @@ func main() { framework.NewController(ctx, &git.Resolver{}), framework.NewController(ctx, &hub.Resolver{TektonHubURL: tektonHubURL, ArtifactHubURL: artifactHubURL}), framework.NewController(ctx, &bundle.Resolver{}), - framework.NewController(ctx, &cluster.Resolver{})) + framework.NewController(ctx, &cluster.Resolver{}), + framework.NewController(ctx, &http.Resolver{})) } func buildHubURL(configAPI, defaultURL, yamlEndpoint string) string { diff --git a/config/resolvers/http-resolver-config.yaml b/config/resolvers/http-resolver-config.yaml new file mode 100644 index 00000000000..47ad9772f48 --- /dev/null +++ b/config/resolvers/http-resolver-config.yaml @@ -0,0 +1,26 @@ +# Copyright 2023 The Tekton Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: http-resolver-config + namespace: tekton-pipelines-resolvers + labels: + app.kubernetes.io/component: resolvers + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: tekton-pipelines +data: + # The maximum amount of time the http resolver will wait for a response from the server. + fetch-timeout: "1m" diff --git a/docs/http-resolver.md b/docs/http-resolver.md new file mode 100644 index 00000000000..23e5217d14f --- /dev/null +++ b/docs/http-resolver.md @@ -0,0 +1,77 @@ + + +# HTTP Resolver + +This resolver responds to type `http`. + +## Parameters + +| Param Name | Description | Example Value | +|------------------|-------------------------------------------------------------------------------|------------------------------------------------------------| +| `url` | The URL to fetch from | https://raw.githubusercontent.com/tektoncd-catalog/git-clone/main/task/git-clone/git-clone.yaml | + +A valid URL must be provided. Only HTTP or HTTPS URLs are supported. + +## Requirements + +- A cluster running Tekton Pipeline v0.41.0 or later. +- The [built-in remote resolvers installed](./install.md#installing-and-configuring-remote-task-and-pipeline-resolution). +- The `enable-http-resolver` feature flag in the `resolvers-feature-flags` ConfigMap in the + `tekton-pipelines-resolvers` namespace set to `true`. +- [Beta features](./additional-configs.md#beta-features) enabled. + +## Configuration + +This resolver uses a `ConfigMap` for its settings. See +[`../config/resolvers/http-resolver-config.yaml`](../config/resolvers/http-resolver-config.yaml) +for the name, namespace and defaults that the resolver ships with. + +### Options + +| Option Name | Description | Example Values | +|-----------------------------|------------------------------------------------------|------------------------| +| `fetch-timeout` | The maximum time any fetching of URL resolution may take. **Note**: a global maximum timeout of 1 minute is currently enforced on _all_ resolution requests. | `1m`, `2s`, `700ms` | + +## Usage + +### Task Resolution + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: remote-task-reference +spec: + taskRef: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/tektoncd-catalog/git-clone/main/task/git-clone/git-clone.yaml +``` + +### Pipeline Resolution + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: http-demo +spec: + pipelineRef: + resolver: http + params: + - name: url # optional + value: https://raw.githubusercontent.com/tektoncd/catalog/main/pipeline/build-push-gke-deploy/0.1/build-push-gke-deploy.yaml +``` + +--- + +Except as otherwise noted, the content of this page is licensed under the +[Creative Commons Attribution 4.0 License](https://creativecommons.org/licenses/by/4.0/), +and code samples are licensed under the +[Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0). diff --git a/pkg/apis/config/resolver/feature_flags.go b/pkg/apis/config/resolver/feature_flags.go index f76ddfdd692..f6fa4db207d 100644 --- a/pkg/apis/config/resolver/feature_flags.go +++ b/pkg/apis/config/resolver/feature_flags.go @@ -33,6 +33,8 @@ const ( DefaultEnableBundlesResolver = true // DefaultEnableClusterResolver is the default value for "enable-cluster-resolver". DefaultEnableClusterResolver = true + // DefaultEnableHttpResolver is the default value for "enable-http-resolver". + DefaultEnableHttpResolver = true // EnableGitResolver is the flag used to enable the git remote resolver EnableGitResolver = "enable-git-resolver" @@ -42,6 +44,8 @@ const ( EnableBundlesResolver = "enable-bundles-resolver" // EnableClusterResolver is the flag used to enable the cluster remote resolver EnableClusterResolver = "enable-cluster-resolver" + // EnableHttpResolver is the flag used to enable the http remote resolver + EnableHttpResolver = "enable-http-resolver" ) // FeatureFlags holds the features configurations @@ -51,6 +55,7 @@ type FeatureFlags struct { EnableHubResolver bool EnableBundleResolver bool EnableClusterResolver bool + EnableHttpResolver bool } // GetFeatureFlagsConfigName returns the name of the configmap containing all @@ -90,6 +95,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) { if err := setFeature(EnableClusterResolver, DefaultEnableClusterResolver, &tc.EnableClusterResolver); err != nil { return nil, err } + if err := setFeature(EnableHttpResolver, DefaultEnableHttpResolver, &tc.EnableHttpResolver); err != nil { + return nil, err + } return &tc, nil } diff --git a/pkg/apis/config/resolver/feature_flags_test.go b/pkg/apis/config/resolver/feature_flags_test.go index 137ec747fe5..8d440b3f47f 100644 --- a/pkg/apis/config/resolver/feature_flags_test.go +++ b/pkg/apis/config/resolver/feature_flags_test.go @@ -38,6 +38,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) { EnableHubResolver: true, EnableBundleResolver: true, EnableClusterResolver: true, + EnableHttpResolver: true, }, fileName: "feature-flags-empty", }, @@ -47,6 +48,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) { EnableHubResolver: false, EnableBundleResolver: false, EnableClusterResolver: false, + EnableHttpResolver: false, }, fileName: "feature-flags-all-flags-set", }, @@ -68,6 +70,7 @@ func TestNewFeatureFlagsFromEmptyConfigMap(t *testing.T) { EnableHubResolver: resolver.DefaultEnableHubResolver, EnableBundleResolver: resolver.DefaultEnableBundlesResolver, EnableClusterResolver: resolver.DefaultEnableClusterResolver, + EnableHttpResolver: resolver.DefaultEnableHttpResolver, } verifyConfigFileWithExpectedFeatureFlagsConfig(t, FeatureFlagsConfigEmptyName, expectedConfig) } diff --git a/pkg/apis/config/resolver/testdata/feature-flags-all-flags-set.yaml b/pkg/apis/config/resolver/testdata/feature-flags-all-flags-set.yaml index aa4e03607ad..d4502ea0d9b 100644 --- a/pkg/apis/config/resolver/testdata/feature-flags-all-flags-set.yaml +++ b/pkg/apis/config/resolver/testdata/feature-flags-all-flags-set.yaml @@ -22,3 +22,4 @@ data: enable-hub-resolver: "false" enable-bundles-resolver: "false" enable-cluster-resolver: "false" + enable-http-resolver: "false" diff --git a/pkg/resolution/resolver/http/config.go b/pkg/resolution/resolver/http/config.go new file mode 100644 index 00000000000..0685fdb07ba --- /dev/null +++ b/pkg/resolution/resolver/http/config.go @@ -0,0 +1,23 @@ +/* +Copyright 2023 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +const ( + // timeoutKey is the configuration field name for controlling + // the maximum duration of a resolution request for a file from http. + timeoutKey = "fetch-timeout" +) diff --git a/pkg/resolution/resolver/http/params.go b/pkg/resolution/resolver/http/params.go new file mode 100644 index 00000000000..c997a90d05a --- /dev/null +++ b/pkg/resolution/resolver/http/params.go @@ -0,0 +1,19 @@ +/* +Copyright 2023 The Tekton Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +const ( + // urlParam is the url to fetch the task from + urlParam string = "url" +) diff --git a/pkg/resolution/resolver/http/resolver.go b/pkg/resolution/resolver/http/resolver.go new file mode 100644 index 00000000000..c99d33e27ba --- /dev/null +++ b/pkg/resolution/resolver/http/resolver.go @@ -0,0 +1,220 @@ +/* +Copyright 2022 The Tekton Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + resolverconfig "github.com/tektoncd/pipeline/pkg/apis/config/resolver" + pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + "github.com/tektoncd/pipeline/pkg/resolution/common" + "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" +) + +const ( + // LabelValueHttpResolverType is the value to use for the + // resolution.tekton.dev/type label on resource requests + LabelValueHttpResolverType string = "http" + + disabledError = "cannot handle resolution request, enable-http-resolver feature flag not true" + + // httpResolverName The name of the resolver + httpResolverName = "Http" + + // ConfigMapName is the http resolver's config map + configMapName = "http-resolver-config" + + // default Timeout value when fetching http resources in seconds + defaultHttpTimeoutValue = "1m" +) + +// Resolver implements a framework.Resolver that can fetch files from OCI bundles. +type Resolver struct{} + +// Initialize sets up any dependencies needed by the resolver. None atm. +func (r *Resolver) Initialize(context.Context) error { + return nil +} + +// GetName returns a string name to refer to this resolver by. +func (r *Resolver) GetName(context.Context) string { + return httpResolverName +} + +// GetConfigName returns the name of the bundle resolver's configmap. +func (r *Resolver) GetConfigName(context.Context) string { + return configMapName +} + +// GetSelector returns a map of labels to match requests to this resolver. +func (r *Resolver) GetSelector(context.Context) map[string]string { + return map[string]string{ + common.LabelKeyResolverType: LabelValueHttpResolverType, + } +} + +// ValidateParams ensures parameters from a request are as expected. +func (r *Resolver) ValidateParams(ctx context.Context, oParams []pipelinev1.Param) error { + if r.isDisabled(ctx) { + return errors.New(disabledError) + } + _, err := populateDefaultParams(ctx, oParams) + if err != nil { + return err + } + return nil +} + +// Resolve uses the given params to resolve the requested file or resource. +func (r *Resolver) Resolve(ctx context.Context, oParams []pipelinev1.Param) (framework.ResolvedResource, error) { + if r.isDisabled(ctx) { + return nil, errors.New(disabledError) + } + + params, err := populateDefaultParams(ctx, oParams) + if err != nil { + return nil, err + } + + httpClient, err := makeHttpClient(ctx) + if err != nil { + return nil, err + } + return fetchHttpResource(ctx, httpClient, params) +} + +func (r *Resolver) isDisabled(ctx context.Context) bool { + cfg := resolverconfig.FromContextOrDefaults(ctx) + return !cfg.FeatureFlags.EnableHttpResolver +} + +// resolvedHttpResource wraps the data we want to return to Pipelines +type resolvedHttpResource struct { + URL string + Content []byte +} + +var _ framework.ResolvedResource = &resolvedHttpResource{} + +// Data returns the bytes of our hard-coded Pipeline +func (rr *resolvedHttpResource) Data() []byte { + return rr.Content +} + +// Annotations returns any metadata needed alongside the data. None atm. +func (*resolvedHttpResource) Annotations() map[string]string { + return nil +} + +// RefSource is the source reference of the remote data that records where the remote +// file came from including the url, digest and the entrypoint. +func (rr *resolvedHttpResource) RefSource() *pipelinev1.RefSource { + h := sha256.New() + h.Write(rr.Content) + sha256CheckSum := hex.EncodeToString(h.Sum(nil)) + + return &pipelinev1.RefSource{ + URI: rr.URL, + Digest: map[string]string{ + "sha256": sha256CheckSum, + }, + } +} + +func populateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[string]string, error) { + paramsMap := make(map[string]string) + for _, p := range params { + paramsMap[p.Name] = p.Value.StringVal + } + + var missingParams []string + + if _, ok := paramsMap[urlParam]; !ok { + missingParams = append(missingParams, urlParam) + } else { + u, err := url.ParseRequestURI(paramsMap[urlParam]) + if err != nil { + return nil, fmt.Errorf("cannot parse url %s: %w", paramsMap[urlParam], err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("url %s is not a valid http(s) url", paramsMap[urlParam]) + } + } + + if len(missingParams) > 0 { + return nil, fmt.Errorf("missing required http resolver params: %s", strings.Join(missingParams, ", ")) + } + + return paramsMap, nil +} + +func makeHttpClient(ctx context.Context) (*http.Client, error) { + conf := framework.GetResolverConfigFromContext(ctx) + timeout, err := time.ParseDuration(defaultHttpTimeoutValue) + if err != nil { + return nil, fmt.Errorf("error parsing default timeout value %s: %w", defaultHttpTimeoutValue, err) + } + if v, ok := conf[timeoutKey]; ok { + timeout, err = time.ParseDuration(v) + if err != nil { + return nil, fmt.Errorf("error parsing timeout value %s: %w", v, err) + } + } + return &http.Client{ + Timeout: timeout, + }, nil +} + +func fetchHttpResource(ctx context.Context, client *http.Client, params map[string]string) (framework.ResolvedResource, error) { + var url string + var ok bool + + if url, ok = params[urlParam]; !ok { + return nil, fmt.Errorf("missing required params: %s", urlParam) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("constructing request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("error fetching URL: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("requested URL '%s' is not found", url) + } + defer func() { + _ = resp.Body.Close() + }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + + return &resolvedHttpResource{ + Content: body, + URL: url, + }, nil +} diff --git a/pkg/resolution/resolver/http/resolver_test.go b/pkg/resolution/resolver/http/resolver_test.go new file mode 100644 index 00000000000..3243fe9be72 --- /dev/null +++ b/pkg/resolution/resolver/http/resolver_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2022 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "testing" + + "github.com/google/go-cmp/cmp" + pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + resolutioncommon "github.com/tektoncd/pipeline/pkg/resolution/common" + "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" +) + +func TestGetSelector(t *testing.T) { + resolver := Resolver{} + sel := resolver.GetSelector(context.Background()) + if typ, has := sel[resolutioncommon.LabelKeyResolverType]; !has { + t.Fatalf("unexpected selector: %v", sel) + } else if typ != LabelValueHttpResolverType { + t.Fatalf("unexpected type: %q", typ) + } +} + +func TestValidateParams(t *testing.T) { + testCases := []struct { + name string + url string + expectedErr error + }{ + { + name: "valid/url", + url: "https://raw.githubusercontent.com/tektoncd/catalog/main/task/git-clone/0.4/git-clone.yaml", + }, { + name: "invalid/url", + url: "xttps:ufoo/bar/", + expectedErr: fmt.Errorf(`url xttps:ufoo/bar/ is not a valid http(s) url`), + }, { + name: "invalid/url empty", + url: "", + expectedErr: fmt.Errorf(`cannot parse url : parse "": empty url`), + }, { + name: "missing/url", + expectedErr: fmt.Errorf(`missing required http resolver params: url`), + url: "nourl", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resolver := Resolver{} + params := map[string]string{} + if tc.url != "nourl" { + params[urlParam] = tc.url + } + err := resolver.ValidateParams(contextWithConfig(defaultHttpTimeoutValue), toParams(params)) + if tc.expectedErr != nil { + checkExpectedErr(t, tc.expectedErr, err) + } else if err != nil { + t.Fatalf("unexpected error validating params: %v", err) + } + }) + } +} + +func TestMakeHTTPClient(t *testing.T) { + tests := []struct { + name string + expectedErr error + duration string + }{ + { + name: "good/duration", + duration: "30s", + }, + { + name: "bad/duration", + duration: "xxx", + expectedErr: fmt.Errorf(`error parsing timeout value xxx: time: invalid duration "xxx"`), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client, err := makeHttpClient(contextWithConfig(tc.duration)) + if tc.expectedErr != nil { + checkExpectedErr(t, tc.expectedErr, err) + return + } else if err != nil { + t.Fatalf("unexpected error creating http client: %v", err) + } + if client == nil { + t.Fatalf("expected an http client but got nil") + } + if client.Timeout.String() != tc.duration { + t.Fatalf("expected timeout %v but got %v", tc.duration, client.Timeout) + } + }) + } +} + +func TestFetchHttpResource(t *testing.T) { + tests := []struct { + name string + expectedErr string + input string + paramSet bool + expectedStatus int + }{ + { + name: "good/params set", + input: "task", + paramSet: true, + }, { + name: "bad/params not set", + input: "task", + expectedErr: `missing required params: url`, + }, { + name: "bad/not found", + input: "task", + paramSet: true, + expectedStatus: http.StatusNotFound, + expectedErr: `requested URL 'http://([^']*)' is not found`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.expectedStatus != 0 { + w.WriteHeader(tc.expectedStatus) + } + fmt.Fprintf(w, tc.input) + })) + params := map[string]string{} + if tc.paramSet { + params[urlParam] = svr.URL + } + output, err := fetchHttpResource(contextWithConfig(defaultHttpTimeoutValue), svr.Client(), params) + if tc.expectedErr != "" { + re := regexp.MustCompile(tc.expectedErr) + if !re.MatchString(err.Error()) { + t.Fatalf("expected error '%v' but got '%v'", tc.expectedErr, err) + } + return + } else if err != nil { + t.Fatalf("unexpected error validating params: %v", err) + } + if o := cmp.Diff(tc.input, string(output.Data())); o != "" { + t.Fatalf("expected output '%v' but got '%v'", tc.input, string(output.Data())) + } + if o := cmp.Diff(svr.URL, output.RefSource().URI); o != "" { + t.Fatalf("expected url '%v' but got '%v'", svr.URL, output.RefSource().URI) + } + + eSum := sha256.New() + eSum.Write([]byte(tc.input)) + eSha256 := hex.EncodeToString(eSum.Sum(nil)) + if o := cmp.Diff(eSha256, output.RefSource().Digest["sha256"]); o != "" { + t.Fatalf("expected sha256 '%v' but got '%v'", eSha256, output.RefSource().Digest["sha256"]) + } + + if output.Annotations() != nil { + t.Fatalf("output annotations should be nil") + } + }) + } +} + +func toParams(m map[string]string) []pipelinev1.Param { + var params []pipelinev1.Param + + for k, v := range m { + params = append(params, pipelinev1.Param{ + Name: k, + Value: *pipelinev1.NewStructuredValues(v), + }) + } + + return params +} + +func contextWithConfig(timeout string) context.Context { + config := map[string]string{ + timeoutKey: timeout, + } + return framework.InjectResolverConfigToContext(context.Background(), config) +} + +func checkExpectedErr(t *testing.T, expectedErr, actualErr error) { + t.Helper() + if actualErr == nil { + t.Fatalf("expected err '%v' but didn't get one", expectedErr) + } + if d := cmp.Diff(expectedErr.Error(), actualErr.Error()); d != "" { + t.Fatalf("expected err '%v' but got '%v'", expectedErr, actualErr) + } +}