Skip to content

Commit

Permalink
webhook: add options to disable resource_namespace tag in metrics
Browse files Browse the repository at this point in the history
To add some context, historically, `resource_name` was removed from this
tag list due to its high potential of causing high metrics cardinality.
See [knative#1464][1] for more information.

While that's great, but it might not be sufficient for large scale use
cases where namespaces can be super dynamic (with generateName, too) or
grows fase enough. There is an issue report from
[tektoncd/pipeline#3171][2] which talks about this.

This proposal makes it possible to disable `resource_namespace` tag via
an option function. The default behavior is not changed, so no user
impact if any of existing users rely on this tag. There is no API
contract change either due to the beauty of variadic functions.

Now downstream projects can consume this by override `StatsReporter` in
webhook context options with their own preference. As a caveat here, if
downstream project does choose to override `StatsReporter`, the default
`ReportMetrics` function shouldn't be called by default as they may now
have a different set of tag keys to report. As such, this function is
only called if the default `StatsReporter` is used.

[1]: knative#1464
[2]: tektoncd/pipeline#3171
  • Loading branch information
zhouhaibing089 committed Jan 10, 2024
1 parent 21d8c37 commit b85215c
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 15 deletions.
3 changes: 0 additions & 3 deletions injection/sharedmain/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,6 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto
// and pass them in.
var wh *webhook.Webhook
if len(webhooks) > 0 {
// Register webhook metrics
webhook.RegisterMetrics()

wh, err = webhook.New(ctx, webhooks)
if err != nil {
logger.Fatalw("Failed to create webhook", zap.Error(err))
Expand Down
45 changes: 36 additions & 9 deletions webhook/stats_reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,37 +71,57 @@ type StatsReporter interface {
ReportConversionRequest(request *apixv1.ConversionRequest, response *apixv1.ConversionResponse, d time.Duration) error
}

type options struct {
resourceNamespace bool
}

type Option func(opts *options)

func WithoutResourceNamespace() Option {
return func(opts *options) {
opts.resourceNamespace = false
}
}

// reporter implements StatsReporter interface
type reporter struct {
ctx context.Context
ctx context.Context
opts options
}

// NewStatsReporter creates a reporter for webhook metrics
func NewStatsReporter() (StatsReporter, error) {
func NewStatsReporter(opts ...Option) (StatsReporter, error) {
ctx, err := tag.New(
context.Background(),
)
if err != nil {
return nil, err
}

return &reporter{ctx: ctx}, nil
options := options{resourceNamespace: true}
for _, opt := range opts {
opt(&options)
}

return &reporter{ctx: ctx, opts: options}, nil
}

// Captures req count metric, recording the count and the duration
func (r *reporter) ReportAdmissionRequest(req *admissionv1.AdmissionRequest, resp *admissionv1.AdmissionResponse, d time.Duration) error {
ctx, err := tag.New(
r.ctx,
mutators := []tag.Mutator{
tag.Insert(requestOperationKey, string(req.Operation)),
tag.Insert(kindGroupKey, req.Kind.Group),
tag.Insert(kindVersionKey, req.Kind.Version),
tag.Insert(kindKindKey, req.Kind.Kind),
tag.Insert(resourceGroupKey, req.Resource.Group),
tag.Insert(resourceVersionKey, req.Resource.Version),
tag.Insert(resourceResourceKey, req.Resource.Resource),
tag.Insert(resourceNamespaceKey, req.Namespace),
tag.Insert(admissionAllowedKey, strconv.FormatBool(resp.Allowed)),
)
}
if r.opts.resourceNamespace {
mutators = append(mutators, tag.Insert(resourceNamespaceKey, req.Namespace))
}
ctx, err := tag.New(r.ctx, mutators...)
if err != nil {
return err
}
Expand Down Expand Up @@ -131,7 +151,7 @@ func (r *reporter) ReportConversionRequest(req *apixv1.ConversionRequest, resp *
return nil
}

func RegisterMetrics() {
func RegisterMetrics(opts ...Option) {
tagKeys := []tag.Key{
requestOperationKey,
kindGroupKey,
Expand All @@ -140,13 +160,20 @@ func RegisterMetrics() {
resourceGroupKey,
resourceVersionKey,
resourceResourceKey,
resourceNamespaceKey,
admissionAllowedKey,
desiredAPIVersionKey,
resultStatusKey,
resultReasonKey,
resultCodeKey}

options := options{resourceNamespace: true}
for _, opt := range opts {
opt(&options)
}
if options.resourceNamespace {
tagKeys = append(tagKeys, resourceNamespaceKey)
}

if err := view.Register(
&view.View{
Description: requestCountM.Description(),
Expand Down
47 changes: 44 additions & 3 deletions webhook/stats_reporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,47 @@ func TestWebhookStatsReporterAdmission(t *testing.T) {
metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
}

func TestWebhookStatsReporterAdmissionWithoutNamespaceTag(t *testing.T) {
setup(WithoutResourceNamespace())
req := &admissionv1.AdmissionRequest{
UID: "705ab4f5-6393-11e8-b7cc-42010a800002",
Kind: metav1.GroupVersionKind{Group: "autoscaling", Version: "v1", Kind: "Scale"},
Resource: metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
Name: "my-deployment",
Namespace: "my-namespace",
Operation: admissionv1.Update,
}

resp := &admissionv1.AdmissionResponse{
UID: req.UID,
Allowed: true,
}

r, _ := NewStatsReporter(WithoutResourceNamespace())

shortTime, longTime := 1100.0, 9100.0
expectedTags := map[string]string{
requestOperationKey.Name(): string(req.Operation),
kindGroupKey.Name(): req.Kind.Group,
kindVersionKey.Name(): req.Kind.Version,
kindKindKey.Name(): req.Kind.Kind,
resourceGroupKey.Name(): req.Resource.Group,
resourceVersionKey.Name(): req.Resource.Version,
resourceResourceKey.Name(): req.Resource.Resource,
admissionAllowedKey.Name(): strconv.FormatBool(resp.Allowed),
}

if err := r.ReportAdmissionRequest(req, resp, time.Duration(shortTime)*time.Millisecond); err != nil {
t.Fatalf("ReportAdmissionRequest() = %v", err)
}
if err := r.ReportAdmissionRequest(req, resp, time.Duration(longTime)*time.Millisecond); err != nil {
t.Fatalf("ReportAdmissionRequest() = %v", err)
}

metricstest.CheckCountData(t, requestCountName, expectedTags, 2)
metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
}

func TestWebhookStatsReporterConversion(t *testing.T) {
setup()
req := &apixv1.ConversionRequest{
Expand Down Expand Up @@ -103,12 +144,12 @@ func TestWebhookStatsReporterConversion(t *testing.T) {
metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
}

func setup() {
func setup(opts ...Option) {
resetMetrics()
}

// opencensus metrics carry global state that need to be reset between unit tests
func resetMetrics() {
func resetMetrics(opts ...Option) {
metricstest.Unregister(requestCountName, requestLatenciesName)
RegisterMetrics()
RegisterMetrics(opts...)
}
1 change: 1 addition & 0 deletions webhook/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func New(
logger := logging.FromContext(ctx)

if opts.StatsReporter == nil {
RegisterMetrics()
reporter, err := NewStatsReporter()
if err != nil {
return nil, err
Expand Down

0 comments on commit b85215c

Please sign in to comment.