diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index de43ff71f6541..6323d13ba3c03 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -40,8 +40,11 @@ var ( func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, ac autodiscovery.Component) { providers.InitConfigFilesReader(confSearchPaths) + + acTelemetryStore := ac.GetTelemetryStore() + ac.AddConfigProvider( - providers.NewFileConfigProvider(), + providers.NewFileConfigProvider(acTelemetryStore), config.Datadog().GetBool("autoconf_config_files_poll"), time.Duration(config.Datadog().GetInt("autoconf_config_files_poll_interval"))*time.Second, ) @@ -107,7 +110,7 @@ func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, for _, cp := range uniqueConfigProviders { factory, found := ac.GetProviderCatalog()[cp.Name] if found { - configProvider, err := factory(&cp, wmeta) + configProvider, err := factory(&cp, wmeta, acTelemetryStore) if err != nil { log.Errorf("Error while adding config provider %v: %v", cp.Name, err) continue diff --git a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig.go b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig.go index 1389826e0089b..41b511ff29200 100644 --- a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig.go +++ b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig.go @@ -26,13 +26,14 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/scheduler" autodiscoveryStatus "github.com/DataDog/datadog-agent/comp/core/autodiscovery/status" - "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" configComponent "github.com/DataDog/datadog-agent/comp/core/config" flaretypes "github.com/DataDog/datadog-agent/comp/core/flare/types" logComp "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/tagger" + "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" @@ -58,6 +59,7 @@ type dependencies struct { TaggerComp tagger.Component Secrets secrets.Component WMeta optional.Option[workloadmeta.Component] + Telemetry telemetry.Component } // AutoConfig implements the agent's autodiscovery mechanism. It is @@ -82,6 +84,7 @@ type AutoConfig struct { wmeta optional.Option[workloadmeta.Component] taggerComp tagger.Component logs logComp.Component + telemetryStore *acTelemetry.Store // m covers the `configPollers`, `listenerCandidates`, `listeners`, and `listenerRetryStop`, but // not the values they point to. @@ -123,17 +126,18 @@ func newProvides(deps dependencies) provides { var _ autodiscovery.Component = (*AutoConfig)(nil) type listenerCandidate struct { - factory listeners.ServiceListenerFactory - config listeners.Config + factory listeners.ServiceListenerFactory + config listeners.Config + telemetryStore *acTelemetry.Store } func (l *listenerCandidate) try() (listeners.ServiceListener, error) { - return l.factory(l.config) + return l.factory(l.config, l.telemetryStore) } // newAutoConfig creates an AutoConfig instance and starts it. func newAutoConfig(deps dependencies) autodiscovery.Component { - ac := createNewAutoConfig(scheduler.NewController(), deps.Secrets, deps.WMeta, deps.TaggerComp, deps.Log) + ac := createNewAutoConfig(scheduler.NewController(), deps.Secrets, deps.WMeta, deps.TaggerComp, deps.Log, deps.Telemetry) deps.Lc.Append(fx.Hook{ OnStart: func(c context.Context) error { ac.Start() @@ -148,7 +152,7 @@ func newAutoConfig(deps dependencies) autodiscovery.Component { } // createNewAutoConfig creates an AutoConfig instance (without starting). -func createNewAutoConfig(schedulerController *scheduler.Controller, secretResolver secrets.Component, wmeta optional.Option[workloadmeta.Component], taggerComp tagger.Component, logs logComp.Component) *AutoConfig { +func createNewAutoConfig(schedulerController *scheduler.Controller, secretResolver secrets.Component, wmeta optional.Option[workloadmeta.Component], taggerComp tagger.Component, logs logComp.Component, telemetryComp telemetry.Component) *AutoConfig { cfgMgr := newReconcilingConfigManager(secretResolver) ac := &AutoConfig{ configPollers: make([]*configPoller, 0, 9), @@ -168,6 +172,7 @@ func createNewAutoConfig(schedulerController *scheduler.Controller, secretResolv wmeta: wmeta, taggerComp: taggerComp, logs: logs, + telemetryStore: acTelemetry.NewStore(telemetryComp), } return ac } @@ -386,7 +391,7 @@ func (ac *AutoConfig) AddConfigProvider(provider providers.ConfigProvider, shoul log.Warnf("Polling interval <= 0 for AD provider: %s, deactivating polling", provider.String()) shouldPoll = false } - cp := newConfigPoller(provider, shouldPoll, pollInterval) + cp := newConfigPoller(provider, shouldPoll, pollInterval, ac.telemetryStore) ac.m.Lock() defer ac.m.Unlock() @@ -448,6 +453,11 @@ func (ac *AutoConfig) GetAllConfigs() []integration.Config { return configs } +// GetTelemetryStore returns autodiscovery telemetry store. +func (ac *AutoConfig) GetTelemetryStore() *acTelemetry.Store { + return ac.telemetryStore +} + // processNewConfig store (in template cache) and resolves a given config, // returning the changes to be made. func (ac *AutoConfig) processNewConfig(config integration.Config) integration.ConfigChanges { @@ -497,7 +507,7 @@ func (ac *AutoConfig) addListenerCandidates(listenerConfigs []config.Listeners) continue } log.Debugf("Listener %s was registered", c.Name) - ac.listenerCandidates[c.Name] = &listenerCandidate{factory: factory, config: &c} + ac.listenerCandidates[c.Name] = &listenerCandidate{factory: factory, config: &c, telemetryStore: ac.telemetryStore} } } @@ -658,17 +668,23 @@ func (ac *AutoConfig) GetAutodiscoveryErrors() map[string]map[string]providers.E // applyChanges applies a configChanges object. This always unschedules first. func (ac *AutoConfig) applyChanges(changes integration.ConfigChanges) { + telemetryStorePresent := ac.telemetryStore != nil + if len(changes.Unschedule) > 0 { for _, conf := range changes.Unschedule { log.Tracef("Unscheduling %s\n", conf.Dump(false)) - telemetry.ScheduledConfigs.Dec(conf.Provider, configType(conf)) + if telemetryStorePresent { + ac.telemetryStore.ScheduledConfigs.Dec(conf.Provider, configType(conf)) + } } } if len(changes.Schedule) > 0 { for _, conf := range changes.Schedule { log.Tracef("Scheduling %s\n", conf.Dump(false)) - telemetry.ScheduledConfigs.Inc(conf.Provider, configType(conf)) + if telemetryStorePresent { + ac.telemetryStore.ScheduledConfigs.Inc(conf.Provider, configType(conf)) + } } } ac.schedulerController.ApplyChanges(changes) @@ -722,6 +738,7 @@ type optionalModuleDeps struct { TaggerComp optional.Option[tagger.Component] Secrets secrets.Component WMeta optional.Option[workloadmeta.Component] + Telemetry telemetry.Component } // OptionalModule defines the fx options when ac should be used as an optional and not started @@ -739,5 +756,5 @@ func newOptionalAutoConfig(deps optionalModuleDeps) optional.Option[autodiscover return optional.NewNoneOption[autodiscovery.Component]() } return optional.NewOption[autodiscovery.Component]( - createNewAutoConfig(scheduler.NewController(), deps.Secrets, deps.WMeta, taggerComp, deps.Log)) + createNewAutoConfig(scheduler.NewController(), deps.Secrets, deps.WMeta, taggerComp, deps.Log, deps.Telemetry)) } diff --git a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_mock.go b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_mock.go index 055dfa8b56e70..9fe47b073883c 100644 --- a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_mock.go +++ b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_mock.go @@ -19,6 +19,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" + "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -40,6 +41,7 @@ type mockdependencies struct { Params MockParams TaggerComp tagger.Mock LogsComp log.Component + Telemetry telemetry.Component } type mockprovides struct { @@ -50,7 +52,7 @@ type mockprovides struct { } func newMockAutoConfig(deps mockdependencies) mockprovides { - ac := createNewAutoConfig(deps.Params.Scheduler, nil, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := createNewAutoConfig(deps.Params.Scheduler, nil, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) return mockprovides{ Comp: ac, Endpoint: api.NewAgentEndpointProvider(ac.mockHandleRequest, "/config-check", "GET"), diff --git a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_test.go b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_test.go index 4d12b9260e042..f7cbfc6e0d4d6 100644 --- a/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_test.go +++ b/comp/core/autodiscovery/autodiscoveryimpl/autoconfig_test.go @@ -26,10 +26,12 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/scheduler" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/secrets" "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" + "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" @@ -80,7 +82,7 @@ func (l *MockListener) Stop() { l.stopReceived = true } -func (l *MockListener) fakeFactory(listeners.Config) (listeners.ServiceListener, error) { +func (l *MockListener) fakeFactory(listeners.Config, *acTelemetry.Store) (listeners.ServiceListener, error) { return l, nil } @@ -96,7 +98,7 @@ type factoryMock struct { returnError error } -func (o *factoryMock) make(listeners.Config) (listeners.ServiceListener, error) { +func (o *factoryMock) make(listeners.Config, *acTelemetry.Store) (listeners.ServiceListener, error) { o.Lock() defer o.Unlock() if o.callChan != nil { @@ -186,15 +188,15 @@ func (suite *AutoConfigTestSuite) SetupTest() { suite.deps = createDeps(suite.T()) } -func getAutoConfig(schedulerController *scheduler.Controller, secretResolver secrets.Component, wmeta optional.Option[workloadmeta.Component], taggerComp tagger.Component, logsComp log.Component) *AutoConfig { - ac := createNewAutoConfig(schedulerController, secretResolver, wmeta, taggerComp, logsComp) +func getAutoConfig(schedulerController *scheduler.Controller, secretResolver secrets.Component, wmeta optional.Option[workloadmeta.Component], taggerComp tagger.Component, logsComp log.Component, telemetryComp telemetry.Component) *AutoConfig { + ac := createNewAutoConfig(schedulerController, secretResolver, wmeta, taggerComp, logsComp, telemetryComp) go ac.serviceListening() return ac } func (suite *AutoConfigTestSuite) TestAddConfigProvider() { mockResolver := MockSecretResolver{suite.T(), nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp, suite.deps.Telemetry) assert.Len(suite.T(), ac.configPollers, 0) mp := &MockProvider{} ac.AddConfigProvider(mp, false, 0) @@ -211,7 +213,7 @@ func (suite *AutoConfigTestSuite) TestAddConfigProvider() { func (suite *AutoConfigTestSuite) TestAddListener() { mockResolver := MockSecretResolver{suite.T(), nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp, suite.deps.Telemetry) assert.Len(suite.T(), ac.listeners, 0) ml := &MockListener{} @@ -249,7 +251,7 @@ func (suite *AutoConfigTestSuite) TestDiffConfigs() { func (suite *AutoConfigTestSuite) TestStop() { mockResolver := MockSecretResolver{suite.T(), nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp, suite.deps.Telemetry) ml := &MockListener{} listeners.Register("mock", ml.fakeFactory, ac.serviceListenerFactories) @@ -262,7 +264,7 @@ func (suite *AutoConfigTestSuite) TestStop() { func (suite *AutoConfigTestSuite) TestListenerRetry() { mockResolver := MockSecretResolver{suite.T(), nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, suite.deps.WMeta, suite.deps.TaggerComp, suite.deps.LogsComp, suite.deps.Telemetry) // Hack the retry delay to shorten the test run time initialListenerCandidateIntl := listenerCandidateIntl @@ -370,7 +372,7 @@ func TestResolveTemplate(t *testing.T) { msch.Register("mock", sch, false) mockResolver := MockSecretResolver{t, nil} - ac := getAutoConfig(msch, &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := getAutoConfig(msch, &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) tpl := integration.Config{ Name: "cpu", ADIdentifiers: []string{"redis"}, @@ -407,7 +409,7 @@ func TestRemoveTemplate(t *testing.T) { mockResolver := MockSecretResolver{t, nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) // Add static config c := integration.Config{ Name: "memory", @@ -460,7 +462,7 @@ func TestDecryptConfig(t *testing.T) { }, }} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) ac.processNewService(ctx, &dummyService{ID: "abcd", ADIdentifiers: []string{"redis"}}) tpl := integration.Config{ @@ -504,7 +506,7 @@ func TestProcessClusterCheckConfigWithSecrets(t *testing.T) { returnedError: nil, }, }} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) tpl := integration.Config{ Provider: names.ClusterChecks, @@ -539,7 +541,7 @@ func TestWriteConfigEndpoint(t *testing.T) { configName := "testConfig" mockResolver := MockSecretResolver{t, nil} - ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp) + ac := getAutoConfig(scheduler.NewController(), &mockResolver, deps.WMeta, deps.TaggerComp, deps.LogsComp, deps.Telemetry) tpl := integration.Config{ Provider: names.ClusterChecks, @@ -593,6 +595,7 @@ type Deps struct { WMeta optional.Option[workloadmeta.Component] TaggerComp tagger.Component LogsComp log.Component + Telemetry telemetry.Component } func createDeps(t *testing.T) Deps { diff --git a/comp/core/autodiscovery/autodiscoveryimpl/config_poller.go b/comp/core/autodiscovery/autodiscoveryimpl/config_poller.go index 8f7fb02818b83..e6086248dde56 100644 --- a/comp/core/autodiscovery/autodiscoveryimpl/config_poller.go +++ b/comp/core/autodiscovery/autodiscoveryimpl/config_poller.go @@ -30,17 +30,19 @@ type configPoller struct { stopChan chan struct{} - configsMu sync.Mutex - configs map[uint64]integration.Config + configsMu sync.Mutex + configs map[uint64]integration.Config + telemetryStore *telemetry.Store } -func newConfigPoller(provider providers.ConfigProvider, canPoll bool, interval time.Duration) *configPoller { +func newConfigPoller(provider providers.ConfigProvider, canPoll bool, interval time.Duration, telemetryStore *telemetry.Store) *configPoller { return &configPoller{ - provider: provider, - configs: make(map[uint64]integration.Config), - canPoll: canPoll, - pollInterval: interval, - stopChan: make(chan struct{}), + provider: provider, + configs: make(map[uint64]integration.Config), + canPoll: canPoll, + pollInterval: interval, + stopChan: make(chan struct{}), + telemetryStore: telemetryStore, } } @@ -203,7 +205,9 @@ func (cp *configPoller) collectOnce(ctx context.Context, provider providers.Coll func (cp *configPoller) collect(ctx context.Context, provider providers.CollectingConfigProvider) ([]integration.Config, []integration.Config) { start := time.Now() defer func() { - telemetry.PollDuration.Observe(time.Since(start).Seconds(), cp.provider.String()) + if cp.telemetryStore != nil { + cp.telemetryStore.PollDuration.Observe(time.Since(start).Seconds(), cp.provider.String()) + } }() fetched, err := provider.Collect(ctx) diff --git a/comp/core/autodiscovery/component.go b/comp/core/autodiscovery/component.go index 38619d92d10d2..ac97e99b05744 100644 --- a/comp/core/autodiscovery/component.go +++ b/comp/core/autodiscovery/component.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/scheduler" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/config" ) @@ -34,6 +35,7 @@ type Component interface { GetIDOfCheckWithEncryptedSecrets(checkID checkid.ID) checkid.ID GetAutodiscoveryErrors() map[string]map[string]providers.ErrorMsgSet GetProviderCatalog() map[string]providers.ConfigProviderFactory + GetTelemetryStore() *telemetry.Store // TODO (component): deprecate start/stop methods Start() Stop() diff --git a/comp/core/autodiscovery/listeners/cloudfoundry.go b/comp/core/autodiscovery/listeners/cloudfoundry.go index 115d9ca384dc2..99f8e1806ce70 100644 --- a/comp/core/autodiscovery/listeners/cloudfoundry.go +++ b/comp/core/autodiscovery/listeners/cloudfoundry.go @@ -16,6 +16,7 @@ import ( "time" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/util/cloudproviders/cloudfoundry" "github.com/DataDog/datadog-agent/pkg/util/containers" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -50,7 +51,7 @@ type CloudFoundryService struct { var _ Service = &CloudFoundryService{} // NewCloudFoundryListener creates a CloudFoundryListener -func NewCloudFoundryListener(Config) (ServiceListener, error) { +func NewCloudFoundryListener(Config, *telemetry.Store) (ServiceListener, error) { bbsCache, err := cloudfoundry.GetGlobalBBSCache() if err != nil { return nil, err diff --git a/comp/core/autodiscovery/listeners/container.go b/comp/core/autodiscovery/listeners/container.go index 64d5627ee5735..b0dce0e562106 100644 --- a/comp/core/autodiscovery/listeners/container.go +++ b/comp/core/autodiscovery/listeners/container.go @@ -14,6 +14,7 @@ import ( "time" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/config" @@ -35,7 +36,7 @@ type ContainerListener struct { } // NewContainerListener returns a new ContainerListener. -func NewContainerListener(_ Config, wmeta optional.Option[workloadmeta.Component]) (ServiceListener, error) { +func NewContainerListener(_ Config, wmeta optional.Option[workloadmeta.Component], telemetryStore *telemetry.Store) (ServiceListener, error) { const name = "ad-containerlistener" l := &ContainerListener{} filter := workloadmeta.NewFilterBuilder(). @@ -47,7 +48,7 @@ func NewContainerListener(_ Config, wmeta optional.Option[workloadmeta.Component return nil, errors.New("workloadmeta store is not initialized") } var err error - l.workloadmetaListener, err = newWorkloadmetaListener(name, filter, l.createContainerService, wmetaInstance) + l.workloadmetaListener, err = newWorkloadmetaListener(name, filter, l.createContainerService, wmetaInstance, telemetryStore) if err != nil { return nil, err } diff --git a/comp/core/autodiscovery/listeners/container_nop.go b/comp/core/autodiscovery/listeners/container_nop.go index c449adc98776a..e65adc04de7cc 100644 --- a/comp/core/autodiscovery/listeners/container_nop.go +++ b/comp/core/autodiscovery/listeners/container_nop.go @@ -8,10 +8,11 @@ package listeners import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/util/optional" ) -type noopServiceListenerFactory func(c Config, wmeta optional.Option[workloadmeta.Component]) (ServiceListener, error) +type noopServiceListenerFactory func(c Config, wmeta optional.Option[workloadmeta.Component], telemetryStore *telemetry.Store) (ServiceListener, error) var NewContainerListener noopServiceListenerFactory diff --git a/comp/core/autodiscovery/listeners/dbm_aurora.go b/comp/core/autodiscovery/listeners/dbm_aurora.go index b8cf46dbe3e3a..f29727a9dc6ef 100644 --- a/comp/core/autodiscovery/listeners/dbm_aurora.go +++ b/comp/core/autodiscovery/listeners/dbm_aurora.go @@ -16,6 +16,7 @@ import ( "time" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/databasemonitoring/aws" dbmconfig "github.com/DataDog/datadog-agent/pkg/databasemonitoring/config" "github.com/DataDog/datadog-agent/pkg/util/containers" @@ -67,7 +68,7 @@ type DBMAuroraService struct { } // NewDBMAuroraListener returns a new DBMAuroraListener -func NewDBMAuroraListener(Config) (ServiceListener, error) { +func NewDBMAuroraListener(Config, *telemetry.Store) (ServiceListener, error) { config, err := dbmconfig.NewAuroraAutodiscoveryConfig() if err != nil { return nil, err diff --git a/comp/core/autodiscovery/listeners/environment.go b/comp/core/autodiscovery/listeners/environment.go index 76ac4bef8dda8..9bf84c366869e 100644 --- a/comp/core/autodiscovery/listeners/environment.go +++ b/comp/core/autodiscovery/listeners/environment.go @@ -9,6 +9,7 @@ import ( "context" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/containers" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -28,7 +29,7 @@ type EnvironmentService struct { var _ Service = &EnvironmentService{} // NewEnvironmentListener creates an EnvironmentListener -func NewEnvironmentListener(Config) (ServiceListener, error) { +func NewEnvironmentListener(Config, *telemetry.Store) (ServiceListener, error) { return &EnvironmentListener{}, nil } diff --git a/comp/core/autodiscovery/listeners/kube_endpoints.go b/comp/core/autodiscovery/listeners/kube_endpoints.go index 3a2138bc5f271..a0ef296d318f7 100644 --- a/comp/core/autodiscovery/listeners/kube_endpoints.go +++ b/comp/core/autodiscovery/listeners/kube_endpoints.go @@ -49,6 +49,7 @@ type KubeEndpointsListener struct { targetAllEndpoints bool m sync.RWMutex containerFilters *containerFilters + telemetryStore *telemetry.Store } // KubeEndpointService represents an endpoint in a Kubernetes Endpoints @@ -65,7 +66,7 @@ type KubeEndpointService struct { var _ Service = &KubeEndpointService{} // NewKubeEndpointsListener returns the kube endpoints implementation of the ServiceListener interface -func NewKubeEndpointsListener(conf Config) (ServiceListener, error) { +func NewKubeEndpointsListener(conf Config, telemetryStore *telemetry.Store) (ServiceListener, error) { // Using GetAPIClient (no wait) as Client should already be initialized by Cluster Agent main entrypoint before ac, err := apiserver.GetAPIClient() if err != nil { @@ -96,6 +97,7 @@ func NewKubeEndpointsListener(conf Config) (ServiceListener, error) { promInclAnnot: getPrometheusIncludeAnnotations(), targetAllEndpoints: conf.IsProviderEnabled(names.KubeEndpointsFileRegisterName), containerFilters: containerFilters, + telemetryStore: telemetryStore, }, nil } @@ -338,13 +340,18 @@ func (l *KubeEndpointsListener) createService(kep *v1.Endpoints, checkServiceAnn l.m.Lock() l.endpoints[kep.UID] = eps l.m.Unlock() + telemetryStorePresent := l.telemetryStore != nil - telemetry.WatchedResources.Inc(kubeEndpointsName, telemetry.ResourceKubeService) + if telemetryStorePresent { + l.telemetryStore.WatchedResources.Inc(kubeEndpointsName, telemetry.ResourceKubeService) + } for _, ep := range eps { log.Debugf("Creating a new AD service: %s", ep.entity) l.newService <- ep - telemetry.WatchedResources.Inc(kubeEndpointsName, telemetry.ResourceKubeEndpoint) + if telemetryStorePresent { + l.telemetryStore.WatchedResources.Inc(kubeEndpointsName, telemetry.ResourceKubeEndpoint) + } } } @@ -390,12 +397,18 @@ func (l *KubeEndpointsListener) removeService(kep *v1.Endpoints) { delete(l.endpoints, kep.UID) l.m.Unlock() - telemetry.WatchedResources.Dec(kubeEndpointsName, telemetry.ResourceKubeService) + telemetryStorePresent := l.telemetryStore != nil + + if telemetryStorePresent { + l.telemetryStore.WatchedResources.Dec(kubeEndpointsName, telemetry.ResourceKubeService) + } for _, ep := range eps { log.Debugf("Deleting AD service: %s", ep.entity) l.delService <- ep - telemetry.WatchedResources.Dec(kubeEndpointsName, telemetry.ResourceKubeEndpoint) + if telemetryStorePresent { + l.telemetryStore.WatchedResources.Dec(kubeEndpointsName, telemetry.ResourceKubeEndpoint) + } } } else { log.Debugf("Entity %s not found, not removing", kep.UID) diff --git a/comp/core/autodiscovery/listeners/kube_services.go b/comp/core/autodiscovery/listeners/kube_services.go index 812a05aa12b27..7119870471e99 100644 --- a/comp/core/autodiscovery/listeners/kube_services.go +++ b/comp/core/autodiscovery/listeners/kube_services.go @@ -45,6 +45,7 @@ type KubeServiceListener struct { targetAllServices bool m sync.RWMutex containerFilters *containerFilters + telemetryStore *telemetry.Store } // KubeServiceService represents a Kubernetes Service @@ -80,7 +81,7 @@ func isServiceAnnotated(ksvc *v1.Service, annotationKey string) bool { } // NewKubeServiceListener returns the kube service implementation of the ServiceListener interface -func NewKubeServiceListener(conf Config) (ServiceListener, error) { +func NewKubeServiceListener(conf Config, telemetryStore *telemetry.Store) (ServiceListener, error) { // Using GetAPIClient (no wait) as Client should already be initialized by Cluster Agent main entrypoint before ac, err := apiserver.GetAPIClient() if err != nil { @@ -103,6 +104,7 @@ func NewKubeServiceListener(conf Config) (ServiceListener, error) { promInclAnnot: getPrometheusIncludeAnnotations(), targetAllServices: conf.IsProviderEnabled(names.KubeServicesFileRegisterName), containerFilters: containerFilters, + telemetryStore: telemetryStore, }, nil } @@ -261,7 +263,9 @@ func (l *KubeServiceListener) createService(ksvc *v1.Service) { l.m.Unlock() l.newService <- svc - telemetry.WatchedResources.Inc(kubeServicesName, telemetry.ResourceKubeService) + if l.telemetryStore != nil { + l.telemetryStore.WatchedResources.Inc(kubeServicesName, telemetry.ResourceKubeService) + } } func processService(ksvc *v1.Service) *KubeServiceService { @@ -312,7 +316,9 @@ func (l *KubeServiceListener) removeService(ksvc *v1.Service) { l.m.Unlock() l.delService <- svc - telemetry.WatchedResources.Dec(kubeServicesName, telemetry.ResourceKubeService) + if l.telemetryStore != nil { + l.telemetryStore.WatchedResources.Dec(kubeServicesName, telemetry.ResourceKubeService) + } } else { log.Debugf("Entity %s not found, not removing", ksvc.UID) } diff --git a/comp/core/autodiscovery/listeners/kubelet.go b/comp/core/autodiscovery/listeners/kubelet.go index c780e41f285f6..6e81e83df6099 100644 --- a/comp/core/autodiscovery/listeners/kubelet.go +++ b/comp/core/autodiscovery/listeners/kubelet.go @@ -13,6 +13,7 @@ import ( "time" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/comp/core/tagger" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/config" @@ -29,7 +30,7 @@ type KubeletListener struct { } // NewKubeletListener returns a new KubeletListener. -func NewKubeletListener(_ Config, wmeta optional.Option[workloadmeta.Component]) (ServiceListener, error) { +func NewKubeletListener(_ Config, wmeta optional.Option[workloadmeta.Component], telemetryStore *telemetry.Store) (ServiceListener, error) { const name = "ad-kubeletlistener" l := &KubeletListener{} @@ -43,7 +44,7 @@ func NewKubeletListener(_ Config, wmeta optional.Option[workloadmeta.Component]) return nil, errors.New("workloadmeta store is not initialized") } var err error - l.workloadmetaListener, err = newWorkloadmetaListener(name, filter, l.processPod, wmetaInstance) + l.workloadmetaListener, err = newWorkloadmetaListener(name, filter, l.processPod, wmetaInstance, telemetryStore) if err != nil { return nil, err } diff --git a/comp/core/autodiscovery/listeners/listeners.go b/comp/core/autodiscovery/listeners/listeners.go index 9de0bc2053285..9741cfd4dcc98 100644 --- a/comp/core/autodiscovery/listeners/listeners.go +++ b/comp/core/autodiscovery/listeners/listeners.go @@ -7,6 +7,7 @@ package listeners import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -27,11 +28,15 @@ const ( func RegisterListeners(serviceListenerFactories map[string]ServiceListenerFactory, wmeta optional.Option[workloadmeta.Component]) { // register the available listeners Register(cloudFoundryBBSListenerName, NewCloudFoundryListener, serviceListenerFactories) - Register(containerListenerName, func(config Config) (ServiceListener, error) { return NewContainerListener(config, wmeta) }, serviceListenerFactories) + Register(containerListenerName, func(config Config, telemetryStore *telemetry.Store) (ServiceListener, error) { + return NewContainerListener(config, wmeta, telemetryStore) + }, serviceListenerFactories) Register(environmentListenerName, NewEnvironmentListener, serviceListenerFactories) Register(kubeEndpointsListenerName, NewKubeEndpointsListener, serviceListenerFactories) Register(kubeServicesListenerName, NewKubeServiceListener, serviceListenerFactories) - Register(kubeletListenerName, func(config Config) (ServiceListener, error) { return NewKubeletListener(config, wmeta) }, serviceListenerFactories) + Register(kubeletListenerName, func(config Config, telemetryStore *telemetry.Store) (ServiceListener, error) { + return NewKubeletListener(config, wmeta, telemetryStore) + }, serviceListenerFactories) Register(snmpListenerName, NewSNMPListener, serviceListenerFactories) Register(staticConfigListenerName, NewStaticConfigListener, serviceListenerFactories) Register(dbmAuroraListenerName, NewDBMAuroraListener, serviceListenerFactories) diff --git a/comp/core/autodiscovery/listeners/snmp.go b/comp/core/autodiscovery/listeners/snmp.go index dd8143d5c7091..cfa9f9f934907 100644 --- a/comp/core/autodiscovery/listeners/snmp.go +++ b/comp/core/autodiscovery/listeners/snmp.go @@ -16,6 +16,7 @@ import ( "time" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/persistentcache" "github.com/DataDog/datadog-agent/pkg/snmp" "github.com/DataDog/datadog-agent/pkg/util/containers" @@ -66,7 +67,7 @@ type snmpJob struct { } // NewSNMPListener creates a SNMPListener -func NewSNMPListener(Config) (ServiceListener, error) { +func NewSNMPListener(Config, *telemetry.Store) (ServiceListener, error) { snmpConfig, err := snmp.NewListenerConfig() if err != nil { return nil, err diff --git a/comp/core/autodiscovery/listeners/snmp_test.go b/comp/core/autodiscovery/listeners/snmp_test.go index 9ae01d45b097a..aca2932d2c394 100644 --- a/comp/core/autodiscovery/listeners/snmp_test.go +++ b/comp/core/autodiscovery/listeners/snmp_test.go @@ -43,7 +43,7 @@ func TestSNMPListener(t *testing.T) { } } - l, err := NewSNMPListener(&config.Listeners{}) + l, err := NewSNMPListener(&config.Listeners{}, nil) assert.Equal(t, nil, err) l.Listen(newSvc, delSvc) @@ -142,7 +142,7 @@ func TestSNMPListenerIgnoredAdresses(t *testing.T) { } } - l, err := NewSNMPListener(&config.Listeners{}) + l, err := NewSNMPListener(&config.Listeners{}, nil) assert.Equal(t, nil, err) l.Listen(newSvc, delSvc) diff --git a/comp/core/autodiscovery/listeners/staticconfig.go b/comp/core/autodiscovery/listeners/staticconfig.go index be9daf8c3701c..1ad89b816c7cb 100644 --- a/comp/core/autodiscovery/listeners/staticconfig.go +++ b/comp/core/autodiscovery/listeners/staticconfig.go @@ -9,6 +9,7 @@ import ( "context" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/containers" ) @@ -27,7 +28,7 @@ type StaticConfigService struct { var _ Service = &StaticConfigService{} // NewStaticConfigListener creates a StaticConfigListener -func NewStaticConfigListener(Config) (ServiceListener, error) { +func NewStaticConfigListener(Config, *telemetry.Store) (ServiceListener, error) { return &StaticConfigListener{}, nil } diff --git a/comp/core/autodiscovery/listeners/types.go b/comp/core/autodiscovery/listeners/types.go index a71254054214e..8718b46b72c85 100644 --- a/comp/core/autodiscovery/listeners/types.go +++ b/comp/core/autodiscovery/listeners/types.go @@ -8,7 +8,9 @@ package listeners import ( "context" "errors" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/util/containers" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -60,7 +62,7 @@ type Config interface { } // ServiceListenerFactory builds a service listener -type ServiceListenerFactory func(Config) (ServiceListener, error) +type ServiceListenerFactory func(Config, *telemetry.Store) (ServiceListener, error) // Register registers a service listener factory func Register(name string, diff --git a/comp/core/autodiscovery/listeners/workloadmeta.go b/comp/core/autodiscovery/listeners/workloadmeta.go index ffd1e17c454cd..75b18f2931c44 100644 --- a/comp/core/autodiscovery/listeners/workloadmeta.go +++ b/comp/core/autodiscovery/listeners/workloadmeta.go @@ -54,6 +54,8 @@ type workloadmetaListenerImpl struct { newService chan<- Service delService chan<- Service + + telemetryStore *telemetry.Store } var _ workloadmetaListener = &workloadmetaListenerImpl{} @@ -69,6 +71,7 @@ func newWorkloadmetaListener( workloadFilters *workloadmeta.Filter, processFn func(workloadmeta.Entity), wmeta workloadmeta.Component, + telemetryStore *telemetry.Store, ) (workloadmetaListener, error) { containerFilters, err := newContainerFilters() if err != nil { @@ -86,6 +89,8 @@ func newWorkloadmetaListener( services: make(map[string]Service), children: make(map[string]map[string]struct{}), + + telemetryStore: telemetryStore, }, nil } @@ -111,12 +116,16 @@ func (l *workloadmetaListenerImpl) AddService(svcID string, svc Service, parentS log.Tracef("%s received an updated service '%s', removing the old one", l.name, svc.GetServiceID()) l.delService <- old - telemetry.WatchedResources.Dec(l.name, kind) + if l.telemetryStore != nil { + l.telemetryStore.WatchedResources.Dec(l.name, kind) + } } l.services[svcID] = svc l.newService <- svc - telemetry.WatchedResources.Inc(l.name, kind) + if l.telemetryStore != nil { + l.telemetryStore.WatchedResources.Inc(l.name, kind) + } } func (l *workloadmetaListenerImpl) IsExcluded(ft containers.FilterType, annotations map[string]string, name, image, ns string) bool { @@ -234,7 +243,9 @@ func (l *workloadmetaListenerImpl) removeService(svcID string) { delete(l.services, svcID) l.delService <- svc - telemetry.WatchedResources.Dec(l.name, kindFromSvcID(svcID)) + if l.telemetryStore != nil { + l.telemetryStore.WatchedResources.Dec(l.name, kindFromSvcID(svcID)) + } } func buildSvcID(entityID workloadmeta.EntityID) string { diff --git a/comp/core/autodiscovery/providers/cloudfoundry.go b/comp/core/autodiscovery/providers/cloudfoundry.go index 6072332e634a5..3bfdd72d071da 100644 --- a/comp/core/autodiscovery/providers/cloudfoundry.go +++ b/comp/core/autodiscovery/providers/cloudfoundry.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/cloudproviders/cloudfoundry" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -33,7 +34,7 @@ type CloudFoundryConfigProvider struct { } // NewCloudFoundryConfigProvider instantiates a new CloudFoundryConfigProvider from given config -func NewCloudFoundryConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewCloudFoundryConfigProvider(*config.ConfigurationProviders, *telemetry.Store) (ConfigProvider, error) { cfp := CloudFoundryConfigProvider{ lastCollected: time.Now(), } diff --git a/comp/core/autodiscovery/providers/cloudfoundry_nop.go b/comp/core/autodiscovery/providers/cloudfoundry_nop.go index 624832210a31f..232dd534bd940 100644 --- a/comp/core/autodiscovery/providers/cloudfoundry_nop.go +++ b/comp/core/autodiscovery/providers/cloudfoundry_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewCloudFoundryConfigProvider instantiates a new CloudFoundryConfigProvider from given config -var NewCloudFoundryConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewCloudFoundryConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/clusterchecks.go b/comp/core/autodiscovery/providers/clusterchecks.go index 031f59b0380d1..d3c98c3a959f1 100644 --- a/comp/core/autodiscovery/providers/clusterchecks.go +++ b/comp/core/autodiscovery/providers/clusterchecks.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks/types" "github.com/DataDog/datadog-agent/pkg/config" ddErrors "github.com/DataDog/datadog-agent/pkg/errors" @@ -41,7 +42,7 @@ type ClusterChecksConfigProvider struct { // NewClusterChecksConfigProvider returns a new ConfigProvider collecting // cluster check configurations from the cluster-agent. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -func NewClusterChecksConfigProvider(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) { +func NewClusterChecksConfigProvider(providerConfig *config.ConfigurationProviders, _ *telemetry.Store) (ConfigProvider, error) { if providerConfig == nil { providerConfig = &config.ConfigurationProviders{} } diff --git a/comp/core/autodiscovery/providers/consul.go b/comp/core/autodiscovery/providers/consul.go index 0f0034c74e1cb..04957065793ff 100644 --- a/comp/core/autodiscovery/providers/consul.go +++ b/comp/core/autodiscovery/providers/consul.go @@ -20,6 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -52,7 +53,7 @@ type ConsulConfigProvider struct { } // NewConsulConfigProvider creates a client connection to consul and create a new ConsulConfigProvider -func NewConsulConfigProvider(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) { +func NewConsulConfigProvider(providerConfig *config.ConfigurationProviders, _ *telemetry.Store) (ConfigProvider, error) { if providerConfig == nil { providerConfig = &config.ConfigurationProviders{} } diff --git a/comp/core/autodiscovery/providers/consul_nop.go b/comp/core/autodiscovery/providers/consul_nop.go index 86ee0022a7d9e..5a7272f3bc3c1 100644 --- a/comp/core/autodiscovery/providers/consul_nop.go +++ b/comp/core/autodiscovery/providers/consul_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewConsulConfigProvider creates a client connection to consul and create a new ConsulConfigProvider -var NewConsulConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewConsulConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/container.go b/comp/core/autodiscovery/providers/container.go index d0d030430332d..7577fc2f89e03 100644 --- a/comp/core/autodiscovery/providers/container.go +++ b/comp/core/autodiscovery/providers/container.go @@ -29,15 +29,17 @@ type ContainerConfigProvider struct { configErrors map[string]ErrorMsgSet // map[entity name]ErrorMsgSet configCache map[string]map[string]integration.Config // map[entity name]map[config digest]integration.Config mu sync.RWMutex + telemetryStore *telemetry.Store } // NewContainerConfigProvider returns a new ConfigProvider subscribed to both container // and pods -func NewContainerConfigProvider(_ *config.ConfigurationProviders, wmeta workloadmeta.Component) (ConfigProvider, error) { +func NewContainerConfigProvider(_ *config.ConfigurationProviders, wmeta workloadmeta.Component, telemetryStore *telemetry.Store) (ConfigProvider, error) { return &ContainerConfigProvider{ workloadmetaStore: wmeta, configCache: make(map[string]map[string]integration.Config), configErrors: make(map[string]ErrorMsgSet), + telemetryStore: telemetryStore, }, nil } @@ -149,7 +151,9 @@ func (k *ContainerConfigProvider) processEvents(evBundle workloadmeta.EventBundl } } - telemetry.Errors.Set(float64(len(k.configErrors)), names.KubeContainer) + if k.telemetryStore != nil { + k.telemetryStore.Errors.Set(float64(len(k.configErrors)), names.KubeContainer) + } return changes } diff --git a/comp/core/autodiscovery/providers/container_test.go b/comp/core/autodiscovery/providers/container_test.go index dbb0e08984041..365d35407c262 100644 --- a/comp/core/autodiscovery/providers/container_test.go +++ b/comp/core/autodiscovery/providers/container_test.go @@ -14,9 +14,12 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" + "github.com/DataDog/datadog-agent/comp/core/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock" @@ -31,10 +34,14 @@ func TestProcessEvents(t *testing.T) { workloadmetafxmock.MockModule(), )) + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + cp := &ContainerConfigProvider{ workloadmetaStore: store, configCache: make(map[string]map[string]integration.Config), configErrors: make(map[string]ErrorMsgSet), + telemetryStore: telemetryStore, } tests := []struct { diff --git a/comp/core/autodiscovery/providers/endpointschecks.go b/comp/core/autodiscovery/providers/endpointschecks.go index 5d36b70092b2e..e6a55897c84cb 100644 --- a/comp/core/autodiscovery/providers/endpointschecks.go +++ b/comp/core/autodiscovery/providers/endpointschecks.go @@ -14,6 +14,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/clusteragent" @@ -34,7 +35,7 @@ type EndpointsChecksConfigProvider struct { // NewEndpointsChecksConfigProvider returns a new ConfigProvider collecting // endpoints check configurations from the cluster-agent. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -func NewEndpointsChecksConfigProvider(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) { +func NewEndpointsChecksConfigProvider(providerConfig *config.ConfigurationProviders, _ *telemetry.Store) (ConfigProvider, error) { c := &EndpointsChecksConfigProvider{ degradedDuration: defaultDegradedDeadline, } diff --git a/comp/core/autodiscovery/providers/endpointschecks_nop.go b/comp/core/autodiscovery/providers/endpointschecks_nop.go index 98914afdece89..6dc6386a834f9 100644 --- a/comp/core/autodiscovery/providers/endpointschecks_nop.go +++ b/comp/core/autodiscovery/providers/endpointschecks_nop.go @@ -7,9 +7,12 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewEndpointsChecksConfigProvider returns a new ConfigProvider collecting // endpoints check configurations from the cluster-agent. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -var NewEndpointsChecksConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewEndpointsChecksConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/etcd.go b/comp/core/autodiscovery/providers/etcd.go index 14c77f142294f..58178a0b6065d 100644 --- a/comp/core/autodiscovery/providers/etcd.go +++ b/comp/core/autodiscovery/providers/etcd.go @@ -19,6 +19,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -36,7 +37,7 @@ type EtcdConfigProvider struct { } // NewEtcdConfigProvider creates a client connection to etcd and create a new EtcdConfigProvider -func NewEtcdConfigProvider(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) { +func NewEtcdConfigProvider(providerConfig *config.ConfigurationProviders, _ *telemetry.Store) (ConfigProvider, error) { if providerConfig == nil { providerConfig = &config.ConfigurationProviders{} } diff --git a/comp/core/autodiscovery/providers/etcd_nop.go b/comp/core/autodiscovery/providers/etcd_nop.go index 7b10eaa273b09..b3e1f8675f8be 100644 --- a/comp/core/autodiscovery/providers/etcd_nop.go +++ b/comp/core/autodiscovery/providers/etcd_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewEtcdConfigProvider creates a client connection to etcd and create a new EtcdConfigProvider -var NewEtcdConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewEtcdConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/file.go b/comp/core/autodiscovery/providers/file.go index a9e1c26117b54..0e764a6d30fa2 100644 --- a/comp/core/autodiscovery/providers/file.go +++ b/comp/core/autodiscovery/providers/file.go @@ -15,13 +15,15 @@ import ( // FileConfigProvider collect configuration files from disk type FileConfigProvider struct { - Errors map[string]string + Errors map[string]string + telemetryStore *telemetry.Store } // NewFileConfigProvider creates a new FileConfigProvider. -func NewFileConfigProvider() *FileConfigProvider { +func NewFileConfigProvider(telemetryStore *telemetry.Store) *FileConfigProvider { return &FileConfigProvider{ - Errors: make(map[string]string), + Errors: make(map[string]string), + telemetryStore: telemetryStore, } } @@ -36,7 +38,9 @@ func (c *FileConfigProvider) Collect(ctx context.Context) ([]integration.Config, } c.Errors = errors - telemetry.Errors.Set(float64(len(errors)), names.File) + if c.telemetryStore != nil { + c.telemetryStore.Errors.Set(float64(len(errors)), names.File) + } return configs, nil } diff --git a/comp/core/autodiscovery/providers/file_test.go b/comp/core/autodiscovery/providers/file_test.go index 95fa18fe0d2e2..eaed5fc0dfa39 100644 --- a/comp/core/autodiscovery/providers/file_test.go +++ b/comp/core/autodiscovery/providers/file_test.go @@ -11,7 +11,11 @@ import ( "testing" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/stretchr/testify/assert" ) @@ -20,8 +24,12 @@ func TestCollect(t *testing.T) { ctx := context.Background() config.Datadog().SetWithoutSource("ignore_autoconf", []string{"ignored"}) paths := []string{"tests", "foo/bar"} + + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + ResetReader(paths) - provider := NewFileConfigProvider() + provider := NewFileConfigProvider(telemetryStore) configs, err := provider.Collect(ctx) assert.Nil(t, err) @@ -86,7 +94,10 @@ func TestEnvVarReplacement(t *testing.T) { paths := []string{"tests"} ResetReader(paths) - provider := NewFileConfigProvider() + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + + provider := NewFileConfigProvider(telemetryStore) configs, err := provider.Collect(ctx) assert.Nil(t, err) diff --git a/comp/core/autodiscovery/providers/kube_endpoints.go b/comp/core/autodiscovery/providers/kube_endpoints.go index 1041f45fb19b8..a5263d4be9fe9 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints.go +++ b/comp/core/autodiscovery/providers/kube_endpoints.go @@ -46,6 +46,7 @@ type kubeEndpointsConfigProvider struct { upToDate bool monitoredEndpoints map[string]bool configErrors map[string]ErrorMsgSet + telemetryStore *telemetry.Store } // configInfo contains an endpoint check config template with its name and namespace @@ -58,7 +59,7 @@ type configInfo struct { // NewKubeEndpointsConfigProvider returns a new ConfigProvider connected to apiserver. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -func NewKubeEndpointsConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewKubeEndpointsConfigProvider(_ *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) { // Using GetAPIClient (no wait) as Client should already be initialized by Cluster Agent main entrypoint before ac, err := apiserver.GetAPIClient() if err != nil { @@ -74,6 +75,7 @@ func NewKubeEndpointsConfigProvider(*config.ConfigurationProviders) (ConfigProvi serviceLister: servicesInformer.Lister(), monitoredEndpoints: make(map[string]bool), configErrors: make(map[string]ErrorMsgSet), + telemetryStore: telemetryStore, } if _, err := servicesInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -280,7 +282,9 @@ func (k *kubeEndpointsConfigProvider) parseServiceAnnotationsForEndpoints(servic k.cleanErrorsOfDeletedEndpoints(setEndpointIDs) - telemetry.Errors.Set(float64(len(k.configErrors)), names.KubeEndpoints) + if k.telemetryStore != nil { + k.telemetryStore.Errors.Set(float64(len(k.configErrors)), names.KubeEndpoints) + } return configsInfo } diff --git a/comp/core/autodiscovery/providers/kube_endpoints_file.go b/comp/core/autodiscovery/providers/kube_endpoints_file.go index 2223c1b6ec8dc..67b19d1d600c8 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints_file.go +++ b/comp/core/autodiscovery/providers/kube_endpoints_file.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -56,7 +57,7 @@ type KubeEndpointsFileConfigProvider struct { } // NewKubeEndpointsFileConfigProvider returns a new KubeEndpointsFileConfigProvider -func NewKubeEndpointsFileConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewKubeEndpointsFileConfigProvider(*config.ConfigurationProviders, *telemetry.Store) (ConfigProvider, error) { templates, _, err := ReadConfigFiles(WithAdvancedADOnly) if err != nil { return nil, err diff --git a/comp/core/autodiscovery/providers/kube_endpoints_file_nop.go b/comp/core/autodiscovery/providers/kube_endpoints_file_nop.go index aa4ad62fcc5b3..a2e5b1ef58239 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints_file_nop.go +++ b/comp/core/autodiscovery/providers/kube_endpoints_file_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewKubeEndpointsFileConfigProvider returns a new KubeEndpointsFileConfigProvider -var NewKubeEndpointsFileConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewKubeEndpointsFileConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/kube_endpoints_nop.go b/comp/core/autodiscovery/providers/kube_endpoints_nop.go index 13af3fbe7c857..a6004ed3cf91a 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints_nop.go +++ b/comp/core/autodiscovery/providers/kube_endpoints_nop.go @@ -7,8 +7,11 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewKubeEndpointsConfigProvider returns a new ConfigProvider connected to apiserver. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -var NewKubeEndpointsConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewKubeEndpointsConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/kube_endpoints_test.go b/comp/core/autodiscovery/providers/kube_endpoints_test.go index c3b86156b3cce..36b78d68d8add 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints_test.go +++ b/comp/core/autodiscovery/providers/kube_endpoints_test.go @@ -24,7 +24,11 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/kubernetes" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" ) @@ -35,6 +39,9 @@ var ( ) func TestParseKubeServiceAnnotationsForEndpoints(t *testing.T) { + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + for _, tc := range []struct { name string service *v1.Service @@ -269,7 +276,9 @@ func TestParseKubeServiceAnnotationsForEndpoints(t *testing.T) { if tc.hybrid { cfg.SetWithoutSource("cluster_checks.support_hybrid_ignore_ad_tags", true) } - provider := kubeEndpointsConfigProvider{} + provider := kubeEndpointsConfigProvider{ + telemetryStore: telemetryStore, + } cfgs := provider.parseServiceAnnotationsForEndpoints([]*v1.Service{tc.service}, cfg) assert.EqualValues(t, tc.expectedOut, cfgs) }) @@ -881,6 +890,9 @@ func TestInvalidateIfChangedEndpoints(t *testing.T) { } func TestGetConfigErrors_KubeEndpoints(t *testing.T) { + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + serviceWithErrors := v1.Service{ TypeMeta: metav1.TypeMeta{ Kind: kubernetes.ServiceKind, @@ -1044,6 +1056,7 @@ func TestGetConfigErrors_KubeEndpoints(t *testing.T) { endpointsLister: endpointsLister, configErrors: test.currentErrors, monitoredEndpoints: make(map[string]bool), + telemetryStore: telemetryStore, } configs, err := provider.Collect(context.TODO()) diff --git a/comp/core/autodiscovery/providers/kube_services.go b/comp/core/autodiscovery/providers/kube_services.go index 4063d96c5fe42..53f9cf65aec98 100644 --- a/comp/core/autodiscovery/providers/kube_services.go +++ b/comp/core/autodiscovery/providers/kube_services.go @@ -33,14 +33,15 @@ const ( // KubeServiceConfigProvider implements the ConfigProvider interface for the apiserver. type KubeServiceConfigProvider struct { - lister listersv1.ServiceLister - upToDate bool - configErrors map[string]ErrorMsgSet + lister listersv1.ServiceLister + upToDate bool + configErrors map[string]ErrorMsgSet + telemetryStore *telemetry.Store } // NewKubeServiceConfigProvider returns a new ConfigProvider connected to apiserver. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -func NewKubeServiceConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewKubeServiceConfigProvider(_ *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) { // Using GetAPIClient() (no retry) ac, err := apiserver.GetAPIClient() if err != nil { @@ -53,8 +54,9 @@ func NewKubeServiceConfigProvider(*config.ConfigurationProviders) (ConfigProvide } p := &KubeServiceConfigProvider{ - lister: servicesInformer.Lister(), - configErrors: make(map[string]ErrorMsgSet), + lister: servicesInformer.Lister(), + configErrors: make(map[string]ErrorMsgSet), + telemetryStore: telemetryStore, } if _, err := servicesInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -201,7 +203,9 @@ func (k *KubeServiceConfigProvider) parseServiceAnnotations(services []*v1.Servi k.cleanErrorsOfDeletedServices(setServiceIDs) - telemetry.Errors.Set(float64(len(k.configErrors)), names.KubeServices) + if k.telemetryStore != nil { + k.telemetryStore.Errors.Set(float64(len(k.configErrors)), names.KubeServices) + } return configs, nil } diff --git a/comp/core/autodiscovery/providers/kube_services_file.go b/comp/core/autodiscovery/providers/kube_services_file.go index 9057ee1c18b4c..721ed15e5d79a 100644 --- a/comp/core/autodiscovery/providers/kube_services_file.go +++ b/comp/core/autodiscovery/providers/kube_services_file.go @@ -12,6 +12,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" ) @@ -21,7 +22,7 @@ type KubeServiceFileConfigProvider struct { } // NewKubeServiceFileConfigProvider returns a new KubeServiceFileConfigProvider -func NewKubeServiceFileConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewKubeServiceFileConfigProvider(*config.ConfigurationProviders, *telemetry.Store) (ConfigProvider, error) { return &KubeServiceFileConfigProvider{}, nil } diff --git a/comp/core/autodiscovery/providers/kube_services_file_nop.go b/comp/core/autodiscovery/providers/kube_services_file_nop.go index e7592bcef86b3..8724b87313061 100644 --- a/comp/core/autodiscovery/providers/kube_services_file_nop.go +++ b/comp/core/autodiscovery/providers/kube_services_file_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewKubeServiceFileConfigProvider returns a new KubeServiceFileConfigProvider -var NewKubeServiceFileConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewKubeServiceFileConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/kube_services_nop.go b/comp/core/autodiscovery/providers/kube_services_nop.go index 17567b64d0469..4f64323cb985c 100644 --- a/comp/core/autodiscovery/providers/kube_services_nop.go +++ b/comp/core/autodiscovery/providers/kube_services_nop.go @@ -7,8 +7,11 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewKubeServiceConfigProvider returns a new ConfigProvider connected to apiserver. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -var NewKubeServiceConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewKubeServiceConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/kube_services_test.go b/comp/core/autodiscovery/providers/kube_services_test.go index 7f510f6cfd179..fcecd31581219 100644 --- a/comp/core/autodiscovery/providers/kube_services_test.go +++ b/comp/core/autodiscovery/providers/kube_services_test.go @@ -24,11 +24,18 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry/telemetryimpl" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/kubernetes" ) func TestParseKubeServiceAnnotations(t *testing.T) { + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + for _, tc := range []struct { name string service *v1.Service @@ -240,7 +247,9 @@ func TestParseKubeServiceAnnotations(t *testing.T) { cfg.SetWithoutSource("cluster_checks.support_hybrid_ignore_ad_tags", true) } - provider := KubeServiceConfigProvider{} + provider := KubeServiceConfigProvider{ + telemetryStore: telemetryStore, + } cfgs, _ := provider.parseServiceAnnotations([]*v1.Service{tc.service}, cfg) assert.EqualValues(t, tc.expectedOut, cfgs) }) @@ -341,6 +350,9 @@ func TestInvalidateIfChanged(t *testing.T) { } func TestGetConfigErrors_KubeServices(t *testing.T) { + telemetry := fxutil.Test[telemetry.Component](t, telemetryimpl.MockModule()) + telemetryStore := acTelemetry.NewStore(telemetry) + serviceWithErrors := v1.Service{ TypeMeta: metav1.TypeMeta{ Kind: kubernetes.ServiceKind, @@ -462,8 +474,9 @@ func TestGetConfigErrors_KubeServices(t *testing.T) { factory.WaitForCacheSync(stop) provider := KubeServiceConfigProvider{ - lister: lister, - configErrors: test.currentErrors, + lister: lister, + configErrors: test.currentErrors, + telemetryStore: telemetryStore, } configs, err := provider.Collect(context.TODO()) diff --git a/comp/core/autodiscovery/providers/prometheus_pods.go b/comp/core/autodiscovery/providers/prometheus_pods.go index 749ebb3dcf9ef..37634a8dbade8 100644 --- a/comp/core/autodiscovery/providers/prometheus_pods.go +++ b/comp/core/autodiscovery/providers/prometheus_pods.go @@ -14,6 +14,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet" ) @@ -27,7 +28,7 @@ type PrometheusPodsConfigProvider struct { // NewPrometheusPodsConfigProvider returns a new Prometheus ConfigProvider connected to kubelet. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -func NewPrometheusPodsConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewPrometheusPodsConfigProvider(*config.ConfigurationProviders, *telemetry.Store) (ConfigProvider, error) { checks, err := getPrometheusConfigs() if err != nil { return nil, err diff --git a/comp/core/autodiscovery/providers/prometheus_pods_nop.go b/comp/core/autodiscovery/providers/prometheus_pods_nop.go index edddc1fb52e0d..e6ce37bbc9a27 100644 --- a/comp/core/autodiscovery/providers/prometheus_pods_nop.go +++ b/comp/core/autodiscovery/providers/prometheus_pods_nop.go @@ -7,8 +7,11 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewPrometheusPodsConfigProvider returns a new Prometheus ConfigProvider connected to kubelet. // Connectivity is not checked at this stage to allow for retries, Collect will do it. -var NewPrometheusPodsConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewPrometheusPodsConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/prometheus_services.go b/comp/core/autodiscovery/providers/prometheus_services.go index 57d299dfeaf82..72d5a33b28326 100644 --- a/comp/core/autodiscovery/providers/prometheus_services.go +++ b/comp/core/autodiscovery/providers/prometheus_services.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -65,7 +66,7 @@ type PrometheusServicesConfigProvider struct { } // NewPrometheusServicesConfigProvider returns a new Prometheus ConfigProvider connected to kube apiserver -func NewPrometheusServicesConfigProvider(*config.ConfigurationProviders) (ConfigProvider, error) { +func NewPrometheusServicesConfigProvider(*config.ConfigurationProviders, *telemetry.Store) (ConfigProvider, error) { // Using GetAPIClient (no wait) as Client should already be initialized by Cluster Agent main entrypoint before ac, err := apiserver.GetAPIClient() if err != nil { diff --git a/comp/core/autodiscovery/providers/prometheus_services_nop.go b/comp/core/autodiscovery/providers/prometheus_services_nop.go index 1c76dd86a5cee..945daf2c80eee 100644 --- a/comp/core/autodiscovery/providers/prometheus_services_nop.go +++ b/comp/core/autodiscovery/providers/prometheus_services_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewPrometheusServicesConfigProvider returns a new Prometheus ConfigProvider connected to kube apiserver -var NewPrometheusServicesConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewPrometheusServicesConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/providers/providers.go b/comp/core/autodiscovery/providers/providers.go index f8fb043886afe..907e4f1b7737d 100644 --- a/comp/core/autodiscovery/providers/providers.go +++ b/comp/core/autodiscovery/providers/providers.go @@ -10,6 +10,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -17,14 +18,15 @@ import ( // RegisterProvider adds a loader to the providers catalog func RegisterProvider(name string, - factory func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error), + factory func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error), providerCatalog map[string]ConfigProviderFactory) { RegisterProviderWithComponents( name, - func(providerConfig *config.ConfigurationProviders, wmeta workloadmeta.Component) (ConfigProvider, error) { - return factory(providerConfig) + func(providerConfig *config.ConfigurationProviders, wmeta workloadmeta.Component, telemetryStore *telemetry.Store) (ConfigProvider, error) { + return factory(providerConfig, telemetryStore) }, - providerCatalog) + providerCatalog, + ) } // RegisterProviderWithComponents adds a loader to the providers catalog @@ -59,7 +61,7 @@ func RegisterProviders(providerCatalog map[string]ConfigProviderFactory) { } // ConfigProviderFactory is any function capable to create a ConfigProvider instance -type ConfigProviderFactory func(providerConfig *config.ConfigurationProviders, wmeta workloadmeta.Component) (ConfigProvider, error) +type ConfigProviderFactory func(providerConfig *config.ConfigurationProviders, wmeta workloadmeta.Component, telemetryStore *telemetry.Store) (ConfigProvider, error) // ConfigProvider represents a source of `integration.Config` values // that can either be applied immediately or resolved for a service and diff --git a/comp/core/autodiscovery/providers/zookeeper.go b/comp/core/autodiscovery/providers/zookeeper.go index b21d580b4cc50..e9da1ed8c515e 100644 --- a/comp/core/autodiscovery/providers/zookeeper.go +++ b/comp/core/autodiscovery/providers/zookeeper.go @@ -20,6 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/common/utils" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/providers/names" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -40,7 +41,7 @@ type ZookeeperConfigProvider struct { } // NewZookeeperConfigProvider returns a new Client connected to a Zookeeper backend. -func NewZookeeperConfigProvider(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) { +func NewZookeeperConfigProvider(providerConfig *config.ConfigurationProviders, _ *telemetry.Store) (ConfigProvider, error) { if providerConfig == nil { providerConfig = &config.ConfigurationProviders{} } diff --git a/comp/core/autodiscovery/providers/zookeeper_nop.go b/comp/core/autodiscovery/providers/zookeeper_nop.go index 4961235ae6069..91306ec420ad4 100644 --- a/comp/core/autodiscovery/providers/zookeeper_nop.go +++ b/comp/core/autodiscovery/providers/zookeeper_nop.go @@ -7,7 +7,10 @@ package providers -import "github.com/DataDog/datadog-agent/pkg/config" +import ( + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" + "github.com/DataDog/datadog-agent/pkg/config" +) // NewZookeeperConfigProvider returns a new Client connected to a Zookeeper backend. -var NewZookeeperConfigProvider func(providerConfig *config.ConfigurationProviders) (ConfigProvider, error) +var NewZookeeperConfigProvider func(providerConfig *config.ConfigurationProviders, telemetryStore *telemetry.Store) (ConfigProvider, error) diff --git a/comp/core/autodiscovery/telemetry/telemetry.go b/comp/core/autodiscovery/telemetry/telemetry.go index 019854afe3fb2..b712d1e5f1b13 100644 --- a/comp/core/autodiscovery/telemetry/telemetry.go +++ b/comp/core/autodiscovery/telemetry/telemetry.go @@ -7,7 +7,7 @@ package telemetry import ( - "github.com/DataDog/datadog-agent/pkg/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/prometheus/client_golang/prometheus" ) @@ -25,42 +25,50 @@ var ( commonOpts = telemetry.Options{NoDoubleUnderscoreSep: true} ) -var ( +// Store holds all the telemetry metrics for Autodiscovery. +type Store struct { // ScheduledConfigs tracks how many configs are scheduled. - ScheduledConfigs = telemetry.NewGaugeWithOpts( - subsystem, - "scheduled_configs", - []string{"provider", "type"}, - "Number of configs scheduled in Autodiscovery by provider and type.", - commonOpts, - ) - + ScheduledConfigs telemetry.Gauge // WatchedResources tracks how many resources are watched by AD listeners. - WatchedResources = telemetry.NewGaugeWithOpts( - subsystem, - "watched_resources", - []string{"listener", "kind"}, - "Number of resources watched in Autodiscovery by listener and kind.", - commonOpts, - ) - + WatchedResources telemetry.Gauge // Errors tracks the current number of AD configs with errors by AD providers. - Errors = telemetry.NewGaugeWithOpts( - subsystem, - "errors", - []string{"provider"}, - "Current number of Autodiscovery configs with errors by provider.", - commonOpts, - ) - + Errors telemetry.Gauge // PollDuration tracks the configs poll duration by AD providers. - PollDuration = telemetry.NewHistogramWithOpts( - subsystem, - "poll_duration", - []string{"provider"}, - "Poll duration distribution by config provider (in seconds).", - // The default prometheus buckets are adapted to measure response time of network services - prometheus.DefBuckets, - telemetry.Options{NoDoubleUnderscoreSep: true}, - ) -) + PollDuration telemetry.Histogram +} + +// NewStore returns a new Store. +func NewStore(telemetryComp telemetry.Component) *Store { + return &Store{ + ScheduledConfigs: telemetryComp.NewGaugeWithOpts( + subsystem, + "scheduled_configs", + []string{"provider", "type"}, + "Number of configs scheduled in Autodiscovery by provider and type.", + commonOpts, + ), + WatchedResources: telemetryComp.NewGaugeWithOpts( + subsystem, + "watched_resources", + []string{"listener", "kind"}, + "Number of resources watched in Autodiscovery by listener and kind.", + commonOpts, + ), + Errors: telemetryComp.NewGaugeWithOpts( + subsystem, + "errors", + []string{"provider"}, + "Current number of Autodiscovery configs with errors by provider.", + commonOpts, + ), + PollDuration: telemetryComp.NewHistogramWithOpts( + subsystem, + "poll_duration", + []string{"provider"}, + "Poll duration distribution by config provider (in seconds).", + // The default prometheus buckets are adapted to measure response time of network services + prometheus.DefBuckets, + telemetry.Options{NoDoubleUnderscoreSep: true}, + ), + } +} diff --git a/test/integration/config_providers/etcd/etcd_provider_test.go b/test/integration/config_providers/etcd/etcd_provider_test.go index 9a232af0acdfc..228495b66e077 100644 --- a/test/integration/config_providers/etcd/etcd_provider_test.go +++ b/test/integration/config_providers/etcd/etcd_provider_test.go @@ -146,7 +146,7 @@ func (suite *EtcdTestSuite) TestWorkingConnectionAnon() { TemplateURL: suite.etcdURL, TemplateDir: "/foo", } - p, err := providers.NewEtcdConfigProvider(&config) + p, err := providers.NewEtcdConfigProvider(&config, nil) if err != nil { panic(err) } @@ -167,7 +167,7 @@ func (suite *EtcdTestSuite) TestBadConnection() { TemplateURL: "http://127.0.0.1:1337", TemplateDir: "/foo", } - p, err := providers.NewEtcdConfigProvider(&config) + p, err := providers.NewEtcdConfigProvider(&config, nil) assert.Nil(suite.T(), err) checks, err := p.(providers.CollectingConfigProvider).Collect(ctx) @@ -184,7 +184,7 @@ func (suite *EtcdTestSuite) TestWorkingAuth() { Username: etcdUser, Password: etcdPass, } - p, err := providers.NewEtcdConfigProvider(&config) + p, err := providers.NewEtcdConfigProvider(&config, nil) assert.Nil(suite.T(), err) checks, err := p.(providers.CollectingConfigProvider).Collect(ctx) @@ -201,7 +201,7 @@ func (suite *EtcdTestSuite) TestBadAuth() { Username: etcdUser, Password: "invalid", } - p, err := providers.NewEtcdConfigProvider(&config) + p, err := providers.NewEtcdConfigProvider(&config, nil) assert.Nil(suite.T(), err) checks, err := p.(providers.CollectingConfigProvider).Collect(ctx) diff --git a/test/integration/config_providers/zookeeper/zookeeper_provider_test.go b/test/integration/config_providers/zookeeper/zookeeper_provider_test.go index fcadfa9acaac7..fd230819aff43 100644 --- a/test/integration/config_providers/zookeeper/zookeeper_provider_test.go +++ b/test/integration/config_providers/zookeeper/zookeeper_provider_test.go @@ -132,7 +132,7 @@ func (suite *ZkTestSuite) populate() error { func (suite *ZkTestSuite) TestCollect() { ctx := context.Background() - zk, err := providers.NewZookeeperConfigProvider(&suite.providerConfig) + zk, err := providers.NewZookeeperConfigProvider(&suite.providerConfig, nil) require.Nil(suite.T(), err) templates, err := zk.(providers.CollectingConfigProvider).Collect(ctx) diff --git a/test/integration/listeners/docker/docker_listener_test.go b/test/integration/listeners/docker/docker_listener_test.go index 8bd834a2d1b8b..8981e860b053e 100644 --- a/test/integration/listeners/docker/docker_listener_test.go +++ b/test/integration/listeners/docker/docker_listener_test.go @@ -23,9 +23,11 @@ import ( "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/listeners" + acTelemetry "github.com/DataDog/datadog-agent/comp/core/autodiscovery/telemetry" compcfg "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" + "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" @@ -39,20 +41,22 @@ import ( type DockerListenerTestSuite struct { suite.Suite - compose utils.ComposeConf - listener listeners.ServiceListener - dockerutil *docker.DockerUtil - newSvc chan listeners.Service - delSvc chan listeners.Service - stop chan struct{} - m sync.RWMutex - wmeta workloadmeta.Component + compose utils.ComposeConf + listener listeners.ServiceListener + dockerutil *docker.DockerUtil + newSvc chan listeners.Service + delSvc chan listeners.Service + stop chan struct{} + m sync.RWMutex + wmeta workloadmeta.Component + telemetryStore *acTelemetry.Store } type deps struct { fx.In - Tagger tagger.Component - WMeta workloadmeta.Component + Tagger tagger.Component + WMeta workloadmeta.Component + Telemetry telemetry.Component } func (suite *DockerListenerTestSuite) SetupSuite() { @@ -87,6 +91,7 @@ func (suite *DockerListenerTestSuite) SetupSuite() { fx.Supply(tagger.NewTaggerParams()), )) suite.wmeta = deps.WMeta + suite.telemetryStore = acTelemetry.NewStore(deps.Telemetry) suite.dockerutil, err = docker.GetDockerUtil() require.Nil(suite.T(), err, "can't connect to docker") @@ -101,7 +106,7 @@ func (suite *DockerListenerTestSuite) TearDownSuite() { } func (suite *DockerListenerTestSuite) SetupTest() { - dl, err := listeners.NewContainerListener(&config.Listeners{}, optional.NewOption(suite.wmeta)) + dl, err := listeners.NewContainerListener(&config.Listeners{}, optional.NewOption(suite.wmeta), suite.telemetryStore) if err != nil { panic(err) }