diff --git a/ydb/core/base/appdata.cpp b/ydb/core/base/appdata.cpp index d2a1d68c32bd..469ecc921edb 100644 --- a/ydb/core/base/appdata.cpp +++ b/ydb/core/base/appdata.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -123,6 +124,7 @@ TAppData::TAppData( , MemoryControllerConfig(Impl->MemoryControllerConfig) , ReplicationConfig(Impl->ReplicationConfig) , KikimrShouldContinue(kikimrShouldContinue) + , TracingConfigurator(MakeIntrusive(TimeProvider, RandomProvider)) {} TAppData::~TAppData() diff --git a/ydb/core/base/appdata_fwd.h b/ydb/core/base/appdata_fwd.h index be4cae86bfdf..c88f82f2af7c 100644 --- a/ydb/core/base/appdata_fwd.h +++ b/ydb/core/base/appdata_fwd.h @@ -22,6 +22,9 @@ namespace NKikimr { namespace NSharedCache { class TSharedCachePages; } + namespace NJaegerTracing { + class TSamplingThrottlingConfigurator; + } } namespace NKikimrCms { @@ -267,6 +270,9 @@ struct TAppData { bool YamlConfigEnabled = false; + // Tracing configurator (look for tracing config in ydb/core/jaeger_tracing/actors_tracing_control) + TIntrusivePtr TracingConfigurator; + TAppData( ui32 sysPoolId, ui32 userPoolId, ui32 ioPoolId, ui32 batchPoolId, TMap servicePools, diff --git a/ydb/core/base/wilson_tracing_control.cpp b/ydb/core/base/wilson_tracing_control.cpp new file mode 100644 index 000000000000..120bf5a8fabf --- /dev/null +++ b/ydb/core/base/wilson_tracing_control.cpp @@ -0,0 +1,72 @@ +#include "wilson_tracing_control.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace NKikimr::NJaegerTracing { + +namespace { + +Y_POD_STATIC_THREAD(TSamplingThrottlingControl*) TracingControlRawPtr; + +class TSamplingThrottlingControlTlsHolder { +public: + TSamplingThrottlingControlTlsHolder() + : Control(CreateNewTracingControl()) + {} + + TSamplingThrottlingControl* GetTracingControlPtr() { + if (Y_UNLIKELY(!Control)) { + Control = CreateNewTracingControl(); + } + return Control.Get(); + } + + void ResetTracingControl() { + Control = nullptr; + } + +private: + static TIntrusivePtr CreateNewTracingControl() { + Y_ASSERT(HasAppData()); // In general we must call this from actor thread + if (Y_UNLIKELY(!HasAppData())) { + return nullptr; + } + + return AppData()->TracingConfigurator->GetControl(); + } + +private: + TIntrusivePtr Control; +}; + +TSamplingThrottlingControl* GetTracingControlTls() { + if (Y_UNLIKELY(!TracingControlRawPtr)) { + TracingControlRawPtr = FastTlsSingleton()->GetTracingControlPtr(); + } + return TracingControlRawPtr; +} + +} // namespace + +void HandleTracing(NWilson::TTraceId& traceId, const TRequestDiscriminator& discriminator) { + TSamplingThrottlingControl* control = GetTracingControlTls(); + if (Y_LIKELY(control)) { + control->HandleTracing(traceId, discriminator); + } +} + +void ClearTracingControl() { + if (TracingControlRawPtr) { + TracingControlRawPtr = nullptr; + FastTlsSingleton()->ResetTracingControl(); + } +} + +} // namespace NKikimr::NJaegerTracing diff --git a/ydb/core/base/wilson_tracing_control.h b/ydb/core/base/wilson_tracing_control.h new file mode 100644 index 000000000000..1d6bd40c7775 --- /dev/null +++ b/ydb/core/base/wilson_tracing_control.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace NKikimr::NJaegerTracing { + +// Generate a new trace id (or throttle existing one) +// with probability according to current configuration and request type. +// Can be called from actor system threads. +void HandleTracing(NWilson::TTraceId& traceId, const TRequestDiscriminator& discriminator); + +// For test purposes +// Clears tracing control TLS variables that depend on AppData +void ClearTracingControl(); + +} // namespace NKikimr::NJaegerTracing diff --git a/ydb/core/base/ya.make b/ydb/core/base/ya.make index 70949ec0af24..e327f19a674b 100644 --- a/ydb/core/base/ya.make +++ b/ydb/core/base/ya.make @@ -11,6 +11,7 @@ SRCS( board_replica.cpp blobstorage.h blobstorage.cpp + blobstorage_grouptype.cpp channel_profiles.h counters.cpp counters.h @@ -72,7 +73,7 @@ SRCS( tx_processing.h tx_processing.cpp user_registry.h - blobstorage_grouptype.cpp + wilson_tracing_control.cpp ) PEERDIR( @@ -92,6 +93,7 @@ PEERDIR( ydb/core/debug ydb/core/erasure ydb/core/graph/api + ydb/core/jaeger_tracing ydb/core/protos ydb/core/protos/out ydb/library/aclib diff --git a/ydb/core/cms/console/jaeger_tracing_configurator.cpp b/ydb/core/cms/console/jaeger_tracing_configurator.cpp index b20fa0429eb5..ed31a889d87e 100644 --- a/ydb/core/cms/console/jaeger_tracing_configurator.cpp +++ b/ydb/core/cms/console/jaeger_tracing_configurator.cpp @@ -17,7 +17,7 @@ class TJaegerTracingConfigurator : public TActorBootstrapped tracingConfigurator, NKikimrConfig::TTracingConfig cfg); void Bootstrap(const TActorContext& ctx); @@ -35,12 +35,12 @@ class TJaegerTracingConfigurator : public TActorBootstrapped GetDatabase(const NKikimrConfig::TTracingConfig::TSelectors& selectors); static TSettings> GetSettings(const NKikimrConfig::TTracingConfig& cfg); - TSamplingThrottlingConfigurator TracingConfigurator; + TIntrusivePtr TracingConfigurator; NKikimrConfig::TTracingConfig initialConfig; }; TJaegerTracingConfigurator::TJaegerTracingConfigurator( - TSamplingThrottlingConfigurator tracingConfigurator, + TIntrusivePtr tracingConfigurator, NKikimrConfig::TTracingConfig cfg) : TracingConfigurator(std::move(tracingConfigurator)) , initialConfig(std::move(cfg)) @@ -73,7 +73,7 @@ void TJaegerTracingConfigurator::Handle(TEvConsole::TEvConfigNotificationRequest void TJaegerTracingConfigurator::ApplyConfigs(const NKikimrConfig::TTracingConfig& cfg) { auto settings = GetSettings(cfg); - return TracingConfigurator.UpdateSettings(std::move(settings)); + return TracingConfigurator->UpdateSettings(std::move(settings)); } TVector TJaegerTracingConfigurator::GetRequestTypes(const NKikimrConfig::TTracingConfig::TSelectors& selectors) { @@ -213,7 +213,7 @@ TSettings> TJaegerTracingConfigurator::Get return settings; } -IActor* CreateJaegerTracingConfigurator(TSamplingThrottlingConfigurator tracingConfigurator, +IActor* CreateJaegerTracingConfigurator(TIntrusivePtr tracingConfigurator, NKikimrConfig::TTracingConfig cfg) { return new TJaegerTracingConfigurator(std::move(tracingConfigurator), std::move(cfg)); } diff --git a/ydb/core/cms/console/jaeger_tracing_configurator.h b/ydb/core/cms/console/jaeger_tracing_configurator.h index 54b22bd0741c..76d546f385bb 100644 --- a/ydb/core/cms/console/jaeger_tracing_configurator.h +++ b/ydb/core/cms/console/jaeger_tracing_configurator.h @@ -7,7 +7,7 @@ namespace NKikimr::NConsole { -IActor* CreateJaegerTracingConfigurator(NJaegerTracing::TSamplingThrottlingConfigurator tracingConfigurator, +IActor* CreateJaegerTracingConfigurator(TIntrusivePtr tracingConfigurator, NKikimrConfig::TTracingConfig cfg); } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/jaeger_tracing_configurator_ut.cpp b/ydb/core/cms/console/jaeger_tracing_configurator_ut.cpp index 8867c49b4522..1da0b2270e1d 100644 --- a/ydb/core/cms/console/jaeger_tracing_configurator_ut.cpp +++ b/ydb/core/cms/console/jaeger_tracing_configurator_ut.cpp @@ -52,7 +52,7 @@ TTenantTestConfig DefaultConsoleTestConfig() { void InitJaegerTracingConfigurator( TTenantTestRuntime& runtime, - TSamplingThrottlingConfigurator configurator, + TIntrusivePtr configurator, const NKikimrConfig::TTracingConfig& initCfg ) { runtime.Register(CreateJaegerTracingConfigurator(std::move(configurator), initCfg)); @@ -99,7 +99,7 @@ class TTracingControls { std::pair HandleTracing(bool isExternal, TRequestDiscriminator discriminator) { auto& control = RandomChoice(Controls); - + NWilson::TTraceId traceId; if (isExternal) { traceId = NWilson::TTraceId::NewTraceId(TComponentTracingLevels::ProductionVerbose, Max()); @@ -125,13 +125,13 @@ class TTracingControls { TVector> Controls; }; -std::pair +std::pair> CreateSamplingThrottlingConfigurator(size_t n, TIntrusivePtr timeProvider) { auto randomProvider = CreateDefaultRandomProvider(); - TSamplingThrottlingConfigurator configurator(timeProvider, randomProvider); + TIntrusivePtr configurator = MakeIntrusive(timeProvider, randomProvider); TVector> controls; for (size_t i = 0; i < n; ++i) { - controls.emplace_back(configurator.GetControl()); + controls.emplace_back(configurator->GetControl()); } return {TTracingControls(std::move(controls)), std::move(configurator)}; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 40d67c946598..3f073ab52be3 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -1633,11 +1633,10 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se if (!IsServiceInitialized(setup, NGRpcService::CreateGRpcRequestProxyId(0))) { const size_t proxyCount = Config.HasGRpcConfig() ? Config.GetGRpcConfig().GetGRpcProxyCount() : 1UL; - NJaegerTracing::TSamplingThrottlingConfigurator tracingConfigurator(appData->TimeProvider, appData->RandomProvider); for (size_t i = 0; i < proxyCount; ++i) { auto grpcReqProxy = Config.HasGRpcConfig() && Config.GetGRpcConfig().GetSkipSchemeCheck() ? NGRpcService::CreateGRpcRequestProxySimple(Config) - : NGRpcService::CreateGRpcRequestProxy(Config, tracingConfigurator.GetControl()); + : NGRpcService::CreateGRpcRequestProxy(Config); setup->LocalServices.push_back(std::pair(NGRpcService::CreateGRpcRequestProxyId(i), TActorSetupCmd(grpcReqProxy, TMailboxType::ReadAsFilled, @@ -1646,7 +1645,7 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se setup->LocalServices.push_back(std::pair( TActorId(), TActorSetupCmd( - NConsole::CreateJaegerTracingConfigurator(std::move(tracingConfigurator), Config.GetTracingConfig()), + NConsole::CreateJaegerTracingConfigurator(appData->TracingConfigurator, Config.GetTracingConfig()), TMailboxType::ReadAsFilled, appData->UserPoolId))); } diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp index cf4eafb7e395..078bfe155602 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.cpp +++ b/ydb/core/grpc_services/grpc_request_proxy.cpp @@ -6,10 +6,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -57,9 +57,8 @@ class TGRpcRequestProxyImpl { using TBase = TActorBootstrapped; public: - explicit TGRpcRequestProxyImpl(const NKikimrConfig::TAppConfig& appConfig, TIntrusivePtr tracingControl) + explicit TGRpcRequestProxyImpl(const NKikimrConfig::TAppConfig& appConfig) : ChannelBufferSize(appConfig.GetTableServiceConfig().GetResourceManager().GetChannelBufferSize()) - , TracingControl(std::move(tracingControl)) { } void Bootstrap(const TActorContext& ctx); @@ -326,7 +325,6 @@ class TGRpcRequestProxyImpl bool DynamicNode = false; TString RootDatabase; IGRpcProxyCounters::TPtr Counters; - TIntrusivePtr TracingControl; }; void TGRpcRequestProxyImpl::Bootstrap(const TActorContext& ctx) { @@ -451,7 +449,7 @@ void TGRpcRequestProxyImpl::MaybeStartTracing(IRequestProxyCtx& ctx) { if (const auto otelHeader = ctx.GetPeerMetaValues(NYdb::OTEL_TRACE_HEADER)) { traceId = NWilson::TTraceId::FromTraceparentHeader(otelHeader.GetRef(), TComponentTracingLevels::ProductionVerbose); } - TracingControl->HandleTracing(traceId, ctx.GetRequestDiscriminator()); + NJaegerTracing::HandleTracing(traceId, ctx.GetRequestDiscriminator()); if (traceId) { NWilson::TSpan grpcRequestProxySpan(TWilsonGrpc::RequestProxy, std::move(traceId), "GrpcRequestProxy"); if (auto database = ctx.GetDatabaseName()) { @@ -616,8 +614,8 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr& ev) { } } -IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig, TIntrusivePtr tracingControl) { - return new TGRpcRequestProxyImpl(appConfig, std::move(tracingControl)); +IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig) { + return new TGRpcRequestProxyImpl(appConfig); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index 4f9f1602a93b..a16bc4adb350 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -6,7 +6,6 @@ #include "grpc_request_proxy_handle_methods.h" #include -#include #include @@ -23,7 +22,7 @@ struct TAppData; namespace NGRpcService { -IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig, TIntrusivePtr tracingControl); +IActor* CreateGRpcRequestProxy(const NKikimrConfig::TAppConfig& appConfig); IActor* CreateGRpcRequestProxySimple(const NKikimrConfig::TAppConfig& appConfig); class TGRpcRequestProxy : public TGRpcRequestProxyHandleMethods, public IFacilityProvider { diff --git a/ydb/core/jaeger_tracing/sampling_throttling_configurator.cpp b/ydb/core/jaeger_tracing/sampling_throttling_configurator.cpp index 4a22d79bda29..a59370890012 100644 --- a/ydb/core/jaeger_tracing/sampling_throttling_configurator.cpp +++ b/ydb/core/jaeger_tracing/sampling_throttling_configurator.cpp @@ -15,7 +15,7 @@ template void PropagateUnspecifiedRequest(TRulesContainer& rules) { constexpr auto unspecifiedRequestType = static_cast(ERequestType::UNSPECIFIED); const auto& unspecifiedRequestTypeRules = rules[unspecifiedRequestType]; - + for (size_t requestType = 0; requestType < kRequestTypesCnt; ++requestType) { if (requestType == unspecifiedRequestType) { continue; @@ -45,7 +45,9 @@ TSamplingThrottlingConfigurator::TSamplingThrottlingConfigurator(TIntrusivePtr TSamplingThrottlingConfigurator::GetControl() { auto control = TIntrusivePtr(new TSamplingThrottlingControl(GenerateSetup())); - IssuedControls.push_back(control); + with_lock (ControlMutex) { + IssuedControls.push_back(control); + } return control; } @@ -55,8 +57,10 @@ void TSamplingThrottlingConfigurator::UpdateSettings(TSettingsUpdateImpl(GenerateSetup()); + with_lock (ControlMutex) { + for (auto& control : IssuedControls) { + control->UpdateImpl(GenerateSetup()); + } } } diff --git a/ydb/core/jaeger_tracing/sampling_throttling_configurator.h b/ydb/core/jaeger_tracing/sampling_throttling_configurator.h index 642c37e0d16f..68795482006f 100644 --- a/ydb/core/jaeger_tracing/sampling_throttling_configurator.h +++ b/ydb/core/jaeger_tracing/sampling_throttling_configurator.h @@ -11,7 +11,9 @@ #include #include +#include #include +#include namespace NKikimr::NJaegerTracing { @@ -22,7 +24,7 @@ struct TWithTag { size_t Tag; }; -class TSamplingThrottlingConfigurator: private TMoveOnly { +class TSamplingThrottlingConfigurator: public TRefCounted { public: TSamplingThrottlingConfigurator(TIntrusivePtr timeProvider, TIntrusivePtr& randomProvider); @@ -34,13 +36,14 @@ class TSamplingThrottlingConfigurator: private TMoveOnly { private: TSettings> GenerateThrottlers( TSettings> settings); - + std::unique_ptr GenerateSetup(); TVector> IssuedControls; TIntrusivePtr TimeProvider; TFastRng64 Rng; - TSettings> CurrentSettings; + TSettings> CurrentSettings; + TMutex ControlMutex; }; } // namespace NKikimr::NJaegerTracing diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index cf3bc1fa3230..61bc1b99315a 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -2899,6 +2899,14 @@ Y_UNIT_TEST_SUITE(THiveTest) { } Y_UNIT_TEST(TestReassignUseRelativeSpace) { + // TODO: Remove this code after issue https://github.com/ydb-platform/ydb/issues/12255 will be resolved + ui64 prevSeed = NActors::DefaultRandomSeed; + NActors::DefaultRandomSeed = 42; + Y_SCOPE_EXIT(prevSeed) { + NActors::DefaultRandomSeed = prevSeed; + }; + // TODO: Remove this code after issue https://github.com/ydb-platform/ydb/issues/12255 will be resolved + TTestBasicRuntime runtime(1, false); Setup(runtime, true, 5); const ui64 hiveTablet = MakeDefaultHiveID(); @@ -7181,7 +7189,7 @@ Y_UNIT_TEST_SUITE(TScaleRecommenderTest) { void AssertScaleRecommencation(TTestBasicRuntime& runtime, NKikimrProto::EReplyStatus expectedStatus, ui32 expectedNodes = 0) { const auto sender = runtime.AllocateEdgeActor(); - + const auto hiveId = MakeDefaultHiveID(); const TSubDomainKey subdomainKey(TTestTxConfig::SchemeShard, 1); runtime.SendToPipe(hiveId, sender, new TEvHive::TEvRequestScaleRecommendation(subdomainKey)); diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index 3127a0d4af06..fa9c522ca3ba 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -101,6 +102,9 @@ namespace NActors { SetRegistrationObserverFunc(&TTestActorRuntimeBase::DefaultRegistrationObserver); CleanupNodes(); + + App0 = nullptr; + NKikimr::NJaegerTracing::ClearTracingControl(); } void TTestActorRuntime::AddAppDataInit(std::function callback) { diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c8184ec6c44b..e9113ad903d5 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -364,17 +364,15 @@ namespace Tests { grpcRequestProxies.reserve(proxyCount); auto& appData = Runtime->GetAppData(grpcServiceNodeId); - NJaegerTracing::TSamplingThrottlingConfigurator tracingConfigurator(appData.TimeProvider, appData.RandomProvider); - for (size_t i = 0; i < proxyCount; ++i) { - auto grpcRequestProxy = NGRpcService::CreateGRpcRequestProxy(*Settings->AppConfig, tracingConfigurator.GetControl()); + auto grpcRequestProxy = NGRpcService::CreateGRpcRequestProxy(*Settings->AppConfig); auto grpcRequestProxyId = system->Register(grpcRequestProxy, TMailboxType::ReadAsFilled, appData.UserPoolId); system->RegisterLocalService(NGRpcService::CreateGRpcRequestProxyId(), grpcRequestProxyId); grpcRequestProxies.push_back(grpcRequestProxyId); } system->Register( - NConsole::CreateJaegerTracingConfigurator(std::move(tracingConfigurator), Settings->AppConfig->GetTracingConfig()), + NConsole::CreateJaegerTracingConfigurator(appData.TracingConfigurator, Settings->AppConfig->GetTracingConfig()), TMailboxType::ReadAsFilled, appData.UserPoolId );