diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index f8193f33e098..4f2b537cb301 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024122501, Last Updated Wed Dec 25 07:07:01 2024 UTC +# Version 2024122800, Last Updated Sat Dec 28 07:07:02 2024 UTC AAA AARP ABB diff --git a/yql/essentials/minikql/codegen/codegen.cpp b/yql/essentials/minikql/codegen/codegen.cpp index a245702f0589..767bef25c9a2 100644 --- a/yql/essentials/minikql/codegen/codegen.cpp +++ b/yql/essentials/minikql/codegen/codegen.cpp @@ -297,7 +297,7 @@ class TCodegen : public ICodegen, private llvm::JITEventListener { // make coverity happy #if LLVM_VERSION_MAJOR == 12 targetOptions.StackProtectorGuardOffset = 0; -#endif +#endif std::string what; auto&& engineBuilder = llvm::EngineBuilder(std::move(module)); @@ -397,14 +397,11 @@ class TCodegen : public ICodegen, private llvm::JITEventListener { #if defined(_msan_enabled_) ReverseGlobalMapping_[(const void*)&__emutls_get_address] = "__emutls_get_address"; #endif -#if defined(_win_) && !defined(__clang__) +#if defined(_win_) + AddGlobalMapping("__security_check_cookie", (const void*)&__security_check_cookie); AddGlobalMapping("__security_cookie", (const void*)&__security_cookie); - AddGlobalMapping("__divti3", (const void*)&__divti3abi); - AddGlobalMapping("__modti3", (const void*)&__modti3abi); - AddGlobalMapping("__floattisf", (const void*)&__floattisfabi); - AddGlobalMapping("__floattidf", (const void*)&__floattidfabi); -#else +#endif AddGlobalMapping("__divti3", (const void*)&__divti3); AddGlobalMapping("__fixdfti", (const void*)&__fixdfti); AddGlobalMapping("__fixsfti", (const void*)&__fixsfti); @@ -417,7 +414,6 @@ class TCodegen : public ICodegen, private llvm::JITEventListener { AddGlobalMapping("__muloti4", (const void*)&__muloti4); AddGlobalMapping("__udivti3", (const void*)&__udivti3); AddGlobalMapping("__umodti3", (const void*)&__umodti3); -#endif for (auto& function : Module_->getFunctionList()) { function.addFnAttr("target-cpu", "x86-64"); @@ -725,7 +721,7 @@ class TCodegen : public ICodegen, private llvm::JITEventListener { std::string Diagnostic_; std::string Triple_; llvm::Module* Module_; -#ifdef __linux__ +#ifdef __linux__ llvm::JITEventListener* PerfListener_ = nullptr; #endif std::unique_ptr Engine_; diff --git a/yql/essentials/minikql/codegen/codegen_ut.cpp b/yql/essentials/minikql/codegen/codegen_ut.cpp index f0443f1a8974..9ac6030358db 100644 --- a/yql/essentials/minikql/codegen/codegen_ut.cpp +++ b/yql/essentials/minikql/codegen/codegen_ut.cpp @@ -194,8 +194,6 @@ Function *CreateUseExternalFromGeneratedFunction128(const ICodegen::TPtr& codege llvm::Argument* retArg = nullptr; if (codegen->GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) { retArg = &*args++; - retArg->addAttr(Attribute::StructRet); - retArg->addAttr(Attribute::NoAlias); } auto ArgX = args++; // Get the arg 1. diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp index f0b6d8bf3d5a..31c685f4438f 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp @@ -364,7 +364,7 @@ TMaybeNode TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa const TParentsMap* parents = getParents(); if (IsOutputUsedMultipleTimes(path.Table().Cast().Ref(), *parents)) { - // Inner reduce output is used more than once + // Inner map output is used more than once newPaths.push_back(path); continue; } @@ -452,7 +452,9 @@ TMaybeNode TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa } YQL_ENSURE(newInput.size() >= origReduceInputs); - YQL_ENSURE(newInput.size() - origReduceInputs <= 1); + // one section can be rewritten into 3: + // (ABA) -> (A), (C), (A) + YQL_ENSURE(newInput.size() - origReduceInputs <= 2); TExprNode::TPtr remapLambda = ctx.Builder(fusedMap->MapLambda.Pos()) .Lambda() diff --git a/yt/yt/core/concurrency/thread_affinity.h b/yt/yt/core/concurrency/thread_affinity.h index c6c465f35e3c..9fa7049063ff 100644 --- a/yt/yt/core/concurrency/thread_affinity.h +++ b/yt/yt/core/concurrency/thread_affinity.h @@ -27,7 +27,7 @@ namespace NYT::NConcurrency { * \endcode * - Write * \code - * VERIFY_THREAD_AFFINITY(Thread); + * YT_ASSERT_THREAD_AFFINITY(Thread); * \endcode * at the beginning of each function in the group. * @@ -56,51 +56,63 @@ class TThreadAffinitySlot #define DECLARE_THREAD_AFFINITY_SLOT(slot) \ mutable ::NYT::NConcurrency::TThreadAffinitySlot PP_CONCAT(slot, _Slot) -#define VERIFY_THREAD_AFFINITY(slot) \ +#define YT_ASSERT_THREAD_AFFINITY(slot) \ PP_CONCAT(slot, _Slot).Check() -#define VERIFY_SPINLOCK_AFFINITY(spinLock) \ +#define YT_ASSERT_SPINLOCK_AFFINITY(spinLock) \ YT_VERIFY((spinLock).IsLocked()); -#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock) \ +#define YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock) \ YT_VERIFY((spinLock).IsLockedByReader()); -#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock) \ +#define YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock) \ YT_VERIFY((spinLock).IsLockedByWriter()); -#define VERIFY_INVOKER_AFFINITY(invoker) \ +#define YT_ASSERT_INVOKER_AFFINITY(invoker) \ YT_VERIFY(::NYT::NConcurrency::VerifyInvokerAffinity(invoker)) -#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker) \ +#define YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker) \ YT_VERIFY(::NYT::NConcurrency::VerifySerializedInvokerAffinity(invoker)) -#define VERIFY_INVOKERS_AFFINITY(...) \ +#define YT_ASSERT_INVOKERS_AFFINITY(...) \ YT_VERIFY(::NYT::NConcurrency::VerifyInvokersAffinity(__VA_ARGS__)) -#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool) \ +#define YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool) \ YT_VERIFY(::NYT::NConcurrency::VerifyInvokerPoolAffinity(invokerPool)) -#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot) \ +#define YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot) \ PP_CONCAT(slot, _Slot).Check((invoker)->GetThreadId()); #else // Expand macros to null but take care of the trailing semicolon. #define DECLARE_THREAD_AFFINITY_SLOT(slot) struct PP_CONCAT(TNullThreadAffinitySlot_, __LINE__) { } -#define VERIFY_THREAD_AFFINITY(slot) do { } while (false) -#define VERIFY_SPINLOCK_AFFINITY(spinLock) do { } while (false) -#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock) do { } while (false) -#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock) do { } while (false) -#define VERIFY_INVOKER_AFFINITY(invoker) do { } while (false) -#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker) do { } while (false) -#define VERIFY_INVOKERS_AFFINITY(...) do { } while (false) -#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool) do { } while (false) -#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot) do { } while (false) +#define YT_ASSERT_THREAD_AFFINITY(slot) do { } while (false) +#define YT_ASSERT_SPINLOCK_AFFINITY(spinLock) do { } while (false) +#define YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock) do { } while (false) +#define YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock) do { } while (false) +#define YT_ASSERT_INVOKER_AFFINITY(invoker) do { } while (false) +#define YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker) do { } while (false) +#define YT_ASSERT_INVOKERS_AFFINITY(...) do { } while (false) +#define YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool) do { } while (false) +#define YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot) do { } while (false) #endif //! This is a mere declaration and intentionally does not check anything. -#define VERIFY_THREAD_AFFINITY_ANY() do { } while (false) +#define YT_ASSERT_THREAD_AFFINITY_ANY() do { } while (false) + +// Temporary compatibility. Remove after removing all usages of VERIFY_*. +#define VERIFY_THREAD_AFFINITY(slot) YT_ASSERT_THREAD_AFFINITY(slot) +#define VERIFY_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_SPINLOCK_AFFINITY(spinLock) +#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock) +#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock) +#define VERIFY_INVOKER_AFFINITY(invoker) YT_ASSERT_INVOKER_AFFINITY(invoker) +#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker) YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker) +#define VERIFY_INVOKERS_AFFINITY(...) YT_ASSERT_INVOKERS_AFFINITY(__VA_ARGS__) +#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool) YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool) +#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot) YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot) +#define VERIFY_THREAD_AFFINITY_ANY() YT_ASSERT_THREAD_AFFINITY_ANY() //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/net/config.cpp b/yt/yt/core/net/config.cpp index b6ade7d35558..3c9dd520d328 100644 --- a/yt/yt/core/net/config.cpp +++ b/yt/yt/core/net/config.cpp @@ -40,8 +40,6 @@ void TAddressResolverConfig::Register(TRegistrar registrar) .Default(); registrar.Parameter("resolve_hostname_into_fqdn", &TThis::ResolveHostNameIntoFqdn) .Default(true); - registrar.Parameter("expected_localhost_name", &TThis::ExpectedLocalHostName) - .Default(); registrar.Preprocessor([] (TThis* config) { config->RefreshTime = TDuration::Seconds(60); diff --git a/yt/yt/core/net/config.h b/yt/yt/core/net/config.h index 413748d35c69..15cc198196f0 100644 --- a/yt/yt/core/net/config.h +++ b/yt/yt/core/net/config.h @@ -51,8 +51,6 @@ class TAddressResolverConfig //! If set, localhost name will be forcefully set to the given value rather //! than retrieved via |NYT::NNet::UpdateLocalHostName|. std::optional LocalHostNameOverride; - //! Used to check that bootstrap is being initialized from a correct container. - std::optional ExpectedLocalHostName; REGISTER_YSON_STRUCT(TAddressResolverConfig); diff --git a/yt/yt/core/yson/protobuf_interop.cpp b/yt/yt/core/yson/protobuf_interop.cpp index 72a1a2300a0d..2a5c1c3e0575 100644 --- a/yt/yt/core/yson/protobuf_interop.cpp +++ b/yt/yt/core/yson/protobuf_interop.cpp @@ -1038,13 +1038,18 @@ class TProtobufTranscoderBase } } - void ValidateString(TStringBuf data, TStringBuf fieldFullName) + void ValidateString(TStringBuf data, TStringBuf fieldFullName, std::optional utf8Check) { - auto config = GetProtobufInteropConfig(); - if (config->Utf8Check == EUtf8Check::Disable || IsUtf(data)) { + // TODO(kmokrov): `.or_else` when C++23 arrives. + auto effectiveCheck = utf8Check ? *utf8Check : std::invoke([] { + auto config = GetProtobufInteropConfig(); + return config->Utf8Check; + }); + + if (effectiveCheck == EUtf8Check::Disable || IsUtf(data)) { return; } - switch (config->Utf8Check) { + switch (effectiveCheck) { case EUtf8Check::Disable: return; case EUtf8Check::LogOnFail: @@ -1164,7 +1169,7 @@ class TProtobufWriter const auto* field = FieldStack_.back().Field; switch (field->GetType()) { case FieldDescriptor::TYPE_STRING: - ValidateString(value, field->GetFullName()); + ValidateString(value, field->GetFullName(), Options_.Utf8Check); [[fallthrough]]; case FieldDescriptor::TYPE_BYTES: @@ -2686,7 +2691,7 @@ class TProtobufParser } TStringBuf data(PooledString_.data(), length); if (field->GetType() == FieldDescriptor::TYPE_STRING) { - ValidateString(data, field->GetFullName()); + ValidateString(data, field->GetFullName(), Options_.Utf8Check); } ParseScalar([&] { if (field->GetBytesFieldConverter()) { diff --git a/yt/yt/core/yson/protobuf_interop_options.h b/yt/yt/core/yson/protobuf_interop_options.h index fbf9833e7948..3ac306a7c8a4 100644 --- a/yt/yt/core/yson/protobuf_interop_options.h +++ b/yt/yt/core/yson/protobuf_interop_options.h @@ -39,6 +39,9 @@ struct TProtobufWriterOptions //! Convert yson keys from snake case to camel case. bool ConvertSnakeToCamelCase = false; + + //! Check if string field contains actual UTF-8 string. Overrides option from config if provided. + std::optional Utf8Check; }; struct TProtobufParserOptions @@ -50,6 +53,9 @@ struct TProtobufParserOptions //! If |true| then required fields not found in protobuf metadata are //! silently skipped; otherwise an exception is thrown. bool SkipRequiredFields = false; + + //! Check if string field contains actual UTF-8 string. Overrides option from config if provided. + std::optional Utf8Check; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp index 451e978a61f7..214d748ace78 100644 --- a/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp +++ b/yt/yt/core/yson/unittests/protobuf_yson_ut.cpp @@ -1023,35 +1023,43 @@ TEST(TYsonToProtobufTest, Entities) TEST(TYsonToProtobufTest, ValidUtf8StringCheck) { - for (auto option: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { - auto config = New(); - config->Utf8Check = option; - SetProtobufInteropConfig(config); - - TString invalidUtf8 = "\xc3\x28"; - - auto check = [&] { - TEST_PROLOGUE_WITH_OPTIONS(TMessage, {}) - .BeginMap() - .Item("string_field").Value(invalidUtf8) - .EndMap(); - }; - if (option == EUtf8Check::ThrowOnFail) { - EXPECT_THROW_WITH_SUBSTRING(check(), "Non UTF-8 value in string field"); - } else { - EXPECT_NO_THROW(check()); - } - - NProto::TMessage message; - message.set_string_field(invalidUtf8); - TString newYsonString; - TStringOutput newYsonOutputStream(newYsonString); - TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty); - if (option == EUtf8Check::ThrowOnFail) { - EXPECT_THROW_WITH_SUBSTRING( - WriteProtobufMessage(&ysonWriter, message), "Non UTF-8 value in string field"); - } else { - EXPECT_NO_THROW(WriteProtobufMessage(&ysonWriter, message)); + for (auto configOption: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) { + for (auto option: std::vector>{ + std::nullopt, EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) + { + auto config = New(); + config->Utf8Check = configOption; + SetProtobufInteropConfig(config); + auto effectiveOption = option.value_or(configOption); + + TString invalidUtf8 = "\xc3\x28"; + auto checkWrite = [&] { + TEST_PROLOGUE_WITH_OPTIONS(TMessage, TProtobufWriterOptions{.Utf8Check = option}) + .BeginMap() + .Item("string_field").Value(invalidUtf8) + .EndMap(); + }; + if (effectiveOption == EUtf8Check::ThrowOnFail) { + EXPECT_THROW_WITH_SUBSTRING(checkWrite(), "Non UTF-8 value in string field"); + } else { + EXPECT_NO_THROW(checkWrite()); + } + NProto::TMessage message; + message.set_string_field(invalidUtf8); + TString newYsonString; + TStringOutput newYsonOutputStream(newYsonString); + TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty); + auto checkParse = [&] { + WriteProtobufMessage( + &ysonWriter, + message, + TProtobufParserOptions{.Utf8Check = option}); + }; + if (effectiveOption == EUtf8Check::ThrowOnFail) { + EXPECT_THROW_WITH_SUBSTRING(checkParse(), "Non UTF-8 value in string field"); + } else { + EXPECT_NO_THROW(checkParse()); + } } } } diff --git a/yt/yt/library/profiling/solomon/exporter.h b/yt/yt/library/profiling/solomon/exporter.h index d8bee70b3ba9..077339c0e2d2 100644 --- a/yt/yt/library/profiling/solomon/exporter.h +++ b/yt/yt/library/profiling/solomon/exporter.h @@ -146,6 +146,7 @@ class TSolomonExporter }; DEFINE_REFCOUNTED_TYPE(TSolomonExporter) +YT_DEFINE_TYPEID(TSolomonExporter) //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/profiling/solomon/public.h b/yt/yt/library/profiling/solomon/public.h index a59e20b28efe..2f4f224da9b1 100644 --- a/yt/yt/library/profiling/solomon/public.h +++ b/yt/yt/library/profiling/solomon/public.h @@ -1,7 +1,9 @@ #pragma once #include + #include +#include namespace NYT::NProfiling { @@ -12,6 +14,7 @@ DECLARE_REFCOUNTED_STRUCT(TSolomonExporterConfig) DECLARE_REFCOUNTED_STRUCT(TSolomonProxyConfig) DECLARE_REFCOUNTED_CLASS(TSolomonExporter) +YT_DECLARE_TYPEID(TSolomonExporter) DECLARE_REFCOUNTED_CLASS(TSolomonRegistry) DECLARE_REFCOUNTED_CLASS(TSolomonProxy)