Merge branch 'rightlib' into merge-libs-241229-0022

library/cpp/tld/tlds-alpha-by-domain.txt

@@ -1,4 +1,4 @@
-# Version 2024122501, Last Updated Wed Dec 25 07:07:01 2024 UTC
+# Version 2024122800, Last Updated Sat Dec 28 07:07:02 2024 UTC
 AAA
 AARP
 ABB

yql/essentials/minikql/codegen/codegen.cpp

@@ -297,7 +297,7 @@ class TCodegen : public ICodegen, private llvm::JITEventListener {
         // make coverity happy
 #if LLVM_VERSION_MAJOR == 12
         targetOptions.StackProtectorGuardOffset = 0;
-#endif        
+#endif
 
         std::string what;
         auto&& engineBuilder = llvm::EngineBuilder(std::move(module));
@@ -397,14 +397,11 @@ class TCodegen : public ICodegen, private llvm::JITEventListener {
 #if defined(_msan_enabled_)
         ReverseGlobalMapping_[(const void*)&__emutls_get_address] = "__emutls_get_address";
 #endif
-#if defined(_win_) && !defined(__clang__)
+#if defined(_win_)
+
         AddGlobalMapping("__security_check_cookie", (const void*)&__security_check_cookie);
         AddGlobalMapping("__security_cookie", (const void*)&__security_cookie);
-        AddGlobalMapping("__divti3", (const void*)&__divti3abi);
-        AddGlobalMapping("__modti3", (const void*)&__modti3abi);
-        AddGlobalMapping("__floattisf", (const void*)&__floattisfabi);
-        AddGlobalMapping("__floattidf", (const void*)&__floattidfabi);
-#else
+#endif
         AddGlobalMapping("__divti3", (const void*)&__divti3);
         AddGlobalMapping("__fixdfti", (const void*)&__fixdfti);
         AddGlobalMapping("__fixsfti", (const void*)&__fixsfti);
@@ -417,7 +414,6 @@ class TCodegen : public ICodegen, private llvm::JITEventListener {
         AddGlobalMapping("__muloti4", (const void*)&__muloti4);
         AddGlobalMapping("__udivti3", (const void*)&__udivti3);
         AddGlobalMapping("__umodti3", (const void*)&__umodti3);
-#endif
 
         for (auto& function : Module_->getFunctionList()) {
             function.addFnAttr("target-cpu", "x86-64");
@@ -725,7 +721,7 @@ class TCodegen : public ICodegen, private llvm::JITEventListener {
     std::string Diagnostic_;
     std::string Triple_;
     llvm::Module* Module_;
-#ifdef __linux__    
+#ifdef __linux__
     llvm::JITEventListener* PerfListener_ = nullptr;
 #endif
     std::unique_ptr<llvm::ExecutionEngine> Engine_;

yql/essentials/minikql/codegen/codegen_ut.cpp

@@ -194,8 +194,6 @@ Function *CreateUseExternalFromGeneratedFunction128(const ICodegen::TPtr& codege
     llvm::Argument* retArg = nullptr;
     if (codegen->GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) {
         retArg = &*args++;
-        retArg->addAttr(Attribute::StructRet);
-        retArg->addAttr(Attribute::NoAlias);
     }
 
     auto ArgX = args++;   // Get the arg 1.

yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp

@@ -364,7 +364,7 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa
 
             const TParentsMap* parents = getParents();
             if (IsOutputUsedMultipleTimes(path.Table().Cast<TYtOutput>().Ref(), *parents)) {
-                // Inner reduce output is used more than once
+                // Inner map output is used more than once
                 newPaths.push_back(path);
                 continue;
             }
@@ -452,7 +452,9 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduceWithTrivialMa
     }
 
     YQL_ENSURE(newInput.size() >= origReduceInputs);
-    YQL_ENSURE(newInput.size() - origReduceInputs <= 1);
+    // one section can be rewritten into 3:
+    // (ABA) -> (A), (C), (A)
+    YQL_ENSURE(newInput.size() - origReduceInputs <= 2);
 
     TExprNode::TPtr remapLambda = ctx.Builder(fusedMap->MapLambda.Pos())
         .Lambda()

yt/yt/core/concurrency/thread_affinity.h

@@ -27,7 +27,7 @@ namespace NYT::NConcurrency {
  *   \endcode
  * - Write
  *   \code
- *   VERIFY_THREAD_AFFINITY(Thread);
+ *   YT_ASSERT_THREAD_AFFINITY(Thread);
  *   \endcode
  *   at the beginning of each function in the group.
  *
@@ -56,51 +56,63 @@ class TThreadAffinitySlot
 #define DECLARE_THREAD_AFFINITY_SLOT(slot) \
     mutable ::NYT::NConcurrency::TThreadAffinitySlot PP_CONCAT(slot, _Slot)
 
-#define VERIFY_THREAD_AFFINITY(slot) \
+#define YT_ASSERT_THREAD_AFFINITY(slot) \
     PP_CONCAT(slot, _Slot).Check()
 
-#define VERIFY_SPINLOCK_AFFINITY(spinLock) \
+#define YT_ASSERT_SPINLOCK_AFFINITY(spinLock) \
     YT_VERIFY((spinLock).IsLocked());
 
-#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock) \
+#define YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock) \
     YT_VERIFY((spinLock).IsLockedByReader());
 
-#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock) \
+#define YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock) \
     YT_VERIFY((spinLock).IsLockedByWriter());
 
-#define VERIFY_INVOKER_AFFINITY(invoker) \
+#define YT_ASSERT_INVOKER_AFFINITY(invoker) \
     YT_VERIFY(::NYT::NConcurrency::VerifyInvokerAffinity(invoker))
 
-#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker) \
+#define YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker) \
     YT_VERIFY(::NYT::NConcurrency::VerifySerializedInvokerAffinity(invoker))
 
-#define VERIFY_INVOKERS_AFFINITY(...) \
+#define YT_ASSERT_INVOKERS_AFFINITY(...) \
     YT_VERIFY(::NYT::NConcurrency::VerifyInvokersAffinity(__VA_ARGS__))
 
-#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool) \
+#define YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool) \
     YT_VERIFY(::NYT::NConcurrency::VerifyInvokerPoolAffinity(invokerPool))
 
-#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot) \
+#define YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot) \
     PP_CONCAT(slot, _Slot).Check((invoker)->GetThreadId());
 
 #else
 
 // Expand macros to null but take care of the trailing semicolon.
 #define DECLARE_THREAD_AFFINITY_SLOT(slot)               struct PP_CONCAT(TNullThreadAffinitySlot_,  __LINE__) { }
-#define VERIFY_THREAD_AFFINITY(slot)                     do { } while (false)
-#define VERIFY_SPINLOCK_AFFINITY(spinLock)               do { } while (false)
-#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock)        do { } while (false)
-#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock)        do { } while (false)
-#define VERIFY_INVOKER_AFFINITY(invoker)                 do { } while (false)
-#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker)      do { } while (false)
-#define VERIFY_INVOKERS_AFFINITY(...)                    do { } while (false)
-#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool)        do { } while (false)
-#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot)    do { } while (false)
+#define YT_ASSERT_THREAD_AFFINITY(slot)                     do { } while (false)
+#define YT_ASSERT_SPINLOCK_AFFINITY(spinLock)               do { } while (false)
+#define YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock)        do { } while (false)
+#define YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock)        do { } while (false)
+#define YT_ASSERT_INVOKER_AFFINITY(invoker)                 do { } while (false)
+#define YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker)      do { } while (false)
+#define YT_ASSERT_INVOKERS_AFFINITY(...)                    do { } while (false)
+#define YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool)        do { } while (false)
+#define YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot)    do { } while (false)
 
 #endif
 
 //! This is a mere declaration and intentionally does not check anything.
-#define VERIFY_THREAD_AFFINITY_ANY() do { } while (false)
+#define YT_ASSERT_THREAD_AFFINITY_ANY() do { } while (false)
+
+// Temporary compatibility. Remove after removing all usages of VERIFY_*.
+#define VERIFY_THREAD_AFFINITY(slot) YT_ASSERT_THREAD_AFFINITY(slot)
+#define VERIFY_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_SPINLOCK_AFFINITY(spinLock)
+#define VERIFY_READER_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_READER_SPINLOCK_AFFINITY(spinLock)
+#define VERIFY_WRITER_SPINLOCK_AFFINITY(spinLock) YT_ASSERT_WRITER_SPINLOCK_AFFINITY(spinLock)
+#define VERIFY_INVOKER_AFFINITY(invoker) YT_ASSERT_INVOKER_AFFINITY(invoker)
+#define VERIFY_SERIALIZED_INVOKER_AFFINITY(invoker) YT_ASSERT_SERIALIZED_INVOKER_AFFINITY(invoker)
+#define VERIFY_INVOKERS_AFFINITY(...) YT_ASSERT_INVOKERS_AFFINITY(__VA_ARGS__)
+#define VERIFY_INVOKER_POOL_AFFINITY(invokerPool) YT_ASSERT_INVOKER_POOL_AFFINITY(invokerPool)
+#define VERIFY_INVOKER_THREAD_AFFINITY(invoker, slot) YT_ASSERT_INVOKER_THREAD_AFFINITY(invoker, slot)
+#define VERIFY_THREAD_AFFINITY_ANY() YT_ASSERT_THREAD_AFFINITY_ANY()
 
 ////////////////////////////////////////////////////////////////////////////////
 

yt/yt/core/net/config.cpp

@@ -40,8 +40,6 @@ void TAddressResolverConfig::Register(TRegistrar registrar)
         .Default();
     registrar.Parameter("resolve_hostname_into_fqdn", &TThis::ResolveHostNameIntoFqdn)
         .Default(true);
-    registrar.Parameter("expected_localhost_name", &TThis::ExpectedLocalHostName)
-        .Default();
 
     registrar.Preprocessor([] (TThis* config) {
         config->RefreshTime = TDuration::Seconds(60);

yt/yt/core/net/config.h

@@ -51,8 +51,6 @@ class TAddressResolverConfig
     //! If set, localhost name will be forcefully set to the given value rather
     //! than retrieved via |NYT::NNet::UpdateLocalHostName|.
     std::optional<TString> LocalHostNameOverride;
-    //! Used to check that bootstrap is being initialized from a correct container.
-    std::optional<TString> ExpectedLocalHostName;
 
     REGISTER_YSON_STRUCT(TAddressResolverConfig);
 

yt/yt/core/yson/protobuf_interop.cpp

@@ -1038,13 +1038,18 @@ class TProtobufTranscoderBase
         }
     }
 
-    void ValidateString(TStringBuf data, TStringBuf fieldFullName)
+    void ValidateString(TStringBuf data, TStringBuf fieldFullName, std::optional<EUtf8Check> utf8Check)
     {
-        auto config = GetProtobufInteropConfig();
-        if (config->Utf8Check == EUtf8Check::Disable || IsUtf(data)) {
+        // TODO(kmokrov): `.or_else` when C++23 arrives.
+        auto effectiveCheck = utf8Check ? *utf8Check : std::invoke([] {
+            auto config = GetProtobufInteropConfig();
+            return config->Utf8Check;
+        });
+
+        if (effectiveCheck == EUtf8Check::Disable || IsUtf(data)) {
             return;
         }
-        switch (config->Utf8Check) {
+        switch (effectiveCheck) {
             case EUtf8Check::Disable:
                 return;
             case EUtf8Check::LogOnFail:
@@ -1164,7 +1169,7 @@ class TProtobufWriter
             const auto* field = FieldStack_.back().Field;
             switch (field->GetType()) {
                 case FieldDescriptor::TYPE_STRING:
-                    ValidateString(value, field->GetFullName());
+                    ValidateString(value, field->GetFullName(), Options_.Utf8Check);
                     [[fallthrough]];
 
                 case FieldDescriptor::TYPE_BYTES:
@@ -2686,7 +2691,7 @@ class TProtobufParser
                         }
                         TStringBuf data(PooledString_.data(), length);
                         if (field->GetType() == FieldDescriptor::TYPE_STRING) {
-                            ValidateString(data, field->GetFullName());
+                            ValidateString(data, field->GetFullName(), Options_.Utf8Check);
                         }
                         ParseScalar([&] {
                             if (field->GetBytesFieldConverter()) {

yt/yt/core/yson/protobuf_interop_options.h

@@ -39,6 +39,9 @@ struct TProtobufWriterOptions
 
     //! Convert yson keys from snake case to camel case.
     bool ConvertSnakeToCamelCase = false;
+
+    //! Check if string field contains actual UTF-8 string. Overrides option from config if provided.
+    std::optional<EUtf8Check> Utf8Check;
 };
 
 struct TProtobufParserOptions
@@ -50,6 +53,9 @@ struct TProtobufParserOptions
     //! If |true| then required fields not found in protobuf metadata are
     //! silently skipped; otherwise an exception is thrown.
     bool SkipRequiredFields = false;
+
+    //! Check if string field contains actual UTF-8 string. Overrides option from config if provided.
+    std::optional<EUtf8Check> Utf8Check;
 };
 
 ////////////////////////////////////////////////////////////////////////////////

yt/yt/core/yson/unittests/protobuf_yson_ut.cpp

@@ -1023,35 +1023,43 @@ TEST(TYsonToProtobufTest, Entities)
 
 TEST(TYsonToProtobufTest, ValidUtf8StringCheck)
 {
-    for (auto option: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) {
-        auto config = New<TProtobufInteropConfig>();
-        config->Utf8Check = option;
-        SetProtobufInteropConfig(config);
-
-        TString invalidUtf8 = "\xc3\x28";
-
-        auto check = [&] {
-            TEST_PROLOGUE_WITH_OPTIONS(TMessage, {})
-                .BeginMap()
-                    .Item("string_field").Value(invalidUtf8)
-                .EndMap();
-        };
-        if (option == EUtf8Check::ThrowOnFail) {
-            EXPECT_THROW_WITH_SUBSTRING(check(), "Non UTF-8 value in string field");
-        } else {
-            EXPECT_NO_THROW(check());
-        }
-
-        NProto::TMessage message;
-        message.set_string_field(invalidUtf8);
-        TString newYsonString;
-        TStringOutput newYsonOutputStream(newYsonString);
-        TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty);
-        if (option == EUtf8Check::ThrowOnFail) {
-            EXPECT_THROW_WITH_SUBSTRING(
-                WriteProtobufMessage(&ysonWriter, message), "Non UTF-8 value in string field");
-        } else {
-            EXPECT_NO_THROW(WriteProtobufMessage(&ysonWriter, message));
+    for (auto configOption: {EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail}) {
+        for (auto option: std::vector<std::optional<EUtf8Check>>{
+            std::nullopt, EUtf8Check::Disable, EUtf8Check::LogOnFail, EUtf8Check::ThrowOnFail})
+        {
+            auto config = New<TProtobufInteropConfig>();
+            config->Utf8Check = configOption;
+            SetProtobufInteropConfig(config);
+            auto effectiveOption = option.value_or(configOption);
+
+            TString invalidUtf8 = "\xc3\x28";
+            auto checkWrite = [&] {
+                TEST_PROLOGUE_WITH_OPTIONS(TMessage, TProtobufWriterOptions{.Utf8Check = option})
+                    .BeginMap()
+                        .Item("string_field").Value(invalidUtf8)
+                    .EndMap();
+            };
+            if (effectiveOption == EUtf8Check::ThrowOnFail) {
+                EXPECT_THROW_WITH_SUBSTRING(checkWrite(), "Non UTF-8 value in string field");
+            } else {
+                EXPECT_NO_THROW(checkWrite());
+            }
+            NProto::TMessage message;
+            message.set_string_field(invalidUtf8);
+            TString newYsonString;
+            TStringOutput newYsonOutputStream(newYsonString);
+            TYsonWriter ysonWriter(&newYsonOutputStream, EYsonFormat::Pretty);
+            auto checkParse = [&] {
+                WriteProtobufMessage(
+                    &ysonWriter,
+                    message,
+                    TProtobufParserOptions{.Utf8Check = option});
+            };
+            if (effectiveOption == EUtf8Check::ThrowOnFail) {
+                EXPECT_THROW_WITH_SUBSTRING(checkParse(), "Non UTF-8 value in string field");
+            } else {
+                EXPECT_NO_THROW(checkParse());
+            }
         }
     }
 }

yt/yt/library/profiling/solomon/exporter.h

@@ -146,6 +146,7 @@ class TSolomonExporter
 };
 
 DEFINE_REFCOUNTED_TYPE(TSolomonExporter)
+YT_DEFINE_TYPEID(TSolomonExporter)
 
 ////////////////////////////////////////////////////////////////////////////////
 

yt/yt/library/profiling/solomon/public.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include <library/cpp/yt/memory/ref_counted.h>
+
 #include <library/cpp/yt/misc/enum.h>
+#include <library/cpp/yt/misc/typeid.h>
 
 namespace NYT::NProfiling {
 
@@ -12,6 +14,7 @@ DECLARE_REFCOUNTED_STRUCT(TSolomonExporterConfig)
 DECLARE_REFCOUNTED_STRUCT(TSolomonProxyConfig)
 
 DECLARE_REFCOUNTED_CLASS(TSolomonExporter)
+YT_DECLARE_TYPEID(TSolomonExporter)
 DECLARE_REFCOUNTED_CLASS(TSolomonRegistry)
 DECLARE_REFCOUNTED_CLASS(TSolomonProxy)