From e13495ec6e49b86b8615618ee90005012089c096 Mon Sep 17 00:00:00 2001 From: sebaszm <45654185+sebaszm@users.noreply.github.com> Date: Sun, 22 Sep 2024 11:17:24 +0200 Subject: [PATCH 01/18] [Core] Improve path normalization (#1758) Co-authored-by: Pierre Wielders --- Source/Thunder/Controller.cpp | 11 +- Source/core/FileSystem.cpp | 144 +++++++++++------- Source/core/FileSystem.h | 9 +- .../LXCImplementation/LXCImplementation.cpp | 5 +- Source/plugins/Service.cpp | 7 +- Tests/unit/core/test_filesystem.cpp | 133 ++++++++++++++++ 6 files changed, 237 insertions(+), 72 deletions(-) diff --git a/Source/Thunder/Controller.cpp b/Source/Thunder/Controller.cpp index 5664d6e10..e41ba97d7 100644 --- a/Source/Thunder/Controller.cpp +++ b/Source/Thunder/Controller.cpp @@ -239,12 +239,12 @@ namespace Plugin { Core::hresult Controller::Delete(const string& path) { Core::hresult result = Core::ERROR_UNKNOWN_KEY; - bool valid; - string normalized_path = Core::File::Normalize(path, valid); ASSERT(_service != nullptr); - if (valid == false) { + const string normalized_path = Core::File::Normalize(path, true /* safe paths only */); + + if (normalized_path.empty() == true) { result = Core::ERROR_PRIVILIGED_REQUEST; } else { @@ -681,10 +681,9 @@ namespace Plugin { remainder = index.Remainder().Text(); } - bool valid; - string normalized(Core::File::Normalize(remainder, valid)); + const string normalized(Core::File::Normalize(remainder, true /* safe paths only */)); - if (valid == false) { + if (normalized.empty() == true) { result->Message = "incorrect path"; result->ErrorCode = Web::STATUS_BAD_REQUEST; } diff --git a/Source/core/FileSystem.cpp b/Source/core/FileSystem.cpp index e233ffbc0..cdb9f3705 100644 --- a/Source/core/FileSystem.cpp +++ b/Source/core/FileSystem.cpp @@ -72,17 +72,15 @@ namespace Core { Close(); } - /* static */ string File::Normalize(const string& location, bool& valid) + /* static */ string File::Normalize(const string& location, const bool inScopeOnly, const bool allowDirs) { string result(location); - valid = true; - // First see if we are not empy. if (result.empty() == false) { uint32_t index = 0; - while (index < result.length()) { + while (index < result.length()) { #ifdef __WINDOWS__ if (result[index] == '\\') { @@ -90,26 +88,37 @@ namespace Core { } #endif - if ((result[index] == '/') && (index >= 1) ) { + if ((result[index] == '/') && (index >= 1)) { if (result[index - 1] == '/') { - if (index >= 2) { - // We have a double slash, clear all till the beginning - result = result.substr(index - 1); - index = 1; - } + // We have a double slash, trim it + result.erase(index, 1); + index--; } else if (result[index - 1] == '.') { if ((index == 1) || (result[index - 2] == '/')) { - // It is a dot, remove it... - uint32_t offset = (index == 1 ? 0 : index - 2); - result.erase(offset, 2); - index = offset; + if (result.length() != 2) { + // It is a dot, remove it... but not if it's just ./ + result.erase(index - (index == 1 ? 1 : 2), 2); + index -= 2; + } } else if ((result[index - 2] == '.') && ((index == 2) || (result[index - 3] == '/'))) { - if (index <= 3) { - valid = false; + if (index == 2) { + // We're going out of scope! + if (inScopeOnly == true) { + result.clear(); + } + } + else if (index == 3) { + // Going up past / thus invalid result.clear(); } +#ifdef __WINDOWS__ + else if ((index == 5) && (result[1] == ':')) { + // Going up past X:/ thus invalid + result.clear(); + } +#endif else { // Seems like we are moving up a directory... execute that on the result... if we can... // there is data we can drop, drop it, drop it till the '/' is found @@ -117,50 +126,79 @@ namespace Core { while ((offset > 0) && (result[offset] != '/')) { offset--; } - result.erase(offset, index - offset); - index = offset; + + // ...but don't swallow another .. + if ((result[offset] != '.') && (result[offset + 1] != '.')) { + result.erase(offset + (result[offset] == '/' ? 1 : 0), index - offset + (result[offset] == '/' ? 0 : -1)); + index = offset; + } } } } } index++; - } - - // It could be that the last slash is not part of the full line, check the last part, assuming there is such a slash, - // normalization rules applyt than as well.... - if ((result.length() >= 1) && (result[result.length() - 1] == '.')) { - - if (result.length() == 1) { - // We have only a dot... - result.clear(); - } - else if ( (result.length() == 2) && (result[0] == '.') ) { - // We have a ".." and nothing more - valid = false; - result.clear(); - } - else if ((result.length() >= 2) && (result[result.length() - 2] == '/')) { - result = result.substr(0, result.length() - 2); - } - else if ((result.length() >= 3) && (result[result.length() - 2] == '.') && (result[result.length() - 3] == '/')) { - // How about ThisFile/.., it is valid, but /.. would not be, both end up at an empty string... but the difference - // is the fact that the first, had a length > 3 and the second was exactly 3, so a length of 3 is invalid and empty.. - if (result.length() == 3) { - valid = false; - result.clear(); - } - else { - // there is data we can drop, drop it, drop it till the '/' is found - uint32_t offset = static_cast(result.length() - 4); - - while ((offset > 0) && (result[offset] != '/')) { - offset--; + } + + // It could be that the last slash is not part of the full line, check the last part, assuming there is such a slash, + // normalization rules apply than as well.... + if (result.length() >= 1) { + if (result[result.length() - 1] == '/') { + if (allowDirs == false) { + result.clear(); + } + else if ((result.length() == 1) && (result[0] == '/') && (location[0] == '.')) { + // Correct corner case where .//// ends up as / + result = "./"; + } + } + else if (result[result.length() - 1] == '.') { + if (allowDirs == false) { + result.clear(); + } + else if ((result.length() >= 2) && (result[result.length() - 2] == '/')) { + // Ends with /. + result.erase(result.length() - 1, 1); + } + else if ((result.length() >= 3) && (result[result.length() - 2] == '.') && (result[result.length() - 3] == '/')) { + // How about ThisFile/.., it is valid, but /.. would not be, both end up at an empty string... but the difference + // is the fact that the first, had a length > 3 and the second was exactly 3, so a length of 3 is invalid and empty.. + if (result.length() == 3) { + // Invalid /.. + result.clear(); + } +#ifdef __WINDOWS__ + else if ((result.length() == 5) && (result[1] == ':')) { + // Invalid X:/.. + result.clear(); + } +#endif + else { + // there is data we can drop, drop it, drop it till the '/' is found + uint32_t offset = static_cast(result.length() - 4); + + while ((offset > 0) && (result[offset] != '/')) { + offset--; + } + + // again do not swallow another .. + if ((result[offset] != '.') && (result[offset + 1] != '.')) { + result.erase(offset + (result[offset] == '/' ? 1 : 0)); + + if ((result.empty() == true) && (allowDirs == true)) { + // Collapsed everything, so insert current dir + result = "./"; + } + } } - result = result.substr(0, offset); - } - } - } + } + else { + // We have . or .. + result += '/'; + } + } + } } + return (result); } diff --git a/Source/core/FileSystem.h b/Source/core/FileSystem.h index a3d56dbf2..3051d41f5 100644 --- a/Source/core/FileSystem.h +++ b/Source/core/FileSystem.h @@ -199,7 +199,7 @@ namespace Core { return (*this); } - static string Normalize(const string& input, bool& valid); + static string Normalize(const string& input, const bool inScopeOnly = false, const bool allowDirs = false); public: inline static string FileName(const string& name) @@ -666,17 +666,16 @@ POP_WARNING() ~Directory(); public: - static string Normalize(const string& location) + static string Normalize(const string& location, const bool inScopeOnly = false) { string result; // First see if we are not empy. if (location.empty() == false) { - bool valid; - result = File::Normalize(location, valid); + result = File::Normalize(location, inScopeOnly, true /* accept directories */); - if ((valid == true) && ((result.empty() == true) || (result[result.length() - 1] != '/'))) { + if ((result.empty() == false) && (result[result.length() - 1] != '/')) { result += '/'; } } diff --git a/Source/extensions/processcontainers/implementations/LXCImplementation/LXCImplementation.cpp b/Source/extensions/processcontainers/implementations/LXCImplementation/LXCImplementation.cpp index 2e43d8311..c5d47fb3a 100644 --- a/Source/extensions/processcontainers/implementations/LXCImplementation/LXCImplementation.cpp +++ b/Source/extensions/processcontainers/implementations/LXCImplementation/LXCImplementation.cpp @@ -568,10 +568,7 @@ namespace ProcessContainers { Core::Directory logDir(dirname.c_str()); logDir.CreatePath(); - bool valid{}; - const string filename(Core::File::Normalize(dirname + logFileName, valid)); - - ASSERT(valid == true); + const string filename(dirname + logFileName); const std::string file = Core::ToString(filename); const std::string level = Core::ToString(loggingOptions); diff --git a/Source/plugins/Service.cpp b/Source/plugins/Service.cpp index 2ddb8b1f3..0e5361162 100644 --- a/Source/plugins/Service.cpp +++ b/Source/plugins/Service.cpp @@ -76,7 +76,7 @@ namespace PluginHost { _notifierLock.Unlock(); } - void Service::FileToServe(const string& webServiceRequest, Web::Response& response, bool allowUnsafePath) + void Service::FileToServe(const string& webServiceRequest, Web::Response& response, const bool allowUnsafePath) { Web::MIMETypes result; Web::EncodingTypes encoding = Web::ENCODING_UNKNOWN; @@ -91,10 +91,9 @@ namespace PluginHost { response.Body(fileBody); } else { ASSERT(fileToService.length() >= _webServerFilePath.length()); - bool safePath = true; - string normalizedPath = Core::File::Normalize(fileToService.substr(_webServerFilePath.length()), safePath); + string normalizedPath = Core::File::Normalize(fileToService.substr(_webServerFilePath.length()), !allowUnsafePath); - if (allowUnsafePath || safePath ) { + if (normalizedPath.empty() == false) { Core::ProxyType fileBody(IFactories::Instance().FileBody()); *fileBody = fileToService; response.ContentType = result; diff --git a/Tests/unit/core/test_filesystem.cpp b/Tests/unit/core/test_filesystem.cpp index ce68d8f0e..2634ec306 100644 --- a/Tests/unit/core/test_filesystem.cpp +++ b/Tests/unit/core/test_filesystem.cpp @@ -120,6 +120,139 @@ namespace Core { system("rm -rf home"); } + TEST (test_file, directory_normalize_path) + { + EXPECT_EQ(::Thunder::Core::Directory::Normalize(""), ""); + + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/"), "/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/."), "/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/././././"), "/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("////"), "/"); + + EXPECT_EQ(::Thunder::Core::Directory::Normalize("."), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./"), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("././././././././"), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./././././././."), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize(".////"), "./"); + + EXPECT_EQ(::Thunder::Core::Directory::Normalize(".."), "../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("../"), "../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("../../.."), "../../../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("../../../"), "../../../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./../"), "../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("././../"), "../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("././../.."), "../../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("..///"), "../"); + + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/bar"), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/bar/."), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/.."), "foo/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/bar/./"), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/../"), "foo/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/bar/."), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/.////"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/././././bar"), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/quux/../../xyzzy"), "foo/xyzzy/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("../foo/bar/."), "../foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("../foo/bar/.."), "../foo/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/bar/.") , "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo////bar////"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("///foo////bar////"), "/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("././././foo/bar"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./foo/bar"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo////bar////././././"), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo////bar////././././."), "foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo////bar////././././."), "/foo/bar/"); + + // Cases where navigating upwards compacts completely + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/../.."), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("foo/bar/../.."), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./foo/../bar/.."), "./"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("./foo/../bar/../.."), "../"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/../bar/.."), "/"); + + // Negative cases navigating past root + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/.."), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/../.."), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/./.."), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/../bar/../.."), ""); + +#ifdef __WINDOWS__ + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\\\bar\\.\\.\\quux\\..\\."), "C:/foo/bar/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\..\\.."), "C:/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\.."), "C:/foo"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\..\\"), "C:/foo"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\."), "C:/foo/bar"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\.\\"), "C:/foo/bar"); + + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\.\\foo\\bar\\..\\.."), "C:/"); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\foo\\bar\\..\\..\\.."), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\.."), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\\\"), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("C:\\..\\.."), ""); +#endif + } + + TEST (test_file, file_normalize_path) + { + EXPECT_EQ(::Thunder::Core::File::Normalize("./foo"), "foo"); + EXPECT_EQ(::Thunder::Core::File::Normalize("./../foo"), "../foo"); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo///bar"), "foo/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo///bar"), "/foo/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/../bar"), "bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/../bar"), "/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/.././././bar"), "/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/../../../.././././././bar"), "../../../bar"); + + // Negative test cases, all fail because they point to a directory + EXPECT_EQ(::Thunder::Core::File::Normalize("/"), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize(".."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/././././././"), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/.."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/.."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/."), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/bar/"), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/bar///../.."), ""); + } + + TEST (test_file, file_safe_normalize_path) + { + EXPECT_EQ(::Thunder::Core::File::Normalize("./foo", true), "foo"); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo///bar", true), "foo/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo///bar", true), "/foo/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/../bar", true), "bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/../bar", true), "/bar"); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/.././././bar", true), "/bar"); + + // Negative test cases, all fail because they point to a directory + EXPECT_EQ(::Thunder::Core::File::Normalize("/", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize(".", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("..", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/././././././", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/..", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/.", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/..", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/.", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("/foo/bar/", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/bar///../..", true), ""); + + // Negative test cases, all fail because they point past current dir + EXPECT_EQ(::Thunder::Core::File::Normalize("./../foo", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("../foo", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("../../foo", true), ""); + EXPECT_EQ(::Thunder::Core::File::Normalize("foo/../../bar", true), ""); + + // Negative test cases, all fail because they point past root + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/../foo", true), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/../../foo", true), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/./../foo", true), ""); + EXPECT_EQ(::Thunder::Core::Directory::Normalize("/foo/../bar/../../quux", true), ""); + } + + } // Core } // Tests } // Thunder From 6beb27eb334c1f5603d4a58c268fac872965a86e Mon Sep 17 00:00:00 2001 From: Bram Oosterhuis Date: Mon, 23 Sep 2024 08:46:43 +0200 Subject: [PATCH 02/18] Development/refine bluez headers handling (#1755) * CMake: add script to handle bluez headers * core: make the bluez headers part of core headers * Bluetooth: move mgmt.h include to the specific file that it needs * bluetooth: header include clean up * bluetooth: remove Bluez5UtilHeaders dependencies * bluetooth: Remove Bluez5UtilHeaders find module * cmake: GetBluezHeaders return multiple include paths * cmake: GetBluezHeaders fix typo in error message * Core: already add a bluez specific define * Bluetooth: rename bluez to bluez5 --------- Co-authored-by: Pierre Wielders --- Source/core/CMakeLists.txt | 23 +++- Source/core/NodeId.h | 8 +- Source/extensions/bluetooth/CMakeLists.txt | 3 - Source/extensions/bluetooth/HCISocket.h | 4 + Source/extensions/bluetooth/Module.h | 5 - .../extensions/bluetooth/audio/CMakeLists.txt | 2 - Source/extensions/bluetooth/audio/Module.h | 3 - .../extensions/bluetooth/gatt/CMakeLists.txt | 3 - Source/extensions/bluetooth/gatt/Module.h | 2 - cmake/common/GetBluez5Headers.cmake | 80 +++++++++++ cmake/modules/FindBluez5UtilHeaders.cmake | 126 ------------------ 11 files changed, 108 insertions(+), 151 deletions(-) create mode 100644 cmake/common/GetBluez5Headers.cmake delete mode 100644 cmake/modules/FindBluez5UtilHeaders.cmake diff --git a/Source/core/CMakeLists.txt b/Source/core/CMakeLists.txt index 65d9b6cb6..256b202f2 100644 --- a/Source/core/CMakeLists.txt +++ b/Source/core/CMakeLists.txt @@ -206,12 +206,27 @@ endif() if(BLUETOOTH_SUPPORT) target_compile_definitions(${TARGET} PUBLIC __CORE_BLUETOOTH_SUPPORT__) - find_package(Bluez5UtilHeaders REQUIRED) + include(GetBluez5Headers) - target_link_libraries(${TARGET} + GetBluez5UtilHeadersFiles(Bluez5UtilHeadersFiles) + + install(FILES ${Bluez5UtilHeadersFiles} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${NAMESPACE}/core/bluez5 COMPONENT ${NAMESPACE}_Development) + + GetBluez5IncludeDirs(Bluez5IncludeDirs) + + foreach(path ${Bluez5IncludeDirs}) + target_include_directories(${TARGET} PUBLIC - Bluez5UtilHeaders::Bluez5UtilHeaders - ) + $) + endforeach() + + CheckBluez5InclusiveLanguage(INCLUSIVE_LANGUAGE) + + if(NOT ${INCLUSIVE_LANGUAGE}) + #TODO: Remove NO_INCLUSIVE_LANGUAGE define when the plugins are synced to BLUEZ_HAS_NO_INCLUSIVE_LANGUAGE + target_compile_definitions(${TARGET} PUBLIC INTERFACE NO_INCLUSIVE_LANGUAGE BLUEZ_HAS_NO_INCLUSIVE_LANGUAGE) + endif() message(STATUS "Enable bluetooth support.") endif() diff --git a/Source/core/NodeId.h b/Source/core/NodeId.h index 4acfec44d..433ae1003 100644 --- a/Source/core/NodeId.h +++ b/Source/core/NodeId.h @@ -46,9 +46,11 @@ #endif #ifdef __CORE_BLUETOOTH_SUPPORT__ -#include <../include/bluetooth/bluetooth.h> -#include <../include/bluetooth/hci.h> -#include <../include/bluetooth/l2cap.h> +PUSH_WARNING(DISABLE_WARNING_PEDANTIC) +#include "bluez5/bluetooth.h" +#include "bluez5/hci.h" +#include "bluez5/l2cap.h" +POP_WARNING() #else #ifndef AF_BLUETOOTH #define AF_BLUETOOTH 60000 diff --git a/Source/extensions/bluetooth/CMakeLists.txt b/Source/extensions/bluetooth/CMakeLists.txt index 17afeccb6..fec0ddf61 100644 --- a/Source/extensions/bluetooth/CMakeLists.txt +++ b/Source/extensions/bluetooth/CMakeLists.txt @@ -31,8 +31,6 @@ option(BCM43XX "Select the serial driver for bluetooth modules found on Raspberr option(BLUETOOTH_GATT_SUPPORT "Include GATT support" OFF) option(BLUETOOTH_AUDIO_SUPPORT "Include audio sink/source support" OFF) -find_package(Bluez5UtilHeaders REQUIRED) - add_library(${TARGET} HCISocket.cpp UUID.cpp @@ -62,7 +60,6 @@ target_link_libraries(${TARGET} CompileSettingsDebug::CompileSettingsDebug ${NAMESPACE}Core::${NAMESPACE}Core ${NAMESPACE}Messaging::${NAMESPACE}Messaging - Bluez5UtilHeaders::Bluez5UtilHeaders ) set_target_properties(${TARGET} diff --git a/Source/extensions/bluetooth/HCISocket.h b/Source/extensions/bluetooth/HCISocket.h index 908c28f7c..fa73ac95c 100644 --- a/Source/extensions/bluetooth/HCISocket.h +++ b/Source/extensions/bluetooth/HCISocket.h @@ -23,6 +23,10 @@ #include "UUID.h" #include "BluetoothUtils.h" +PUSH_WARNING(DISABLE_WARNING_PEDANTIC) +#include +POP_WARNING() + namespace Thunder { namespace Bluetooth { diff --git a/Source/extensions/bluetooth/Module.h b/Source/extensions/bluetooth/Module.h index 499c7391b..9c24b5323 100644 --- a/Source/extensions/bluetooth/Module.h +++ b/Source/extensions/bluetooth/Module.h @@ -26,11 +26,6 @@ #include #include -#include <../include/bluetooth/bluetooth.h> -#include <../include/bluetooth/hci.h> -#include <../include/bluetooth/mgmt.h> -#include <../include/bluetooth/l2cap.h> - #include "Debug.h" #if defined(__WINDOWS__) && defined(BLUETOOTH_EXPORTS) diff --git a/Source/extensions/bluetooth/audio/CMakeLists.txt b/Source/extensions/bluetooth/audio/CMakeLists.txt index 267750ba4..1a3d470b2 100644 --- a/Source/extensions/bluetooth/audio/CMakeLists.txt +++ b/Source/extensions/bluetooth/audio/CMakeLists.txt @@ -27,7 +27,6 @@ message("Setup ${TARGET} v${PROJECT_VERSION}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") -find_package(Bluez5UtilHeaders REQUIRED) find_package(SBC REQUIRED) file(GLOB CODEC_HEADERS codecs/*.h) @@ -59,7 +58,6 @@ target_link_libraries(${TARGET} CompileSettingsDebug::CompileSettingsDebug ${NAMESPACE}Core::${NAMESPACE}Core ${NAMESPACE}Messaging::${NAMESPACE}Messaging - Bluez5UtilHeaders::Bluez5UtilHeaders SBC::SBC ) diff --git a/Source/extensions/bluetooth/audio/Module.h b/Source/extensions/bluetooth/audio/Module.h index 9cb529d5a..0aa369927 100644 --- a/Source/extensions/bluetooth/audio/Module.h +++ b/Source/extensions/bluetooth/audio/Module.h @@ -26,9 +26,6 @@ #include #include -#include <../include/bluetooth/bluetooth.h> -#include <../include/bluetooth/l2cap.h> - #include "../Debug.h" #include "../UUID.h" diff --git a/Source/extensions/bluetooth/gatt/CMakeLists.txt b/Source/extensions/bluetooth/gatt/CMakeLists.txt index 89992f9cf..d14b5403a 100644 --- a/Source/extensions/bluetooth/gatt/CMakeLists.txt +++ b/Source/extensions/bluetooth/gatt/CMakeLists.txt @@ -25,8 +25,6 @@ project(${NAMESPACE}BluetoothGATT set(TARGET ${PROJECT_NAME}) message("Setup ${TARGET} v${PROJECT_VERSION}") -find_package(Bluez5UtilHeaders REQUIRED) - set(PUBLIC_HEADERS GATTSocket.h GATTProfile.h @@ -45,7 +43,6 @@ target_link_libraries(${TARGET} CompileSettingsDebug::CompileSettingsDebug ${NAMESPACE}Core::${NAMESPACE}Core ${NAMESPACE}Messaging::${NAMESPACE}Messaging - Bluez5UtilHeaders::Bluez5UtilHeaders ) set_target_properties(${TARGET} diff --git a/Source/extensions/bluetooth/gatt/Module.h b/Source/extensions/bluetooth/gatt/Module.h index a3168d40c..b7e4c669d 100644 --- a/Source/extensions/bluetooth/gatt/Module.h +++ b/Source/extensions/bluetooth/gatt/Module.h @@ -26,8 +26,6 @@ #include #include -#include <../include/bluetooth/bluetooth.h> - #include "../Debug.h" #include "../UUID.h" diff --git a/cmake/common/GetBluez5Headers.cmake b/cmake/common/GetBluez5Headers.cmake new file mode 100644 index 000000000..259f242dd --- /dev/null +++ b/cmake/common/GetBluez5Headers.cmake @@ -0,0 +1,80 @@ +option(DOWNLOAD_BLUEZ_UTIL_HEADERS "Download bluez5 headers" OFF) +set(DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION "5.78" CACHE STRING "version of the bluez5 headers to download...") +set(DOWNLOAD_BLUEZ_UTIL_HEADERS_REPO "https://github.com/bluez/bluez.git" CACHE STRING "Repo where to get the bluez5 headers...") + +set(BLUEZ_LOCAL_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/core/bluez5") + +include(CreateLink) + +if(DOWNLOAD_BLUEZ_UTIL_HEADERS) + message(STATUS "Downloaded bluez5 headers are used, assuming your \"the expert\"!") + + include(GetExternalCode) + + + GetExternalCode( + GIT_REPOSITORY "${DOWNLOAD_BLUEZ_UTIL_HEADERS_REPO}" + GIT_VERSION "${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}" + SOURCE_DIR "${CMAKE_BINARY_DIR}/bluez5-${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}" + ) + + CreateLink( + LINK "${BLUEZ_LOCAL_INCLUDE_DIR}" + TARGET "${CMAKE_BINARY_DIR}/bluez5-${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}/lib" + ) + +else() + set(BLUEZ_INCLUDE_DIR) + + # Find the bluez5 headers in a sysroot/staging location + find_path(_header_path bluetooth/bluetooth.h) + if(_header_path) + message(VERBOSE "Found bluetooth.h in ${_header_path}") + set(BLUEZ_INCLUDE_DIR "${_header_path}") + endif() + + CreateLink( + LINK "${BLUEZ_LOCAL_INCLUDE_DIR}" + TARGET "${BLUEZ_INCLUDE_DIR}/bluetooth" + ) +endif() + +function(GetBluez5UtilHeadersFiles var) + file(GLOB Bluez5UtilHeadersFiles "${BLUEZ_LOCAL_INCLUDE_DIR}/*.h") + set(${var} ${Bluez5UtilHeadersFiles} PARENT_SCOPE) +endfunction(GetBluez5UtilHeadersFiles) + +function(GetBluez5IncludeDirs var) + set(${var} "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/core" PARENT_SCOPE) +endfunction(GetBluez5IncludeDirs) + +# +# From Bluez >= v5.64 the mgmt_ltk_info struct is changed due to inclusive language changes. +# https://github.com/bluez/bluez/commit/b7d6a7d25628e9b521a29a5c133fcadcedeb2102 +# This a function so CMAKE_REQUIRED_FLAGS is only valid in this scope +# it sets the `var` variable in the calling scope accordingly. +# +function(CheckBluez5InclusiveLanguage var) + include(CheckStructHasMember) + + # is needed because of pedenic warnings in the bluez headers + set(CMAKE_REQUIRED_FLAGS "-Wno-error") + + check_struct_has_member("struct mgmt_ltk_info" master + "${BLUEZ_LOCAL_INCLUDE_DIR}/bluetooth.h;${BLUEZ_LOCAL_INCLUDE_DIR}/mgmt.h" + INCLUSIVE_LANGUAGE + LANGUAGE C) + + check_struct_has_member("struct mgmt_ltk_info" central + "${BLUEZ_LOCAL_INCLUDE_DIR}/bluetooth.h;${BLUEZ_LOCAL_INCLUDE_DIR}/mgmt.h" + NO_INCLUSIVE_LANGUAGE + LANGUAGE C) + + if(NOT INCLUSIVE_LANGUAGE AND NOT NO_INCLUSIVE_LANGUAGE) + message(FATAL_ERROR "Could not determine the usage of inclusive language, probably due to a compilation error. Please check the cmake error log for details.") + elseif(NO_INCLUSIVE_LANGUAGE) + set(${var} FALSE PARENT_SCOPE) + elseif(INCLUSIVE_LANGUAGE) + set(${var} TRUE PARENT_SCOPE) + endif() +endfunction(CheckBluez5InclusiveLanguage) diff --git a/cmake/modules/FindBluez5UtilHeaders.cmake b/cmake/modules/FindBluez5UtilHeaders.cmake deleted file mode 100644 index 00cd3bbb3..000000000 --- a/cmake/modules/FindBluez5UtilHeaders.cmake +++ /dev/null @@ -1,126 +0,0 @@ -# - Try to find Bluez Utils Headers -# Once done this will define -# Bluez5UtilHeaders::Bluez5UtilHeaders - The bluez include directories -# -# Copyright (C) 2022 Metrological. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND ITS CONTRIBUTORS ``AS -# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR ITS -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE - -option(DOWNLOAD_BLUEZ_UTIL_HEADERS "Download bluez5 headers, only for testing or development..." OFF) -set(DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION "5.78" CACHE STRING "version of the bluez5 headers to download...") -set(DOWNLOAD_BLUEZ_UTIL_HEADERS_REPO "https://github.com/bluez/bluez.git" CACHE STRING "Repo where to get the bluez5 headers...") - -if(Bluez5UtilHeaders_FIND_QUIETLY) - set(_FIND_MODE QUIET) -elseif(Bluez5UtilHeaders_FIND_REQUIRED) - set(_FIND_MODE REQUIRED) -endif() - -set(NEEDED_BLUEZ_HEADERS - bluetooth.h - hci.h - mgmt.h - l2cap.h -) - -if(NOT TARGET Bluez5UtilHeaders) - set(BLUEZ_INCLUDE_DIRS) - - if(DOWNLOAD_BLUEZ_UTIL_HEADERS AND NOT TARGET DownloadedBluez5Headers) - message(STATUS "Downloaded bluez5 headers are used, assuming your \"the expert\"!") - - include(GetExternalCode) - include(CreateLink) - - GetExternalCode( - GIT_REPOSITORY "${DOWNLOAD_BLUEZ_UTIL_HEADERS_REPO}" - GIT_VERSION "${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}" - SOURCE_DIR "${CMAKE_BINARY_DIR}/bluez-${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}" - ) - - add_library(DownloadedBluez5Headers INTERFACE) - - #FIX ME: Hack for weird include paths in the source... - CreateLink( - LINK "${CMAKE_BINARY_DIR}/bluez/include/bluetooth" - TARGET "${CMAKE_BINARY_DIR}/bluez-${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}/lib" - ) - - set(BLUEZ_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/bluez/include") - else() - foreach(_header ${NEEDED_BLUEZ_HEADERS}) - find_path(_header_path bluetooth/${_header}) - if(_header_path) - message(VERBOSE "Found ${_header} in ${_header_path}") - list(APPEND BLUEZ_INCLUDE_DIRS ${_header_path}) - endif() - endforeach() - - list(REMOVE_DUPLICATES BLUEZ_INCLUDE_DIRS) - endif() - - add_library(Bluez5UtilHeaders INTERFACE) - add_library(Bluez5UtilHeaders::Bluez5UtilHeaders ALIAS Bluez5UtilHeaders) - - # - # From Bluez >= v5.64 the mgmt_ltk_info struct is changed due to inclusive language changes. - # https://github.com/bluez/bluez/commit/b7d6a7d25628e9b521a29a5c133fcadcedeb2102 - # - include(CheckStructHasMember) - check_struct_has_member("struct mgmt_ltk_info" central - "${BLUEZ_INCLUDE_DIRS}/bluetooth/bluetooth.h;${BLUEZ_INCLUDE_DIRS}/bluetooth/mgmt.h" - NO_INCLUSIVE_LANGUAGE - LANGUAGE C) - - if(${NO_INCLUSIVE_LANGUAGE}) - message(VERBOSE "Your bluez version does not use inclusive language anymore") - target_compile_definitions(Bluez5UtilHeaders INTERFACE NO_INCLUSIVE_LANGUAGE) - endif() - - if(DOWNLOAD_BLUEZ_UTIL_HEADERS) - target_include_directories(Bluez5UtilHeaders - INTERFACE - $ - $ - ) - - install(TARGETS Bluez5UtilHeaders EXPORT Bluez5UtilHeadersTargets) - - file(GLOB Bluez5UtilHeadersFiles "${CMAKE_BINARY_DIR}/bluez-${DOWNLOAD_BLUEZ_UTIL_HEADERS_VERSION}/lib/*.h") - - install(FILES ${Bluez5UtilHeadersFiles} - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${NAMESPACE}/Bluez5UtilHeaders/include/bluetooth COMPONENT ${NAMESPACE}_Development) - - include(HeaderOnlyInstall) - HeaderOnlyInstallCMakeConfig(TARGET Bluez5UtilHeaders TREAT_AS_NORMAL) - else() - target_include_directories(Bluez5UtilHeaders - INTERFACE - ${BLUEZ_INCLUDE_DIRS}) - endif() - - message(TRACE "BLUEZ_INCLUDE_DIRS ${BLUEZ_INCLUDE_DIRS}") - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Bluez5UtilHeaders DEFAULT_MSG BLUEZ_INCLUDE_DIRS) - mark_as_advanced(BLUEZ_INCLUDE_DIRS) -endif() \ No newline at end of file From da77127f06d952d8208aa185e025c36fb974670e Mon Sep 17 00:00:00 2001 From: Mateusz Daniluk <121170681+VeithMetro@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:46:07 +0200 Subject: [PATCH 03/18] [Actions] Build the Bluetooth extension with audio and GATT support (#1759) * Trying to build two new extensions in the Linux Actions * Make sure to use the template from development to build * Installing BlueZ, updating the multilib and 32-bit libraries to be only include for 32-bit builds * Adding a missing workflow variable operator * Trying to fix the bluetooth extenstion include paths * Bringing back the old include paths as it does not solve the issue * Looks like building the bluetooth extension seems causes some issues, let's keep it off for now * Trying to build bluetooth again with the mgmt header * Try building without downloading the BlueZ packages first * Adding the DOWNLOAD_BLUEZ_UTIL_HEADERS flag and setting it up * Adding a SYSTEM keyword to supress warnings from BlueZ headers * Trying to set the NO_INCLUSIVE_LANGUAGE flag to PUBLIC to see if that fixes the issue * Removing SYSTEM keyword for now to test if that causes problem with flags propagation * For now removing treatment of warnings as error to test the changes * Changing BlueZ to interface in target compile definitions * Disabling pedanitc warnings in the included BlueZ headers * Add the -Werror cmake flags back * Disabling pedantic warning in BlueZ headers includes in NodeId * Changing the FindBluez5UtilHeaders cmake to suppress warnings treated as errors * Removing PUSH_WARNING macro use in NodeId as it is no longer needed * Removing PUSH_WARNING macro use in the bluetooth extension * Removing FindBluez5UtilHeaders from cmake/modules to merge master changes * Adding Bluetooth audio and gatt build options * Installing the libsbc-dev * Make sure to install a 32-bit version if necessary * Make sure the PKG paths are correct for each build type * Make sure to use master template before merging --- .github/workflows/Linux build template.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Linux build template.yml b/.github/workflows/Linux build template.yml index d492db1df..81f52d83d 100644 --- a/.github/workflows/Linux build template.yml +++ b/.github/workflows/Linux build template.yml @@ -26,9 +26,9 @@ jobs: sudo apt-spy2 fix --commit --launchpad --country=US echo "deb http://archive.ubuntu.com/ubuntu/ jammy main universe restricted multiverse" | sudo tee -a /etc/apt/sources.list echo "deb http://archive.ubuntu.com/ubuntu/ jammy-updates main universe restricted multiverse" | sudo tee -a /etc/apt/sources.list - sudo dpkg --add-architecture i386 + ${{matrix.architecture == '32' && 'sudo dpkg --add-architecture i386' || ':'}} sudo apt-get update - sudo apt install python3-pip build-essential cmake ninja-build libusb-1.0-0-dev zlib1g-dev zlib1g-dev:i386 libssl-dev gcc-11-multilib g++-11-multilib + sudo apt install python3-pip build-essential cmake ninja-build libusb-1.0-0-dev ${{matrix.architecture == '32' && 'zlib1g-dev:i386 libssl-dev:i386 libsbc-dev:i386 gcc-11-multilib g++-11-multilib' || 'zlib1g-dev libssl-dev libsbc-dev'}} sudo pip install jsonref # ----- Checkout & DependsOn regex ----- @@ -85,6 +85,7 @@ jobs: # ----- Installing generators & Options regex ----- - name: Install generators run: | + ${{matrix.architecture == '32' && 'export PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig:$PKG_CONFIG_PATH' || 'PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH'}} cmake -G Ninja -S ThunderTools -B ${{matrix.build_type}}/build/ThunderTools \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ -DCMAKE_C_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ @@ -103,6 +104,7 @@ jobs: # ----- Building & uploading artifacts ----- - name: Build Thunder run: | + ${{matrix.architecture == '32' && 'export PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig:$PKG_CONFIG_PATH' || 'PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH'}} cmake -G Ninja -S Thunder -B ${{matrix.build_type}}/build/Thunder \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ -DCMAKE_C_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ @@ -117,6 +119,11 @@ jobs: -DPROXYSTUB_PATH="${PWD}/${{matrix.build_type}}/install/usr/lib/wpeframework/proxystubs" \ -DSYSTEM_PATH="${PWD}/${{matrix.build_type}}/install/usr/lib/wpeframework/plugins" \ -DVOLATILE_PATH="tmp" \ + -DBLUETOOTH_SUPPORT=ON \ + -DBLUETOOTH=ON \ + -DDOWNLOAD_BLUEZ_UTIL_HEADERS=ON \ + -DBLUETOOTH_AUDIO_SUPPORT=ON \ + -DBLUETOOTH_GATT_SUPPORT=ON \ -DLOCALTRACER=ON \ -DWARNING_REPORTING=ON \ -DPROCESSCONTAINERS=ON \ From 7622fefb0c4f9d6f405bef39e268adaf2216590a Mon Sep 17 00:00:00 2001 From: sebaszm <45654185+sebaszm@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:21:05 +0200 Subject: [PATCH 04/18] [Core] Serialize null JSON Array and Container (#1762) --- Source/core/JSON.h | 308 ++++++++++++++++++++++++++------------------- 1 file changed, 178 insertions(+), 130 deletions(-) diff --git a/Source/core/JSON.h b/Source/core/JSON.h index c4ba49f78..7fd5a4438 100644 --- a/Source/core/JSON.h +++ b/Source/core/JSON.h @@ -3190,10 +3190,19 @@ namespace Core { bool IsNull() const override { - //TODO: Implement null for Arrays return ((_state & UNDEFINED) != 0); } + void Null(const bool enabled) + { + if (enabled == true) { + _state |= (UNDEFINED | SET); + } + else { + _state &= ~(UNDEFINED | SET); + } + } + void Set(const bool enabled) { if (enabled == true) { @@ -3360,33 +3369,40 @@ namespace Core { { uint16_t loaded = 0; - if (offset == FIND_MARKER) { - _iterator.Reset(); - if (((_state & modus::EXTRACT) == 0) || (_data.size() != 1)) { - stream[loaded++] = '['; - } - offset = (_iterator.Next() == false ? ~0 : PARSE); - } else if (offset == END_MARKER) { - offset = ~0; - } - while ((loaded < maxLength) && (offset != static_cast(~0))) { - if (offset >= PARSE) { - offset -= PARSE; - loaded += static_cast(_iterator.Current()).Serialize(&(stream[loaded]), maxLength - loaded, offset); - offset = (offset != FIND_MARKER ? offset + PARSE : (_iterator.Next() == true ? BEGIN_MARKER : ~0)); - } else if (offset == BEGIN_MARKER) { - stream[loaded++] = ','; - offset = PARSE; - } + if ((_state & modus::UNDEFINED) != 0) { + ASSERT(offset < (sizeof(IElement::NullTag) - 1)); + loaded = std::min(static_cast((sizeof(IElement::NullTag) - 1) - offset), maxLength); + ::memcpy(stream, &(IElement::NullTag[offset]), loaded); } - if (offset == static_cast(~0)) { - if (loaded < maxLength) { + else { + if (offset == FIND_MARKER) { + _iterator.Reset(); if (((_state & modus::EXTRACT) == 0) || (_data.size() != 1)) { - stream[loaded++] = ']'; + stream[loaded++] = '['; + } + offset = (_iterator.Next() == false ? ~0 : PARSE); + } else if (offset == END_MARKER) { + offset = ~0; + } + while ((loaded < maxLength) && (offset != static_cast(~0))) { + if (offset >= PARSE) { + offset -= PARSE; + loaded += static_cast(_iterator.Current()).Serialize(&(stream[loaded]), maxLength - loaded, offset); + offset = (offset != FIND_MARKER ? offset + PARSE : (_iterator.Next() == true ? BEGIN_MARKER : ~0)); + } else if (offset == BEGIN_MARKER) { + stream[loaded++] = ','; + offset = PARSE; + } + } + if (offset == static_cast(~0)) { + if (loaded < maxLength) { + if (((_state & modus::EXTRACT) == 0) || (_data.size() != 1)) { + stream[loaded++] = ']'; + } + offset = FIND_MARKER; + } else { + offset = END_MARKER; } - offset = FIND_MARKER; - } else { - offset = END_MARKER; } } @@ -3482,34 +3498,42 @@ namespace Core { { uint16_t loaded = 0; - if (offset == 0) { - _iterator.Reset(); - if (_data.size() <= 15) { - stream[loaded++] = (0x90 | static_cast(_data.size())); - if (_data.size() > 0) { - offset = PARSE; - } - } else { - stream[loaded++] = 0xDC; - offset = 1; + if ((_state & modus::UNDEFINED) != 0) { + if (offset == 0) { + stream[0] = IMessagePack::NullValue; + loaded = 1; } - _iterator.Next(); } - while ((loaded < maxLength) && (offset > 0) && (offset < PARSE)) { - if (offset == 1) { - stream[loaded++] = (_data.size() >> 8) & 0xFF; - offset = 2; - } else if (offset == 2) { - stream[loaded++] = _data.size() & 0xFF; - offset = PARSE; + else { + if (offset == 0) { + _iterator.Reset(); + if (_data.size() <= 15) { + stream[loaded++] = (0x90 | static_cast(_data.size())); + if (_data.size() > 0) { + offset = PARSE; + } + } else { + stream[loaded++] = 0xDC; + offset = 1; + } + _iterator.Next(); } - } - while ((loaded < maxLength) && (offset >= PARSE)) { - offset -= PARSE; - loaded += static_cast(_iterator.Current()).Serialize(&(stream[loaded]), maxLength - loaded, offset); - offset += PARSE; - if ((offset == PARSE) && (_iterator.Next() != true)) { - offset = 0; + while ((loaded < maxLength) && (offset > 0) && (offset < PARSE)) { + if (offset == 1) { + stream[loaded++] = (_data.size() >> 8) & 0xFF; + offset = 2; + } else if (offset == 2) { + stream[loaded++] = _data.size() & 0xFF; + offset = PARSE; + } + } + while ((loaded < maxLength) && (offset >= PARSE)) { + offset -= PARSE; + loaded += static_cast(_iterator.Current()).Serialize(&(stream[loaded]), maxLength - loaded, offset); + offset += PARSE; + if ((offset == PARSE) && (_iterator.Next() != true)) { + offset = 0; + } } } @@ -3700,9 +3724,18 @@ namespace Core { return (index != _data.end()); } + void Null(const bool enabled) + { + if (enabled == true) { + _state |= UNDEFINED; + } + else { + _state &= ~UNDEFINED; + } + } + bool IsNull() const override { - // TODO: Implement null for conrtainers return ((_state & UNDEFINED) != 0); } @@ -3741,49 +3774,56 @@ namespace Core { { uint16_t loaded = 0; - if (offset == FIND_MARKER) { - _iterator = _data.begin(); - stream[loaded++] = '{'; - - offset = (_iterator == _data.end() ? ~0 : ((_iterator->second->IsSet() == false) && (FindNext() == false)) ? ~0 : BEGIN_MARKER); - if (offset == BEGIN_MARKER) { - _fieldName = string(_iterator->first); - _current.json = &_fieldName; - offset = PARSE; - } - } else if (offset == END_MARKER) { - offset = ~0; + if ((_state & UNDEFINED) != 0) { + ASSERT(offset < (sizeof(IElement::NullTag) - 1)); + loaded = std::min(static_cast((sizeof(IElement::NullTag) - 1) - offset), maxLength); + ::memcpy(stream, &(IElement::NullTag[offset]), loaded); } + else { + if (offset == FIND_MARKER) { + _iterator = _data.begin(); + stream[loaded++] = '{'; - while ((loaded < maxLength) && (offset != static_cast(~0))) { - if (offset >= PARSE) { - offset -= PARSE; - loaded += _current.json->Serialize(&(stream[loaded]), maxLength - loaded, offset); - offset = (offset == FIND_MARKER ? BEGIN_MARKER : offset + PARSE); - } else if (offset == BEGIN_MARKER) { - if (_current.json == &_fieldName) { - stream[loaded++] = ':'; - _current.json = _iterator->second; + offset = (_iterator == _data.end() ? ~0 : ((_iterator->second->IsSet() == false) && (FindNext() == false)) ? ~0 : BEGIN_MARKER); + if (offset == BEGIN_MARKER) { + _fieldName = string(_iterator->first); + _current.json = &_fieldName; offset = PARSE; - } else { - if (FindNext() != false) { - stream[loaded++] = ','; - _fieldName = string(_iterator->first); - _current.json = &_fieldName; + } + } else if (offset == END_MARKER) { + offset = ~0; + } + + while ((loaded < maxLength) && (offset != static_cast(~0))) { + if (offset >= PARSE) { + offset -= PARSE; + loaded += _current.json->Serialize(&(stream[loaded]), maxLength - loaded, offset); + offset = (offset == FIND_MARKER ? BEGIN_MARKER : offset + PARSE); + } else if (offset == BEGIN_MARKER) { + if (_current.json == &_fieldName) { + stream[loaded++] = ':'; + _current.json = _iterator->second; offset = PARSE; } else { - offset = ~0; + if (FindNext() != false) { + stream[loaded++] = ','; + _fieldName = string(_iterator->first); + _current.json = &_fieldName; + offset = PARSE; + } else { + offset = ~0; + } } } } - } - if (offset == static_cast(~0)) { - if (loaded < maxLength) { - stream[loaded++] = '}'; - offset = FIND_MARKER; - _fieldName.Clear(); - } else { - offset = END_MARKER; + if (offset == static_cast(~0)) { + if (loaded < maxLength) { + stream[loaded++] = '}'; + offset = FIND_MARKER; + _fieldName.Clear(); + } else { + offset = END_MARKER; + } } } @@ -3952,60 +3992,68 @@ namespace Core { { uint16_t loaded = 0; - uint16_t elementSize = Size(); - if (offset == 0) { - _iterator = _data.begin(); - if (elementSize <= 15) { - stream[loaded++] = (0x80 | static_cast(Size())); - if (_iterator != _data.end()) { - offset = PARSE; - } - } else { - stream[loaded++] = 0xDE; - offset = 1; + if ((_state & UNDEFINED) != 0) { + if (offset == 0) { + stream[0] = IMessagePack::NullValue; + loaded = 1; } - if (offset != 0) { - if ((_iterator->second->IsSet() == false) && (FindNext() == false)) { - offset = 0; + } + else { + uint16_t elementSize = Size(); + if (offset == 0) { + _iterator = _data.begin(); + if (elementSize <= 15) { + stream[loaded++] = (0x80 | static_cast(Size())); + if (_iterator != _data.end()) { + offset = PARSE; + } } else { - _fieldName = string(_iterator->first); + stream[loaded++] = 0xDE; + offset = 1; + } + if (offset != 0) { + if ((_iterator->second->IsSet() == false) && (FindNext() == false)) { + offset = 0; + } else { + _fieldName = string(_iterator->first); + } } } - } - while ((loaded < maxLength) && (offset > 0) && (offset < PARSE)) { - if (offset == 1) { - stream[loaded++] = (elementSize >> 8) & 0xFF; - offset = 2; - } else if (offset == 2) { - stream[loaded++] = elementSize & 0xFF; - offset = PARSE; - } - } - while ((loaded < maxLength) && (offset >= PARSE)) { - offset -= PARSE; - if (_fieldName.IsSet() == true) { - loaded += static_cast(_fieldName).Serialize(&(stream[loaded]), maxLength - loaded, offset); - if (offset == 0) { - _fieldName.Clear(); + while ((loaded < maxLength) && (offset > 0) && (offset < PARSE)) { + if (offset == 1) { + stream[loaded++] = (elementSize >> 8) & 0xFF; + offset = 2; + } else if (offset == 2) { + stream[loaded++] = elementSize & 0xFF; + offset = PARSE; } - offset += PARSE; - } else { - const IMessagePack* element = dynamic_cast(_iterator->second); - if (element != nullptr) { - loaded += element->Serialize(&(stream[loaded]), maxLength - loaded, offset); + } + while ((loaded < maxLength) && (offset >= PARSE)) { + offset -= PARSE; + if (_fieldName.IsSet() == true) { + loaded += static_cast(_fieldName).Serialize(&(stream[loaded]), maxLength - loaded, offset); if (offset == 0) { _fieldName.Clear(); } + offset += PARSE; } else { - stream[loaded++] = IMessagePack::NullValue; - } - offset += PARSE; - if (offset == PARSE) { - if (FindNext() != false) { - _fieldName = string(_iterator->first); + const IMessagePack* element = dynamic_cast(_iterator->second); + if (element != nullptr) { + loaded += element->Serialize(&(stream[loaded]), maxLength - loaded, offset); + if (offset == 0) { + _fieldName.Clear(); + } } else { - offset = 0; - _fieldName.Clear(); + stream[loaded++] = IMessagePack::NullValue; + } + offset += PARSE; + if (offset == PARSE) { + if (FindNext() != false) { + _fieldName = string(_iterator->first); + } else { + offset = 0; + _fieldName.Clear(); + } } } } From 257d635de870107d76b310310969a23f5a7510f4 Mon Sep 17 00:00:00 2001 From: sebaszm <45654185+sebaszm@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:47:31 +0200 Subject: [PATCH 05/18] DSO search in-process/OOP unification (#1761) Co-authored-by: Pierre Wielders --- Source/Thunder/PluginServer.h | 80 +++++++++++++++------------ Source/ThunderPlugin/Process.cpp | 95 ++++++++++++++++++-------------- Source/com/Communicator.h | 24 ++++++-- Source/core/Library.cpp | 2 - Source/plugins/Shell.cpp | 4 +- 5 files changed, 119 insertions(+), 86 deletions(-) diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index 769fcb036..6d1b72ad5 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -1287,7 +1287,7 @@ namespace PluginHost { { ASSERT(_connection == nullptr); - void* result(_administrator.Instantiate(object, waitTime, sessionId, DataPath(), PersistentPath(), VolatilePath())); + void* result(_administrator.Instantiate(object, waitTime, sessionId, DataPath(), PersistentPath(), VolatilePath(), _administrator.Configuration().LinkerPluginPaths())); if (result != nullptr) { _connection = _administrator.RemoteConnection(sessionId); @@ -1373,43 +1373,38 @@ namespace PluginHost { RPC::IStringIterator* GetLibrarySearchPaths(const string& locator) const override { - std::vector all_paths; - const std::vector temp = _administrator.Configuration().LinkerPluginPaths(); - string rootPath(PluginHost::Service::Configuration().SystemRootPath.Value()); + std::vector searchPaths; - if (rootPath.empty() == false) { - rootPath = Core::Directory::Normalize(rootPath); - } + const string normalized(Core::File::Normalize(locator)); + const string rootPath(Core::Directory::Normalize(PluginHost::Service::Configuration().SystemRootPath)); - if (!temp.empty()) - { - // additionaly defined user paths - for (const string& s : temp) { - if (rootPath.empty() == true) { - all_paths.push_back(Core::Directory::Normalize(s) + locator); + ASSERT_VERBOSE((normalized.empty() == locator.empty()), "path normalization failed"); + ASSERT_VERBOSE((rootPath.empty() == PluginHost::Service::Configuration().SystemRootPath.Value().empty()), "path normalization failed"); + + if (normalized.empty() == false) { + if (Core::File::IsPathAbsolute(locator) == true) { + searchPaths.push_back(Core::File::Normalize(rootPath + normalized)); + } + else { + const std::vector& linkerPaths = _administrator.Configuration().LinkerPluginPaths(); + + if (linkerPaths.empty() == false) { + // override system paths + for (const string& path : linkerPaths) { + searchPaths.push_back(Core::Directory::Normalize(rootPath + path) + normalized); + } } else { - all_paths.push_back(rootPath + Core::Directory::Normalize(s) + locator); + // system configured paths + searchPaths.push_back(Core::Directory::Normalize(rootPath + DataPath()) + normalized); + searchPaths.push_back(Core::Directory::Normalize(rootPath + PersistentPath()) + normalized); + searchPaths.push_back(Core::Directory::Normalize(rootPath + SystemPath()) + normalized); + searchPaths.push_back(Core::Directory::Normalize(rootPath + PluginPath()) + normalized); } } } - else if (rootPath.empty() == false) - { - // system configured paths - all_paths.push_back(rootPath + DataPath() + locator); - all_paths.push_back(rootPath + PersistentPath() + locator); - all_paths.push_back(rootPath + SystemPath() + locator); - all_paths.push_back(rootPath + PluginPath() + locator); - } - else { - // system configured paths - all_paths.push_back(DataPath() + locator); - all_paths.push_back(PersistentPath() + locator); - all_paths.push_back(SystemPath() + locator); - all_paths.push_back(PluginPath() + locator); - } - return (Core::ServiceType::Create(all_paths)); + return (Core::ServiceType::Create(searchPaths)); } Core::Library LoadLibrary(const string& name) { @@ -1417,10 +1412,15 @@ namespace PluginHost { Core::Library result; RPC::IStringIterator* all_paths = GetLibrarySearchPaths(name); + ASSERT(all_paths != nullptr); + string element; while((all_paths->Next(element) == true) && (progressedState <= 2)) { + + TRACE_L1("attempting to load library %s", element.c_str()); + Core::File libraryToLoad(element); - + if (libraryToLoad.Exists() == true) { if (progressedState == 0) { progressedState = 1; @@ -2086,6 +2086,7 @@ namespace PluginHost { const string& proxyStubPath, const string& observableProxyStubPath, const string& postMortemPath, + const std::vector& linkerPaths, const uint8_t softKillCheckWaitTime, const uint8_t hardKillCheckWaitTime, const bool delegatedReleases, @@ -2103,6 +2104,7 @@ namespace PluginHost { #else , _application(EXPAND_AND_QUOTE(HOSTING_COMPROCESS)) #endif + , _linkerPaths(linkerPaths) , _adminLock() , _requestObservers() , _proxyStubObserver(*this, observableProxyStubPath) @@ -2141,9 +2143,9 @@ namespace PluginHost { } public: - void* Create(uint32_t& connectionId, const RPC::Object& instance, const uint32_t waitTime, const string& dataPath, const string& persistentPath, const string& volatilePath) + void* Create(uint32_t& connectionId, const RPC::Object& instance, const uint32_t waitTime, const string& dataPath, const string& persistentPath, const string& volatilePath, const std::vector& linkerPaths) { - return (RPC::Communicator::Create(connectionId, instance, RPC::Config(RPC::Communicator::Connector(), _application, persistentPath, _systemPath, dataPath, volatilePath, _appPath, RPC::Communicator::ProxyStubPath(), _postMortemPath), waitTime)); + return (RPC::Communicator::Create(connectionId, instance, RPC::Config(RPC::Communicator::Connector(), _application, persistentPath, _systemPath, dataPath, volatilePath, _appPath, RPC::Communicator::ProxyStubPath(), _postMortemPath, linkerPaths), waitTime)); } const string& PersistentPath() const { @@ -2177,6 +2179,10 @@ namespace PluginHost { { return (_application); } + const std::vector& LinkerPaths() const + { + return (_linkerPaths); + } void Register(RPC::IRemoteConnection::INotification* sink) { RPC::Communicator::Register(sink); @@ -2328,6 +2334,7 @@ namespace PluginHost { const string _observableProxyStubPath; const string _postMortemPath; const string _application; + std::vector _linkerPaths; mutable Core::CriticalSection _adminLock; Observers _requestObservers; ProxyStubObserver _proxyStubObserver; @@ -2405,7 +2412,7 @@ namespace PluginHost { } uint32_t id; - RPC::Config config(_connector, _comms.Application(), persistentPath, _comms.SystemPath(), dataPath, volatilePath, _comms.AppPath(), _comms.ProxyStubPath(), _comms.PostMortemPath()); + RPC::Config config(_connector, _comms.Application(), persistentPath, _comms.SystemPath(), dataPath, volatilePath, _comms.AppPath(), _comms.ProxyStubPath(), _comms.PostMortemPath(), _comms.LinkerPaths()); RPC::Object instance(libraryName, className, callsign, interfaceId, version, user, group, threads, priority, RPC::Object::HostType::LOCAL, systemRootPath, _T(""), configuration); RPC::Communicator::Process process(requestId, config, instance); @@ -2652,6 +2659,7 @@ namespace PluginHost { server._config.ProxyStubPath(), server._config.ObservableProxyStubPath(), server._config.PostMortemPath(), + server._config.LinkerPluginPaths(), server._config.SoftKillCheckWaitTime(), server._config.HardKillCheckWaitTime(), server._config.DelegatedReleases(), @@ -2895,9 +2903,9 @@ namespace PluginHost { return (result); } - void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId, const string& dataPath, const string& persistentPath, const string& volatilePath) + void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId, const string& dataPath, const string& persistentPath, const string& volatilePath, const std::vector& linkerPaths) { - return (_processAdministrator.Create(sessionId, object, waitTime, dataPath, persistentPath, volatilePath)); + return (_processAdministrator.Create(sessionId, object, waitTime, dataPath, persistentPath, volatilePath, linkerPaths)); } void Destroy(const uint32_t id) { _processAdministrator.Destroy(id); diff --git a/Source/ThunderPlugin/Process.cpp b/Source/ThunderPlugin/Process.cpp index e8510e0ef..df3d64432 100644 --- a/Source/ThunderPlugin/Process.cpp +++ b/Source/ThunderPlugin/Process.cpp @@ -81,7 +81,6 @@ POP_WARNING() TRACE_L1("We still have living object [%d].", instances); } else { - TRACE_L1("All living objects are killed. Time for HaraKiri!!."); // Seems there is no more live here, time to signal the @@ -130,7 +129,6 @@ POP_WARNING() } void Run() { - Core::WorkerPool::Run(); Core::WorkerPool::Join(); } @@ -161,7 +159,7 @@ POP_WARNING() class ConsoleOptions : public Core::Options { public: ConsoleOptions(int argumentCount, TCHAR* arguments[]) - : Core::Options(argumentCount, arguments, _T("h:l:c:C:r:p:s:d:a:m:i:u:g:t:e:x:V:v:P:S:")) + : Core::Options(argumentCount, arguments, _T("h:l:c:C:r:p:s:d:a:m:i:u:g:t:e:x:V:v:P:S:f:")) , Locator(nullptr) , ClassName(nullptr) , Callsign(nullptr) @@ -180,6 +178,7 @@ POP_WARNING() , User(nullptr) , Group(nullptr) , Threads(1) + , LinkerPaths() { Parse(); } @@ -206,6 +205,7 @@ POP_WARNING() const TCHAR* User; const TCHAR* Group; uint8_t Threads; + std::vector LinkerPaths; private: string Strip(const TCHAR text[]) const @@ -236,28 +236,28 @@ POP_WARNING() RemoteChannel = argument; break; case 'p': - PersistentPath = Strip(argument); + PersistentPath = Core::Directory::Normalize(Strip(argument)); break; case 's': - SystemPath = Strip(argument); + SystemPath = Core::Directory::Normalize(Strip(argument)); break; case 'd': - DataPath = Strip(argument); + DataPath = Core::Directory::Normalize(Strip(argument)); break; case 'P': - PostMortemPath = Strip(argument); + PostMortemPath = Core::Directory::Normalize(Strip(argument)); break; case 'S': - SystemRootPath = Strip(argument); + SystemRootPath = Core::Directory::Normalize(Strip(argument)); break; case 'v': - VolatilePath = Strip(argument); + VolatilePath = Core::Directory::Normalize(Strip(argument)); break; case 'a': - AppPath = Strip(argument); + AppPath = Core::Directory::Normalize(Strip(argument)); break; case 'm': - ProxyStubPath = Strip(argument); + ProxyStubPath = Core::Directory::Normalize(Strip(argument)); break; case 'u': User = argument; @@ -277,6 +277,9 @@ POP_WARNING() case 't': Threads = Core::NumberType(Core::TextFragment(argument)).Value(); break; + case 'f': + LinkerPaths.push_back(Strip(argument)); + break; case 'h': default: RequestUsage(true); @@ -285,52 +288,59 @@ POP_WARNING() } }; - static void* CheckInstance(const string& path, const TCHAR locator[], const TCHAR className[], const uint32_t ID, const uint32_t version) + static void* CheckInstance(const string& path, const ConsoleOptions& options) { void* result = nullptr; if (path.empty() == false) { - Core::ServiceAdministrator& admin(Core::ServiceAdministrator::Instance()); - string libraryPath = locator; - if (libraryPath.empty() || (libraryPath[0] != '/')) { - // Relative path, prefix with path name. - string pathName(Core::Directory::Normalize(path)); - libraryPath = pathName + locator; - } + TRACE_L1("Attempting to load '%s' from %s...", options.ClassName, path.c_str()); - Core::Library library(libraryPath.c_str()); + Core::Library library(path.c_str()); if (library.IsLoaded() == true) { // Instantiate the object - result = admin.Instantiate(library, className, version, ID); + result = Core::ServiceAdministrator::Instance().Instantiate(library, options.ClassName, options.Version, options.InterfaceId); } } return (result); } - static void* AcquireInterfaces(ConsoleOptions& options) + static void* AcquireInterfaces(const ConsoleOptions& options) { void* result = nullptr; if ((options.Locator != nullptr) && (options.ClassName != nullptr)) { - string path = (!options.SystemRootPath.empty() ? options.SystemRootPath : "") + options.PersistentPath; - result = CheckInstance(path, options.Locator, options.ClassName, options.InterfaceId, options.Version); - if (result == nullptr) { - path = (!options.SystemRootPath.empty() ? options.SystemRootPath : "") + options.SystemPath; - result = CheckInstance(path, options.Locator, options.ClassName, options.InterfaceId, options.Version); + const string rootPath = Core::Directory::Normalize(options.SystemRootPath); + + if (Core::File::IsPathAbsolute(options.Locator) == true) { + result = CheckInstance(Core::File::Normalize(rootPath + options.Locator), options); + } + else if (options.LinkerPaths.empty() == false) { + // Linker paths override system paths + for (const string& linkerPath : options.LinkerPaths) { + result = CheckInstance((Core::Directory::Normalize(options.SystemRootPath + linkerPath) + options.Locator), options); + + if (result != nullptr) { + break; + } + } + } + else { + // System paths + result = CheckInstance((Core::Directory::Normalize(options.SystemRootPath + options.PersistentPath) + options.Locator), options); if (result == nullptr) { - path = (!options.SystemRootPath.empty() ? options.SystemRootPath : "") + options.DataPath; - result = CheckInstance(path, options.Locator, options.ClassName, options.InterfaceId, options.Version); + result = CheckInstance((Core::Directory::Normalize(options.SystemRootPath + options.SystemPath) + options.Locator), options); if (result == nullptr) { - string searchPath(options.AppPath.empty() == false ? Core::Directory::Normalize(options.AppPath) : string()); + result = CheckInstance((Core::Directory::Normalize(options.SystemRootPath + options.DataPath) + options.Locator), options); - path = (!options.SystemRootPath.empty() ? options.SystemRootPath : "") + searchPath; - result = CheckInstance((path + _T("Plugins/")), options.Locator, options.ClassName, options.InterfaceId, options.Version); + if (result == nullptr) { + result = CheckInstance((Core::Directory::Normalize(options.SystemRootPath + options.AppPath + _T("Plugins")) + options.Locator), options); + } } } } @@ -577,23 +587,26 @@ int main(int argc, char** argv) printf(" [-s ]\n"); printf(" [-d ]\n"); printf(" [-v ]\n"); + printf(" [-f ...\n"); printf(" [-a ]\n"); printf(" [-m ]\n"); printf(" [-P ]\n\n"); printf(" [-S ]\n\n"); + printf("\n"); printf("This application spawns a seperate process space for a plugin. The plugins"); printf("are searched in the same order as they are done in process. Starting from:\n"); - printf(" 1) /\n"); - printf(" 2) /\n"); - printf(" 3) /\n"); - printf(" 4) /Plugins/\n\n"); + printf(" 1) [system_path/]/\n"); + printf(" 2) [system_path/]/\n"); + printf(" 3) [system_path/]/\n"); + printf(" 4) [system_path/]/Plugins/\n\n"); + printf("\n"); + printf("Alternatively, if linker paths are specified:\n"); + printf(" 1) [system_path/]/locator\n"); + printf(" 2) [system_path/]/locator, ...\n"); + printf("\n"); printf("Within the DSO, the system looks for an object with , this object must implement "); printf("the interface, indicated byt the Id , and if passed, the object should be of "); printf("version . All these conditions must met for an object to be instantiated and thus run.\n\n"); - - for (uint8_t teller = 0; teller < argc; teller++) { - printf("Argument [%02d]: %s\n", teller, argv[teller]); - } } else { string callsign; if (options.Callsign != nullptr) { @@ -640,7 +653,7 @@ int main(int argc, char** argv) if (remoteNode.IsValid()) { void* base = nullptr; - TRACE_L1("Spawning a new plugin %s.", options.ClassName); + TRACE_L1("Spawning a new plugin %s", options.Callsign); // Firts make sure we apply the correct rights to our selves.. if (options.Group != nullptr) { diff --git a/Source/com/Communicator.h b/Source/com/Communicator.h index 35aea1ced..054a8644b 100644 --- a/Source/com/Communicator.h +++ b/Source/com/Communicator.h @@ -150,7 +150,6 @@ namespace RPC { return (*this); } - Object& operator=(Object&& move) noexcept { if (this != &move) { @@ -172,9 +171,9 @@ namespace RPC { move._version = ~0; move._threads = 0; move._priority = 0; - } - return (*this); - } + } + return (*this); + } public: inline const string& Locator() const @@ -262,6 +261,7 @@ namespace RPC { , _application() , _proxyStub() , _postMortem() + , _linker() { } Config( @@ -273,7 +273,8 @@ namespace RPC { const string& volatilePath, const string& applicationPath, const string& proxyStubPath, - const string& postMortem) + const string& postMortem, + const std::vector& linker) : _connector(connector) , _hostApplication(hostApplication) , _persistent(persistentPath) @@ -283,6 +284,7 @@ namespace RPC { , _application(applicationPath) , _proxyStub(proxyStubPath) , _postMortem(postMortem) + , _linker(linker) { } Config(const Config& copy) @@ -295,6 +297,7 @@ namespace RPC { , _application(copy._application) , _proxyStub(copy._proxyStub) , _postMortem(copy._postMortem) + , _linker(copy._linker) { } Config(Config&& move) noexcept @@ -307,6 +310,7 @@ namespace RPC { , _application(std::move(move._application)) , _proxyStub(std::move(move._proxyStub)) , _postMortem(std::move(move._postMortem)) + , _linker(std::move(move._linker)) { } ~Config() @@ -350,6 +354,10 @@ namespace RPC { { return (_postMortem); } + inline const std::vector& LinkerPaths() const + { + return (_linker); + } private: string _connector; @@ -361,6 +369,7 @@ namespace RPC { string _application; string _proxyStub; string _postMortem; + std::vector _linker; }; struct EXTERNAL IMonitorableProcess : public virtual Core::IUnknown { @@ -490,6 +499,9 @@ namespace RPC { if (config.VolatilePath().empty() == false) { _options.Add(_T("-v")).Add('"' + config.VolatilePath() + '"'); } + for (auto const& path : config.LinkerPaths()) { + _options.Add(_T("-f")).Add('"' + path + '"'); + } if (config.ProxyStubPath().empty() == false) { _options.Add(_T("-m")).Add('"' + config.ProxyStubPath() + '"'); } @@ -865,7 +877,7 @@ namespace RPC { TRACE_L1("Invalid process container configuration"); } else { - std::vector searchPaths(3); + std::vector searchPaths; #ifdef __DEBUG__ if (config.ContainerPath.IsSet() == true) { diff --git a/Source/core/Library.cpp b/Source/core/Library.cpp index 04575b3c7..4541ba3e0 100644 --- a/Source/core/Library.cpp +++ b/Source/core/Library.cpp @@ -114,8 +114,6 @@ namespace Core { _refCountedHandle->_name = GlobalSymbols; TRACE_L1("Loaded library with global symbols of the program"); } - - TRACE_L1("Loaded library: %s", fileName); } else { #ifdef __LINUX__ _error = dlerror(); diff --git a/Source/plugins/Shell.cpp b/Source/plugins/Shell.cpp index edcd0c8b5..9d3c8024f 100644 --- a/Source/plugins/Shell.cpp +++ b/Source/plugins/Shell.cpp @@ -42,8 +42,10 @@ namespace PluginHost result = Core::ServiceAdministrator::Instance().Instantiate(Core::Library(), className.c_str(), version, interface); } else { RPC::IStringIterator* all_paths = GetLibrarySearchPaths(locator); + ASSERT(all_paths != nullptr); + string element; - while (all_paths->Next(element) == true) { + while ((all_paths->Next(element) == true) && (result == nullptr)) { Core::File file(element.c_str()); if (file.Exists()) { From aaf530cfc03fbc7e673bb5c677fb891d7f93efd7 Mon Sep 17 00:00:00 2001 From: Mateusz Daniluk <121170681+VeithMetro@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:43:55 +0200 Subject: [PATCH 06/18] Correct the Detach and Notify order in Server::Service::Deactivate() (#1765) --- Source/Thunder/PluginServer.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/Source/Thunder/PluginServer.cpp b/Source/Thunder/PluginServer.cpp index 6a5a9f406..379ee0a09 100644 --- a/Source/Thunder/PluginServer.cpp +++ b/Source/Thunder/PluginServer.cpp @@ -581,26 +581,28 @@ namespace PluginHost { Lock(); - if (_jsonrpc != nullptr) { - PluginHost::IShell::IConnectionServer::INotification* sink = nullptr; - _jsonrpc->Detach(sink); - if (sink != nullptr) { - Unregister(sink); - sink->Release(); - } + if (currentState != IShell::state::ACTIVATION) { + SYSLOG(Logging::Shutdown, (_T("Deactivated plugin [%s]:[%s]"), className.c_str(), callSign.c_str())); + +#ifdef THUNDER_RESTFULL_API + Notify(EMPTY_STRING, string(_T("{\"state\":\"deactivated\",\"reason\":\"")) + textReason.Data() + _T("\"}")); +#endif + Notify(_T("statechange"), string(_T("{\"state\":\"deactivated\",\"reason\":\"")) + textReason.Data() + _T("\"}")); } if (_external.Connector().empty() == false) { _external.Close(0); } - if (currentState != IShell::state::ACTIVATION) { - SYSLOG(Logging::Shutdown, (_T("Deactivated plugin [%s]:[%s]"), className.c_str(), callSign.c_str())); + if (_jsonrpc != nullptr) { + PluginHost::IShell::IConnectionServer::INotification* sink = nullptr; - #ifdef THUNDER_RESTFULL_API - Notify(EMPTY_STRING, string(_T("{\"state\":\"deactivated\",\"reason\":\"")) + textReason.Data() + _T("\"}")); - #endif - Notify(_T("statechange"), string(_T("{\"state\":\"deactivated\",\"reason\":\"")) + textReason.Data() + _T("\"}")); + _jsonrpc->Detach(sink); + + if (sink != nullptr) { + Unregister(sink); + sink->Release(); + } } } From 0bffff03a769cbb45ff623aa8136ede24acbeea8 Mon Sep 17 00:00:00 2001 From: HaseenaSainul <41037131+HaseenaSainul@users.noreply.github.com> Date: Mon, 30 Sep 2024 20:04:57 +0530 Subject: [PATCH 07/18] SetEnv added for plugin Config as well (#1764) * SetEnv added for plugin Config as well * SetEnv: JSON parameter passing through command is changed * Update based on comments * Update based on testing * Review comments addressed --------- Co-authored-by: Pierre Wielders --- Source/Thunder/Config.h | 64 +-------------- Source/Thunder/IRemoteInstantiation.h | 4 +- Source/Thunder/PluginServer.h | 66 +++++++++++++-- Source/ThunderPlugin/Process.cpp | 21 ++++- Source/com/Communicator.cpp | 8 ++ Source/com/Communicator.h | 36 ++++++++- Source/com/Ids.h | 1 + Source/plugins/Configuration.h | 111 ++++++++++++++++++++++++++ Source/plugins/Shell.cpp | 4 +- 9 files changed, 242 insertions(+), 73 deletions(-) diff --git a/Source/Thunder/Config.h b/Source/Thunder/Config.h index d3fc8f160..9152bdd4b 100644 --- a/Source/Thunder/Config.h +++ b/Source/Thunder/Config.h @@ -114,64 +114,6 @@ namespace PluginHost { // Configuration to get a server (PluginHost server) up and running. class JSONConfig : public Core::JSON::Container { public: - class Environment : public Core::JSON::Container { - public: - Environment() - : Core::JSON::Container() - , Key() - , Value() - , Override(false) - { - Add(_T("key"), &Key); - Add(_T("value"), &Value); - Add(_T("override"), &Override); - } - Environment(const Environment& copy) - : Core::JSON::Container() - , Key(copy.Key) - , Value(copy.Value) - , Override(copy.Override) - { - Add(_T("key"), &Key); - Add(_T("value"), &Value); - Add(_T("override"), &Override); - } - Environment(Environment&& move) noexcept - : Core::JSON::Container() - , Key(std::move(move.Key)) - , Value(std::move(move.Value)) - , Override(std::move(move.Override)) - { - Add(_T("key"), &Key); - Add(_T("value"), &Value); - Add(_T("override"), &Override); - } - ~Environment() override = default; - Environment& operator=(const Environment& RHS) - { - Key = RHS.Key; - Value = RHS.Value; - Override = RHS.Override; - - return (*this); - } - Environment& operator=(Environment&& move) noexcept - { - if (this != &move) { - Key = std::move(move.Key); - Value = std::move(move.Value); - Override = std::move(move.Override); - } - - return (*this); - } - - public: - Core::JSON::String Key; - Core::JSON::String Value; - Core::JSON::Boolean Override; - }; - class ProcessSet : public Core::JSON::Container { public: ProcessSet() @@ -526,7 +468,7 @@ namespace PluginHost { Core::JSON::String Configs; Core::JSON::String EthernetCard; Core::JSON::ArrayType Plugins; - Core::JSON::ArrayType Environments; + Core::JSON::ArrayType Environments; Core::JSON::ArrayType> ExitReasons; Core::JSON::DecSInt32 Latitude; Core::JSON::DecSInt32 Longitude; @@ -773,12 +715,12 @@ namespace PluginHost { } bool status = true; - Core::JSON::ArrayType::ConstIterator index(static_cast(config).Environments.Elements()); + Core::JSON::ArrayType::ConstIterator index(static_cast(config).Environments.Elements()); while (index.Next() == true) { if ((index.Current().Key.IsSet() == true) && (index.Current().Value.IsSet() == true)) { string value = _substituter.Substitute(index.Current().Value.Value(), nullptr); if (value.empty() != true) { - status = Core::SystemInfo::SetEnvironment(index.Current().Key.Value(), value, index.Current().Override.Value()); + status = Core::SystemInfo::SetEnvironment(index.Current().Key.Value(), value, ((index.Current().Override.Value() == RPC::Object::Environment::Scope::GLOBAL) ? true : false)); if (status != true) { SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); } diff --git a/Source/Thunder/IRemoteInstantiation.h b/Source/Thunder/IRemoteInstantiation.h index f3249295b..169bb4a0a 100644 --- a/Source/Thunder/IRemoteInstantiation.h +++ b/Source/Thunder/IRemoteInstantiation.h @@ -27,6 +27,7 @@ namespace PluginHost { struct IRemoteInstantiation : virtual public Core::IUnknown { enum { ID = RPC::IDS::ID_REMOTE_INSTANTIATION }; + using IEnvironmentIterator = RPC::IIteratorType; ~IRemoteInstantiation() override = default; @@ -56,7 +57,8 @@ namespace PluginHost { const string& systemRootPath, const uint8_t threads, const int8_t priority, - const string configuration) = 0; + const string configuration, + IEnvironmentIterator* const& environments) = 0; }; } } diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index 6d1b72ad5..5d6af136a 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -377,6 +377,8 @@ namespace PluginHost { DYNAMIC }; + using Services = std::unordered_map; + private: using BaseClass = PluginHost::Service; using Jobs = Core::ThrottleQueueType, ServiceMap&>; @@ -1277,16 +1279,29 @@ namespace PluginHost { string Substitute(const string& input) const override { return (_administrator.Configuration().Substitute(input, PluginHost::Service::Configuration())); } - Core::hresult Metadata(string& info /* @out */) const override { + Core::hresult Metadata(string& info /* @out */) const override { Metadata::Service result; GetMetadata(result); result.ToString(info); return (Core::ERROR_NONE); } + std::vector& SubstituteList(const std::vector& environmentList) const { + std::vector& environments = const_cast&>(environmentList); + for (auto& environment : environments) { + if ((environment.key.empty() != true) && (environment.value.empty() != true)) { + environment.value = Substitute(environment.value); + } else { + SYSLOG(Logging::Startup, (_T("Failure in Substituting Value of Key:Value:[%s]:[%s]\n"), environment.key.c_str(), environment.value.c_str())); + } + } + return environments; + } void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId) override { ASSERT(_connection == nullptr); + const_cast(object).Environments(SubstituteList(object.Environments())); + void* result(_administrator.Instantiate(object, waitTime, sessionId, DataPath(), PersistentPath(), VolatilePath(), _administrator.Configuration().LinkerPluginPaths())); if (result != nullptr) { @@ -1499,7 +1514,9 @@ namespace PluginHost { else { uint32_t pid; Core::ServiceAdministrator::Instance().ReleaseLibrary(std::move(_library)); - + string environments; + PluginHost::Service::Configuration().Root.Environments.ToString(environments); + RPC::Object definition(locator, classNameString, Callsign(), @@ -1512,7 +1529,8 @@ namespace PluginHost { PluginHost::Service::Configuration().Root.HostType(), SystemRootPath(), PluginHost::Service::Configuration().Root.RemoteAddress.Value(), - PluginHost::Service::Configuration().Root.Configuration.Value()); + PluginHost::Service::Configuration().Root.Configuration.Value(), + Plugin::Config::Environment::List(PluginHost::Service::Configuration().Root.Environments)); newIF = reinterpret_cast(Instantiate(definition, _administrator.Configuration().OutOfProcessWaitTime(), pid)); if (newIF == nullptr) { @@ -2007,6 +2025,10 @@ namespace PluginHost { if (instantiation == nullptr) { result = Core::ERROR_ILLEGAL_STATE; } else { + + using Iterator = IRemoteInstantiation::IEnvironmentIterator; + Iterator* environment = Core::Service>::Create(_object.Environments()); + result = instantiation->Instantiate( RPC::Communicator::RemoteConnection::Id(), _object.Locator(), @@ -2019,7 +2041,8 @@ namespace PluginHost { _object.SystemRootPath(), _object.Threads(), _object.Priority(), - _object.Configuration()); + _object.Configuration(), + environment); instantiation->Release(); } @@ -2399,7 +2422,8 @@ namespace PluginHost { const string& systemRootPath, const uint8_t threads, const int8_t priority, - const string configuration) override + const string configuration, + IRemoteInstantiation::IEnvironmentIterator* const& environments) override { string persistentPath(_comms.PersistentPath()); string dataPath(_comms.DataPath()); @@ -2411,10 +2435,17 @@ namespace PluginHost { volatilePath += callsign + '/'; } + std::vector _environmentList; + if (environments != nullptr) { + RPC::Object::Environment environment; + while (environments->Next(environment) == true) { + _environmentList.push_back(environment); + } + } + uint32_t id; RPC::Config config(_connector, _comms.Application(), persistentPath, _comms.SystemPath(), dataPath, volatilePath, _comms.AppPath(), _comms.ProxyStubPath(), _comms.PostMortemPath(), _comms.LinkerPaths()); - RPC::Object instance(libraryName, className, callsign, interfaceId, version, user, group, threads, priority, RPC::Object::HostType::LOCAL, systemRootPath, _T(""), configuration); - + RPC::Object instance(libraryName, className, callsign, interfaceId, version, user, group, threads, priority, RPC::Object::HostType::LOCAL, systemRootPath, _T(""), configuration, SubstituteList(callsign, _environmentList)); RPC::Communicator::Process process(requestId, config, instance); return (process.Launch(id)); @@ -2424,6 +2455,16 @@ namespace PluginHost { INTERFACE_ENTRY(IRemoteInstantiation) END_INTERFACE_MAP + private: + std::vector& SubstituteList(const string& callsign, std::vector& environments) const + { + Core::ProxyType service = _parent.GetService(callsign); + if (service.IsValid() == true) { + environments = service->SubstituteList(environments); + } + return environments; + } + private: mutable uint32_t _refCount; ServiceMap& _parent; @@ -3064,6 +3105,17 @@ namespace PluginHost { _adminLock.Unlock(); } + inline Core::ProxyType GetService(const string& callsign) + { + Core::ProxyType service; + for (std::pair>& entry : _services) { + if (entry.first == callsign) { + service = entry.second; + break; + } + } + return service; + } inline Iterator Services() { Shells workingList; diff --git a/Source/ThunderPlugin/Process.cpp b/Source/ThunderPlugin/Process.cpp index df3d64432..e7cab38c8 100644 --- a/Source/ThunderPlugin/Process.cpp +++ b/Source/ThunderPlugin/Process.cpp @@ -159,7 +159,7 @@ POP_WARNING() class ConsoleOptions : public Core::Options { public: ConsoleOptions(int argumentCount, TCHAR* arguments[]) - : Core::Options(argumentCount, arguments, _T("h:l:c:C:r:p:s:d:a:m:i:u:g:t:e:x:V:v:P:S:f:")) + : Core::Options(argumentCount, arguments, _T("h:l:c:C:r:p:s:d:a:m:i:u:g:t:e:E:x:V:v:P:S:f:")) , Locator(nullptr) , ClassName(nullptr) , Callsign(nullptr) @@ -202,6 +202,7 @@ POP_WARNING() string ProxyStubPath; string PostMortemPath; string SystemRootPath; + std::vector Environments; const TCHAR* User; const TCHAR* Group; uint8_t Threads; @@ -250,6 +251,13 @@ POP_WARNING() case 'S': SystemRootPath = Core::Directory::Normalize(Strip(argument)); break; + case 'e': + case 'E': { + Environments.push_back(Plugin::Config::Environment::Info(Strip(argument), + ((option == 'E') ? RPC::Object::Environment::Scope::GLOBAL : + RPC::Object::Environment::Scope::LOCAL))); + break; + } case 'v': VolatilePath = Core::Directory::Normalize(Strip(argument)); break; @@ -588,6 +596,8 @@ int main(int argc, char** argv) printf(" [-d ]\n"); printf(" [-v ]\n"); printf(" [-f ...\n"); + printf(" [-e/-E ...]\n"); + printf(" e: means set as local scope, E: means set as global scope\n"); printf(" [-a ]\n"); printf(" [-m ]\n"); printf(" [-P ]\n\n"); @@ -664,6 +674,15 @@ int main(int argc, char** argv) Core::ProcessCurrent().User(string(options.User)); } + for (const auto& info : options.Environments) { + if ((info.key.empty() != true) && (info.value.empty() != true)) { + uint32_t status = Core::SystemInfo::SetEnvironment(info.key, info.value.c_str(), ((info.overriding == RPC::Object::Environment::Scope::GLOBAL) ? true : false)); + if (status != true) { + SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s]\n"), info.key.c_str(), info.value.c_str())); + } + } + } + process.Startup(options.Threads, remoteNode, callsign); // Register an interface to handle incoming requests for interfaces. diff --git a/Source/com/Communicator.cpp b/Source/com/Communicator.cpp index 266c64772..e8efcac3d 100644 --- a/Source/com/Communicator.cpp +++ b/Source/com/Communicator.cpp @@ -575,4 +575,12 @@ namespace RPC { constexpr uint32_t RPC::ProcessShutdown::DestructionStackSize; } + +ENUM_CONVERSION_BEGIN(RPC::Object::Environment::Scope) + + { RPC::Object::Environment::Scope::LOCAL, _TXT("Local") }, + { RPC::Object::Environment::Scope::GLOBAL, _TXT("Global") }, + +ENUM_CONVERSION_END(RPC::Object::Environment::Scope) + } diff --git a/Source/com/Communicator.h b/Source/com/Communicator.h index 054a8644b..085e3b237 100644 --- a/Source/com/Communicator.h +++ b/Source/com/Communicator.h @@ -45,12 +45,23 @@ namespace RPC { class EXTERNAL Object { public: + static constexpr const TCHAR EnvironmentSeparator = _T('='); enum class HostType { LOCAL, DISTRIBUTED, CONTAINER }; + struct Environment { + enum class Scope { + LOCAL, + GLOBAL + }; + string key; + string value; + Scope overriding; + }; + Object() : _locator() , _className() @@ -65,6 +76,7 @@ namespace RPC { , _systemRootPath() , _remoteAddress() , _configuration() + , _environments() { } Object(const Object& copy) @@ -81,6 +93,7 @@ namespace RPC { , _systemRootPath(copy._systemRootPath) , _remoteAddress(copy._remoteAddress) , _configuration(copy._configuration) + , _environments(copy._environments) { } Object(Object&& move) noexcept @@ -97,6 +110,7 @@ namespace RPC { , _systemRootPath(std::move(move._systemRootPath)) , _remoteAddress(std::move(move._remoteAddress)) , _configuration(std::move(move._configuration)) + , _environments(std::move(move._environments)) { } Object(const string& locator, @@ -111,7 +125,8 @@ namespace RPC { const HostType type, const string& systemRootPath, const string& remoteAddress, - const string& configuration) + const string& configuration, + const std::vector& environments) : _locator(locator) , _className(className) , _callsign(callsign) @@ -125,6 +140,7 @@ namespace RPC { , _systemRootPath(systemRootPath) , _remoteAddress(remoteAddress) , _configuration(configuration) + , _environments(environments) { } ~Object() @@ -146,6 +162,7 @@ namespace RPC { _type = RHS._type; _remoteAddress = RHS._remoteAddress; _configuration = RHS._configuration; + _environments = RHS._environments; return (*this); } @@ -166,6 +183,7 @@ namespace RPC { _systemRootPath = std::move(move._systemRootPath); _remoteAddress = std::move(move._remoteAddress); _configuration = std::move(move._configuration); + _environments = std::move(move._environments); move._interface = ~0; move._version = ~0; @@ -228,6 +246,13 @@ namespace RPC { { return (_configuration); } + inline const std::vector& Environments() const + { + return (_environments); + } + inline void Environments(const std::vector& environments) { + _environments = std::move(environments); + } private: string _locator; @@ -243,6 +268,7 @@ namespace RPC { string _systemRootPath; string _remoteAddress; string _configuration; + std::vector _environments; }; class EXTERNAL Config { @@ -300,7 +326,7 @@ namespace RPC { , _linker(copy._linker) { } - Config(Config&& move) noexcept + Config(Config&& move) noexcept : _connector(std::move(move._connector)) , _hostApplication(std::move(move._hostApplication)) , _persistent(std::move(move._persistent)) @@ -515,6 +541,12 @@ namespace RPC { if (instance.Threads() > 1) { _options.Add(_T("-t")).Add(Core::NumberType(instance.Threads()).Text()); } + for (auto const& environment : instance.Environments()) { + string env = environment.key + RPC::Object::EnvironmentSeparator + + "\"" + environment.value + "\""; + string option = (environment.overriding == RPC::Object::Environment::Scope::GLOBAL) ? "-E" : "-e"; + _options.Add(_T(option)).Add('"' + env + '"'); + } _priority = instance.Priority(); } const string& Command() const diff --git a/Source/com/Ids.h b/Source/com/Ids.h index 9c387bdbb..6c1a3b07d 100644 --- a/Source/com/Ids.h +++ b/Source/com/Ids.h @@ -89,6 +89,7 @@ namespace RPC { ID_SUBSYSTEM_DECRYPTION = (ID_OFFSET_INTERNAL + 0x003E), ID_REMOTE_INSTANTIATION = (ID_OFFSET_INTERNAL + 0x003F), ID_SYSTEM_METADATA = (ID_OFFSET_INTERNAL + 0x0040), + ID_ENVIRONMENT_ITERATOR = (ID_OFFSET_INTERNAL + 0x0041), ID_EXTERNAL_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0x0080), ID_EXTERNAL_QA_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0xA000) diff --git a/Source/plugins/Configuration.h b/Source/plugins/Configuration.h index 3bbf40d04..325f6ed6d 100644 --- a/Source/plugins/Configuration.h +++ b/Source/plugins/Configuration.h @@ -34,6 +34,106 @@ namespace Plugin { */ class EXTERNAL Config : public Core::JSON::Container { public: + class Environment : public Core::JSON::Container { + public: + Environment() + : Core::JSON::Container() + , Key() + , Value() + , Override(RPC::Object::Environment::Scope::LOCAL) + { + Add(_T("key"), &Key); + Add(_T("value"), &Value); + Add(_T("override"), &Override); + } + Environment(const Environment& copy) + : Core::JSON::Container() + , Key(copy.Key) + , Value(copy.Value) + , Override(copy.Override) + { + Add(_T("key"), &Key); + Add(_T("value"), &Value); + Add(_T("override"), &Override); + } + Environment(Environment&& move) noexcept + : Core::JSON::Container() + , Key(std::move(move.Key)) + , Value(std::move(move.Value)) + , Override(std::move(move.Override)) + { + Add(_T("key"), &Key); + Add(_T("value"), &Value); + Add(_T("override"), &Override); + } + ~Environment() override = default; + Environment& operator=(const Environment& RHS) + { + Key = RHS.Key; + Value = RHS.Value; + Override = RHS.Override; + + return (*this); + } + Environment& operator=(Environment&& move) noexcept + { + if (this != &move) { + Key = std::move(move.Key); + Value = std::move(move.Value); + Override = std::move(move.Override); + } + + return (*this); + } + static std::vector List(const Core::JSON::ArrayType& environments) + { + std::vector environmentList; + + if (environments.IsSet() == true) { + Core::JSON::ArrayType::ConstIterator index(environments.Elements()); + while (index.Next() == true) { + if ((index.Current().Key.IsSet() == true) && (index.Current().Value.IsSet() == true)) { + RPC::Object::Environment env; + env.key = index.Current().Key.Value(); + env.value = index.Current().Value.Value(); + env.overriding = index.Current().Override.Value(); + environmentList.push_back(env); + } else { + SYSLOG(Logging::Startup, (_T("Failure in Substituting Value of Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); + } + } + } + return environmentList; + } + static RPC::Object::Environment Info(const string& env, RPC::Object::Environment::Scope overriding) + { + RPC::Object::Environment info; + size_t start = env.find_first_of(RPC::Object::EnvironmentSeparator); + if ((start != string::npos) && (start < env.length())) { + string key = env.substr(0, start); + string value = env.substr(start + 1); + + if ((key.empty() != true) && (value.empty() != true) && (value.length() > 2) && + ((value.at(0) == '\"') && (value.at(value.length() - 1) == '\"'))) { + info.key = key; + info.value = value.substr(1, value.length() - 2); + info.overriding = overriding; + } else { + SYSLOG(Logging::Startup, (_T("Environment key:value fromat is invalid :[%s]:[%s]\n"), key.c_str(), value.c_str())); + } + } else { + SYSLOG(Logging::Startup, (_T("Invalid Enviroment value :[%s]\n"), env.c_str())); + } + return info; + } + + public: + Core::JSON::String Key; + Core::JSON::String Value; + Core::JSON::EnumType Override; + }; + using EnvironmentList = Core::JSON::ArrayType; + class EXTERNAL RootConfig : public Core::JSON::Container { private: class RootObject : public Core::JSON::Container { @@ -73,6 +173,7 @@ namespace Plugin { , Mode(ModeType::LOCAL) , RemoteAddress() , Configuration(false) + , Environments() { Add(_T("locator"), &Locator); Add(_T("user"), &User); @@ -83,6 +184,7 @@ namespace Plugin { Add(_T("mode"), &Mode); Add(_T("remoteaddress"), &RemoteAddress); Add(_T("configuration"), &Configuration); + Add(_T("environments"), &Environments); } RootConfig(const PluginHost::IShell* info) : Core::JSON::Container() @@ -95,6 +197,7 @@ namespace Plugin { , Mode(ModeType::LOCAL) , RemoteAddress() , Configuration(false) + , Environments() { Add(_T("locator"), &Locator); Add(_T("user"), &User); @@ -105,6 +208,7 @@ namespace Plugin { Add(_T("mode"), &Mode); Add(_T("remoteaddress"), &RemoteAddress); Add(_T("configuration"), &Configuration); + Add(_T("environments"), &Environments); RootObject config; Core::OptionalType error; @@ -139,6 +243,7 @@ namespace Plugin { , Mode(copy.Mode) , RemoteAddress(copy.RemoteAddress) , Configuration(copy.Configuration) + , Environments(copy.Environments) { Add(_T("locator"), &Locator); Add(_T("user"), &User); @@ -149,6 +254,7 @@ namespace Plugin { Add(_T("mode"), &Mode); Add(_T("remoteaddress"), &RemoteAddress); Add(_T("configuration"), &Configuration); + Add(_T("environments"), &Environments); } RootConfig(RootConfig&& move) noexcept : Core::JSON::Container() @@ -161,6 +267,7 @@ namespace Plugin { , Mode(std::move(move.Mode)) , RemoteAddress(std::move(move.RemoteAddress)) , Configuration(std::move(move.Configuration)) + , Environments(std::move(move.Environments)) { Add(_T("locator"), &Locator); Add(_T("user"), &User); @@ -171,6 +278,7 @@ namespace Plugin { Add(_T("mode"), &Mode); Add(_T("remoteaddress"), &RemoteAddress); Add(_T("configuration"), &Configuration); + Add(_T("environments"), &Environments); } ~RootConfig() override = default; @@ -186,6 +294,7 @@ namespace Plugin { Mode = RHS.Mode; RemoteAddress = RHS.RemoteAddress; Configuration = RHS.Configuration; + Environments = RHS.Environments; return (*this); } @@ -202,6 +311,7 @@ namespace Plugin { Mode = std::move(move.Mode); RemoteAddress = std::move(move.RemoteAddress); Configuration = std::move(move.Configuration); + Environments = std::move(move.Environments); } return (*this); @@ -233,6 +343,7 @@ namespace Plugin { Core::JSON::EnumType Mode; Core::JSON::String RemoteAddress; Core::JSON::String Configuration; + EnvironmentList Environments; }; public: diff --git a/Source/plugins/Shell.cpp b/Source/plugins/Shell.cpp index 9d3c8024f..e8d052665 100644 --- a/Source/plugins/Shell.cpp +++ b/Source/plugins/Shell.cpp @@ -50,6 +50,7 @@ namespace PluginHost if (file.Exists()) { Core::Library resource = Core::ServiceAdministrator::Instance().LoadLibrary(element.c_str()); + if (resource.IsLoaded()) result = Core::ServiceAdministrator::Instance().Instantiate( resource, @@ -80,7 +81,8 @@ namespace PluginHost rootConfig.HostType(), SystemRootPath(), rootConfig.RemoteAddress.Value(), - rootConfig.Configuration.Value()); + rootConfig.Configuration.Value(), + Plugin::Config::Environment::List(rootConfig.Environments)); result = handler->Instantiate(definition, waitTime, pid); } From a015de22dcd5681be77220d2ec0ac0cbaa667af1 Mon Sep 17 00:00:00 2001 From: nxtum <94901881+nxtum@users.noreply.github.com> Date: Tue, 1 Oct 2024 09:18:39 +0200 Subject: [PATCH 08/18] Expose proxies by default (#1763) * expose all proxies * formatting * replace pushback with insert * use .end() * keep arrow tight * return pod of data * use visitor way * remove unwanted struct --------- Co-authored-by: Pierre Wielders --- Source/Thunder/Controller.cpp | 28 +++++++++++----------------- Source/Thunder/PluginHost.cpp | 8 +++----- Source/com/Administrator.h | 20 +++++++++++++------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Source/Thunder/Controller.cpp b/Source/Thunder/Controller.cpp index e41ba97d7..f0683b703 100644 --- a/Source/Thunder/Controller.cpp +++ b/Source/Thunder/Controller.cpp @@ -1220,31 +1220,25 @@ namespace Plugin { { Core::hresult result = Core::ERROR_UNKNOWN_KEY; - RPC::Administrator::Proxies collection; - - // Search for the Dangling proxies - if (RPC::Administrator::Instance().Allocations(linkId, collection) == true) { - - using Iterator = IMetadata::Data::IProxiesIterator; - - std::list< IMetadata::Data::Proxy> elements; - - for (const ProxyStub::UnknownProxy* proxy : collection) { + std::vector collection; + bool proxySearch = RPC::Administrator::Instance().Allocations(linkId, [&collection](const std::vector& proxies) { + for (const auto& proxy : proxies) { IMetadata::Data::Proxy data; + data.Count = proxy->ReferenceCount(); data.Instance = proxy->Implementation(); data.Interface = proxy->InterfaceId(); - data.Count = proxy->ReferenceCount(); data.Name = proxy->Name(); - - elements.emplace_back(std::move(data)); - } + collection.emplace_back(std::move(data)); + } + }); - outProxies = Core::ServiceType>::Create(std::move(elements)); - ASSERT(outProxies != nullptr); + if (proxySearch == true) { + using Iterator = IMetadata::Data::IProxiesIterator; + outProxies = Core::ServiceType>::Create(std::move(collection)); + ASSERT(outProxies != nullptr); result = Core::ERROR_NONE; } - return (result); } diff --git a/Source/Thunder/PluginHost.cpp b/Source/Thunder/PluginHost.cpp index 307589c5e..be22a0b7a 100644 --- a/Source/Thunder/PluginHost.cpp +++ b/Source/Thunder/PluginHost.cpp @@ -712,15 +712,13 @@ POP_WARNING() printf("Link: %s\n", index.Current().Remote.Value().c_str()); printf("------------------------------------------------------------\n"); - RPC::Administrator::Proxies proxies; - - RPC::Administrator::Instance().Allocations(index.Current().ID.Value(), proxies); - - for (const ProxyStub::UnknownProxy* proxy : proxies) { + RPC::Administrator::Instance().Allocations(index.Current().ID.Value(), [](const std::vector& proxies) { + for (const auto& proxy: proxies) { Core::instance_id instanceId = proxy->Implementation(); printf("[%s] InstanceId: 0x%" PRIx64 ", RefCount: %d, InterfaceId %d [0x%X]\n", proxy->Name().c_str(), static_cast(instanceId), proxy->ReferenceCount(), proxy->InterfaceId(), proxy->InterfaceId()); } printf("\n"); + }); } } break; diff --git a/Source/com/Administrator.h b/Source/com/Administrator.h index a378c6244..95b5eeea0 100644 --- a/Source/com/Administrator.h +++ b/Source/com/Administrator.h @@ -147,28 +147,34 @@ namespace RPC { void DelegatedReleases(const bool enabled) { _delegatedReleases = enabled; } - bool Allocations(const uint32_t id, Proxies& proxies) const { + + template + bool Allocations(const uint32_t id, ACTION&& action) const { bool found = false; + _adminLock.Lock(); if (id == 0) { + for (const auto& proxy : _channelProxyMap) { + action(proxy.second); + } + action(_danglingProxies); found = true; - proxies = _danglingProxies; - } + } else { ChannelMap::const_iterator index(_channelProxyMap.begin()); - while ((found == false) && (index != _channelProxyMap.end())) { - if (index->first != id) { index++; } else { found = true; - proxies = index->second; + action(index->second); } } } - return (found); + _adminLock.Unlock(); + return found; } + template void Announce() { From 79e4ee05ec0f4e80bb93a56ba20f4adb530e6f57 Mon Sep 17 00:00:00 2001 From: Pierre Wielders Date: Wed, 2 Oct 2024 20:14:47 +0200 Subject: [PATCH 09/18] Development/distributed com (#1767) * [REMOTEHOST] Enable and Test the COMRPC off host. Phase 1: Start *moving* * [DISTRIBUTED] Aligning the code to the latest state of the PluginServer. --- Source/Thunder/Config.h | 22 +- Source/Thunder/IRemoteInstantiation.h | 5 +- Source/Thunder/PluginServer.h | 372 ++++++++---------- Source/ThunderPlugin/Process.cpp | 42 +- Source/com/CMakeLists.txt | 2 + Source/com/Communicator.cpp | 8 +- Source/com/Communicator.h | 165 ++++++-- Source/com/ICOM.h | 13 + Source/com/Ids.h | 113 +++--- .../localtracer/example/CMakeLists.txt | 2 + .../privilegedrequest/example/CMakeLists.txt | 2 + Source/plugins/Configuration.h | 75 ++-- Source/plugins/IController.h | 2 +- Source/plugins/IShell.h | 2 +- Source/plugins/Shell.cpp | 2 +- 15 files changed, 448 insertions(+), 379 deletions(-) diff --git a/Source/Thunder/Config.h b/Source/Thunder/Config.h index 9152bdd4b..545d57008 100644 --- a/Source/Thunder/Config.h +++ b/Source/Thunder/Config.h @@ -596,7 +596,7 @@ namespace PluginHost { Config& operator=(const Config&) = delete; PUSH_WARNING(DISABLE_WARNING_THIS_IN_MEMBER_INITIALIZER_LIST) - Config(Core::File& file, const bool background, Core::OptionalType& error) + Config(Core::File& file, const bool background, Core::OptionalType& error) : _background(background) , _prefix() , _webPrefix() @@ -714,18 +714,15 @@ namespace PluginHost { } } - bool status = true; Core::JSON::ArrayType::ConstIterator index(static_cast(config).Environments.Elements()); while (index.Next() == true) { - if ((index.Current().Key.IsSet() == true) && (index.Current().Value.IsSet() == true)) { + if (index.Current().Key.IsSet() == false) { + SYSLOG(Logging::Startup, (_T("Failure n setting an environmet variable. Empty key defined!!\n"))); + } + else { string value = _substituter.Substitute(index.Current().Value.Value(), nullptr); - if (value.empty() != true) { - status = Core::SystemInfo::SetEnvironment(index.Current().Key.Value(), value, ((index.Current().Override.Value() == RPC::Object::Environment::Scope::GLOBAL) ? true : false)); - if (status != true) { - SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); - } - } else { - SYSLOG(Logging::Startup, (_T("Failure in Substituting Value of Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); + if (Core::SystemInfo::SetEnvironment(index.Current().Key.Value(), value, ((index.Current().Scope.Value() == RPC::Environment::scope::GLOBAL) ? true : false)) != true) { + SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); } } } @@ -736,11 +733,12 @@ namespace PluginHost { _plugins = config.Plugins; Core::JSON::ArrayType::Iterator itr(config.LinkerPluginPaths.Elements()); - while (itr.Next()) + while (itr.Next() == true) { _linkerPluginPaths.push_back(itr.Current().Value()); + } } } -POP_WARNING() + POP_WARNING() ~Config() { ASSERT(_security != nullptr); diff --git a/Source/Thunder/IRemoteInstantiation.h b/Source/Thunder/IRemoteInstantiation.h index 169bb4a0a..831eeb6ca 100644 --- a/Source/Thunder/IRemoteInstantiation.h +++ b/Source/Thunder/IRemoteInstantiation.h @@ -27,7 +27,6 @@ namespace PluginHost { struct IRemoteInstantiation : virtual public Core::IUnknown { enum { ID = RPC::IDS::ID_REMOTE_INSTANTIATION }; - using IEnvironmentIterator = RPC::IIteratorType; ~IRemoteInstantiation() override = default; @@ -58,7 +57,9 @@ namespace PluginHost { const uint8_t threads, const int8_t priority, const string configuration, - IEnvironmentIterator* const& environments) = 0; + RPC::IEnvironmentIterator* const& environments) = 0; + + virtual uint32_t Kill (const uint32_t requestId) = 0; }; } } diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index 5d6af136a..df783746f 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -402,7 +402,7 @@ namespace PluginHost { } public: - void AquireInterfaces(IPlugin* plugin) { + void AcquireInterfaces(IPlugin* plugin) { _composit = plugin->QueryInterface(); if (_composit != nullptr) { // Seems this is a plugin that can be a composition of more plugins.. @@ -1285,23 +1285,10 @@ namespace PluginHost { result.ToString(info); return (Core::ERROR_NONE); } - std::vector& SubstituteList(const std::vector& environmentList) const { - std::vector& environments = const_cast&>(environmentList); - for (auto& environment : environments) { - if ((environment.key.empty() != true) && (environment.value.empty() != true)) { - environment.value = Substitute(environment.value); - } else { - SYSLOG(Logging::Startup, (_T("Failure in Substituting Value of Key:Value:[%s]:[%s]\n"), environment.key.c_str(), environment.value.c_str())); - } - } - return environments; - } - void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId) override + void* Instantiate(RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId) override { ASSERT(_connection == nullptr); - const_cast(object).Environments(SubstituteList(object.Environments())); - void* result(_administrator.Instantiate(object, waitTime, sessionId, DataPath(), PersistentPath(), VolatilePath(), _administrator.Configuration().LinkerPluginPaths())); if (result != nullptr) { @@ -1514,8 +1501,6 @@ namespace PluginHost { else { uint32_t pid; Core::ServiceAdministrator::Instance().ReleaseLibrary(std::move(_library)); - string environments; - PluginHost::Service::Configuration().Root.Environments.ToString(environments); RPC::Object definition(locator, classNameString, @@ -1530,7 +1515,7 @@ namespace PluginHost { SystemRootPath(), PluginHost::Service::Configuration().Root.RemoteAddress.Value(), PluginHost::Service::Configuration().Root.Configuration.Value(), - Plugin::Config::Environment::List(PluginHost::Service::Configuration().Root.Environments)); + PluginHost::Service::Configuration().Root.Environment()); newIF = reinterpret_cast(Instantiate(definition, _administrator.Configuration().OutOfProcessWaitTime(), pid)); if (newIF == nullptr) { @@ -1549,7 +1534,7 @@ namespace PluginHost { _webSecurity = newIF->QueryInterface(); _jsonrpc = newIF->QueryInterface(); - _composit.AquireInterfaces(newIF); + _composit.AcquireInterfaces(newIF); if (_webSecurity == nullptr) { _webSecurity = _administrator.Configuration().Security(); _webSecurity->AddRef(); @@ -1989,72 +1974,176 @@ namespace PluginHost { using Proxy = std::pair; using Proxies = std::vector; - class RemoteHost : public RPC::Communicator::RemoteConnection { + class DistributedServer { private: - friend Core::ServiceType; + // This class keeps track of a plugin instantiated on a different Host. It is a connection that + // will not pass Messaging information to this instance of Thunder. The plugin will connect to + // This instance directly. + class RemoteLink : public RPC::Communicator::RemoteConnection { + public: + RemoteLink() = delete; + RemoteLink(RemoteLink&&) = delete; + RemoteLink(const RemoteLink&) = delete; + RemoteLink& operator=(RemoteLink&&) = delete; + RemoteLink& operator=(const RemoteLink&) = delete; + + RemoteLink(DistributedServer& parent, const RPC::Object& instance) + : RemoteConnection() + , _parent(parent) + , _object(instance) { + } + ~RemoteLink() override = default; + public: + uint32_t Launch() override + { + return (_parent.Launch(RPC::Communicator::RemoteConnection::Id(), _object)); + } + + private: + DistributedServer& _parent; + const RPC::Object _object; + }; + public: - RemoteHost(RemoteHost&&) = delete; - RemoteHost(const RemoteHost&) = delete; - RemoteHost& operator=(RemoteHost&&) = delete; - RemoteHost& operator=(const RemoteHost&) = delete; - - RemoteHost(const RPC::Object& instance, const RPC::Config& config) - : RemoteConnection() - , _object(instance) - , _config(config) - { + DistributedServer(const Core::NodeId& remoteConnection) + : _client(Core::ProxyType::Create(remoteConnection, Core::ProxyType(RPC::WorkerPoolInvokeServer()))) + , _remoteServer(_client->Open(_T("DistributedServer"), ~0, 3000)) { + } + ~DistributedServer() { + if (_remoteServer != nullptr) { + _remoteServer->Release(); + } + _client->Close(Core::infinite); } - ~RemoteHost() override = default; public: - uint32_t Launch() override + RPC::Communicator::RemoteConnection* Client (const RPC::Object& instance) { + // Core::ProxyType result = _clients.Instance(*this, instance); + // result->AddRef(); + // return (result.operator->()); + return (Core::ServiceType::Create(*this, instance)); + } + + private: + uint32_t Launch(const uint32_t id, const RPC::Object& instance) { uint32_t result = Core::ERROR_INVALID_DESIGNATOR; - Core::NodeId remoteNode(_object.RemoteAddress()); - - if (remoteNode.IsValid() == true) { - Core::ProxyType engine(Core::ProxyType::Create(&Core::WorkerPool::Instance())); - Core::ProxyType client( - Core::ProxyType::Create(remoteNode, engine)); - - // Oke we have ou selves a COMClient link. Lets see if we can get the proepr interface... - IRemoteInstantiation* instantiation = client->Open(_config.Connector(), ~0, 3000); - - if (instantiation == nullptr) { - result = Core::ERROR_ILLEGAL_STATE; - } else { - - using Iterator = IRemoteInstantiation::IEnvironmentIterator; - Iterator* environment = Core::Service>::Create(_object.Environments()); - - result = instantiation->Instantiate( - RPC::Communicator::RemoteConnection::Id(), - _object.Locator(), - _object.ClassName(), - _object.Callsign(), - _object.Interface(), - _object.Version(), - _object.User(), - _object.Group(), - _object.SystemRootPath(), - _object.Threads(), - _object.Priority(), - _object.Configuration(), - environment); - - instantiation->Release(); + if (_remoteServer == nullptr) { + result = Core::ERROR_ILLEGAL_STATE; + } else { + RPC::IEnvironmentIterator* environment = Core::Service::Create(instance.Environments()); + + result = _remoteServer->Instantiate( + id, + instance.Locator(), + instance.ClassName(), + instance.Callsign(), + instance.Interface(), + instance.Version(), + instance.User(), + instance.Group(), + instance.SystemRootPath(), + instance.Threads(), + instance.Priority(), + instance.Configuration(), + environment); } + + return (result); } + private: + Core::ProxyType _client; + IRemoteInstantiation* _remoteServer; + }; + + // This class handles incoming request to instantiate plugins that do *not* have any configuration + // here but will be insatntiated on request from a different Thunder plugin somewhere else. This is the + // class that gets called by the RemoteHost instance above! + class HostingServer : public IRemoteInstantiation { + public: + HostingServer(HostingServer&&) = delete; + HostingServer(const HostingServer&) = delete; + HostingServer& operator=(HostingServer&&) = delete; + HostingServer& operator=(const HostingServer&) = delete; + + HostingServer(ServiceMap& parent, const CommunicatorServer& comms, const string& connector) + : _parent(parent) + , _comms(comms) + , _connector(connector) + { + } + ~HostingServer() override = default; + + public: + uint32_t Instantiate( + const uint32_t requestId, + const string& libraryName, + const string& className, + const string& callsign, + const uint32_t interfaceId, + const uint32_t version, + const string& user, + const string& group, + const string& systemRootPath, + const uint8_t threads, + const int8_t priority, + const string configuration, + RPC::IEnvironmentIterator* const& environments) override + { + uint32_t result = Core::ERROR_BAD_REQUEST; + + if (callsign.empty() == false) { + Core::ProxyType service = _parent.GetService(callsign); + + if (service.IsValid() == false) { + result = Core::ERROR_UNKNOWN_KEY; + } + else { + std::vector environmentList; + string persistentPath(_comms.PersistentPath()); + string dataPath(_comms.DataPath()); + string volatilePath(_comms.VolatilePath()); + + dataPath += callsign + '/'; + persistentPath += callsign + '/'; + volatilePath += callsign + '/'; + + if (environments != nullptr) { + RPC::Environment data; + while (environments->Next(data) == true) { + ASSERT (data.Key.empty() == false); + data.Value = service->Substitute(data.Value); + environmentList.emplace_back(data); + } + } + + uint32_t id; + RPC::Config config(_connector, _comms.Application(), persistentPath, _comms.SystemPath(), dataPath, volatilePath, _comms.AppPath(), _comms.ProxyStubPath(), _comms.PostMortemPath(), _comms.LinkerPaths()); + RPC::Object instance(libraryName, className, callsign, interfaceId, version, user, group, threads, priority, RPC::Object::HostType::LOCAL, systemRootPath, _T(""), configuration, std::move(environmentList)); + RPC::Communicator::Process process(requestId, config, instance); + + result = process.Launch(id); + } + } return (result); } + uint32_t Kill (const uint32_t /* requestId */) override { + return (Core::ERROR_NONE); + } + + BEGIN_INTERFACE_MAP(HostingServer) + INTERFACE_ENTRY(IRemoteInstantiation) + END_INTERFACE_MAP private: - RPC::Object _object; - const RPC::Config& _config; + ServiceMap& _parent; + const CommunicatorServer& _comms; + const string _connector; }; + class ProxyStubObserver : public Core::FileSystemMonitor::ICallback { public: ProxyStubObserver() = delete; @@ -2132,6 +2221,8 @@ namespace PluginHost { , _requestObservers() , _proxyStubObserver(*this, observableProxyStubPath) , _deadProxies() + , _hostingServer(parent, *this, _T("ToBeFilledIn")) + , _distributedServers() , _job(*this) { // Shall we enable the non-happy day functionality to cleanup Release on behalf of unexpected @@ -2254,6 +2345,9 @@ namespace PluginHost { _adminLock.Unlock(); } } + IRemoteInstantiation* Hosting() { + return (reinterpret_cast(_hostingServer.QueryInterface(IRemoteInstantiation::ID))); + } void Dispatch() { // Oke time to notify the destruction of some proxies... _adminLock.Lock(); @@ -2338,7 +2432,13 @@ namespace PluginHost { RPC::Communicator::RemoteConnection* result = nullptr; if (instance.Type() == RPC::Object::HostType::DISTRIBUTED) { - result = Core::ServiceType::Create(instance, config); + Core::NodeId remoteNode(instance.RemoteAddress()); + if (remoteNode.IsValid() == true) { + Core::ProxyType connector = _distributedServers.Instance(remoteNode, remoteNode); + if (connector.IsValid() == true) { + result = connector->Client(instance); + } + } } else { result = RPC::Communicator::CreateStarter(config, instance); @@ -2362,115 +2462,10 @@ namespace PluginHost { Observers _requestObservers; ProxyStubObserver _proxyStubObserver; Proxies _deadProxies; + Core::SinkType _hostingServer; + Core::ProxyMapType _distributedServers; Core::WorkerPool::JobType _job; }; - class RemoteInstantiation : public IRemoteInstantiation { - private: - RemoteInstantiation(ServiceMap& parent, const CommunicatorServer& comms, const string& connector) - : _refCount(1) - , _parent(parent) - , _comms(comms) - , _connector(connector) - { - } - - public: - RemoteInstantiation(RemoteInstantiation&&) = delete; - RemoteInstantiation(const RemoteInstantiation&) = delete; - RemoteInstantiation& operator=(RemoteInstantiation&&) = delete; - RemoteInstantiation& operator=(const RemoteInstantiation&) = delete; - - ~RemoteInstantiation() override = default; - - public: - static IRemoteInstantiation* Create(ServiceMap& parent, const CommunicatorServer& comms, const string& connector) - { - return (new RemoteInstantiation(parent, comms, connector)); - } - uint32_t AddRef() const override - { - Core::InterlockedIncrement(_refCount); - return (Core::ERROR_NONE); - } - uint32_t Release() const override - { - _parent._adminLock.Lock(); - - if (Core::InterlockedDecrement(_refCount) == 0) { - _parent.Remove(_connector); - - _parent._adminLock.Unlock(); - - delete this; - - return (Core::ERROR_DESTRUCTION_SUCCEEDED); - } else { - _parent._adminLock.Unlock(); - } - - return (Core::ERROR_NONE); - } - uint32_t Instantiate( - const uint32_t requestId, - const string& libraryName, - const string& className, - const string& callsign, - const uint32_t interfaceId, - const uint32_t version, - const string& user, - const string& group, - const string& systemRootPath, - const uint8_t threads, - const int8_t priority, - const string configuration, - IRemoteInstantiation::IEnvironmentIterator* const& environments) override - { - string persistentPath(_comms.PersistentPath()); - string dataPath(_comms.DataPath()); - string volatilePath(_comms.VolatilePath()); - - if (callsign.empty() == false) { - dataPath += callsign + '/'; - persistentPath += callsign + '/'; - volatilePath += callsign + '/'; - } - - std::vector _environmentList; - if (environments != nullptr) { - RPC::Object::Environment environment; - while (environments->Next(environment) == true) { - _environmentList.push_back(environment); - } - } - - uint32_t id; - RPC::Config config(_connector, _comms.Application(), persistentPath, _comms.SystemPath(), dataPath, volatilePath, _comms.AppPath(), _comms.ProxyStubPath(), _comms.PostMortemPath(), _comms.LinkerPaths()); - RPC::Object instance(libraryName, className, callsign, interfaceId, version, user, group, threads, priority, RPC::Object::HostType::LOCAL, systemRootPath, _T(""), configuration, SubstituteList(callsign, _environmentList)); - RPC::Communicator::Process process(requestId, config, instance); - - return (process.Launch(id)); - } - - BEGIN_INTERFACE_MAP(RemoteInstantiation) - INTERFACE_ENTRY(IRemoteInstantiation) - END_INTERFACE_MAP - - private: - std::vector& SubstituteList(const string& callsign, std::vector& environments) const - { - Core::ProxyType service = _parent.GetService(callsign); - if (service.IsValid() == true) { - environments = service->SubstituteList(environments); - } - return environments; - } - - private: - mutable uint32_t _refCount; - ServiceMap& _parent; - const CommunicatorServer& _comms; - const string _connector; - }; class SubSystems : public Core::IDispatch, public SystemInfo { private: class Job { @@ -2944,7 +2939,7 @@ namespace PluginHost { return (result); } - void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId, const string& dataPath, const string& persistentPath, const string& volatilePath, const std::vector& linkerPaths) + void* Instantiate(RPC::Object& object, const uint32_t waitTime, uint32_t& sessionId, const string& dataPath, const string& persistentPath, const string& volatilePath, const std::vector& linkerPaths) { return (_processAdministrator.Create(sessionId, object, waitTime, dataPath, persistentPath, volatilePath, linkerPaths)); } @@ -3407,27 +3402,8 @@ namespace PluginHost { { void* result = nullptr; - if ((interfaceId == IRemoteInstantiation::ID) && (version == static_cast(~0))) { - - _adminLock.Lock(); - // className == Connector.. - RemoteInstantiators::iterator index(_instantiators.find(className)); - - if (index == _instantiators.end()) { - IRemoteInstantiation* newIF = RemoteInstantiation::Create(*this, _processAdministrator, className); - - ASSERT(newIF != nullptr); - - _instantiators.emplace(std::piecewise_construct, - std::make_tuple(className), - std::make_tuple(newIF)); - - result = newIF; - } else { - result = index->second->QueryInterface(IRemoteInstantiation::ID); - } - - _adminLock.Unlock(); + if ((interfaceId == PluginHost::IRemoteInstantiation::ID) && (version == static_cast(~0))) { + result = _processAdministrator.Hosting(); } else { result = QueryInterfaceByCallsign(interfaceId, className); } diff --git a/Source/ThunderPlugin/Process.cpp b/Source/ThunderPlugin/Process.cpp index e7cab38c8..b9008e6c2 100644 --- a/Source/ThunderPlugin/Process.cpp +++ b/Source/ThunderPlugin/Process.cpp @@ -68,9 +68,7 @@ PUSH_WARNING(DISABLE_WARNING_THIS_IN_MEMBER_INITIALIZER_LIST) { } POP_WARNING() - ~Sink() override - { - } + ~Sink() override = default; public: void Dispatch() { @@ -103,7 +101,9 @@ POP_WARNING() public: WorkerPoolImplementation() = delete; + WorkerPoolImplementation(WorkerPoolImplementation&&) = delete; WorkerPoolImplementation(const WorkerPoolImplementation&) = delete; + WorkerPoolImplementation& operator=(WorkerPoolImplementation&&) = delete; WorkerPoolImplementation& operator=(const WorkerPoolImplementation&) = delete; PUSH_WARNING(DISABLE_WARNING_THIS_IN_MEMBER_INITIALIZER_LIST) @@ -158,6 +158,12 @@ POP_WARNING() class ConsoleOptions : public Core::Options { public: + ConsoleOptions() = delete; + ConsoleOptions(ConsoleOptions&&) = delete; + ConsoleOptions(const ConsoleOptions&&) = delete; + ConsoleOptions& operator= (ConsoleOptions&&) = delete; + ConsoleOptions& operator= (const ConsoleOptions&&) = delete; + ConsoleOptions(int argumentCount, TCHAR* arguments[]) : Core::Options(argumentCount, arguments, _T("h:l:c:C:r:p:s:d:a:m:i:u:g:t:e:E:x:V:v:P:S:f:")) , Locator(nullptr) @@ -182,9 +188,7 @@ POP_WARNING() { Parse(); } - ~ConsoleOptions() - { - } + ~ConsoleOptions() = default; public: const TCHAR* Locator; @@ -252,12 +256,11 @@ POP_WARNING() SystemRootPath = Core::Directory::Normalize(Strip(argument)); break; case 'e': - case 'E': { - Environments.push_back(Plugin::Config::Environment::Info(Strip(argument), - ((option == 'E') ? RPC::Object::Environment::Scope::GLOBAL : - RPC::Object::Environment::Scope::LOCAL))); + Environments.emplace_back(RPC::Object::Environment(Strip(argument), RPC::Environment::scope::LOCAL)); + break; + case 'E': + Environments.emplace_back(RPC::Object::Environment(Strip(argument), RPC::Environment::scope::GLOBAL)); break; - } case 'v': VolatilePath = Core::Directory::Normalize(Strip(argument)); break; @@ -651,6 +654,14 @@ int main(int argc, char** argv) Core::SystemInfo::SetEnvironment(_T("COM_PARENT_INFO"), parentInfo); + for (const auto& info : options.Environments) { + ASSERT (info.Key().empty() == false); + uint32_t status = Core::SystemInfo::SetEnvironment(info.Key(), info.Value().c_str(), ((info.Scope() == RPC::Environment::scope::GLOBAL) ? true : false)); + if (status != Core::ERROR_NONE) { + SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s], error: [%d]\n"), info.Key().c_str(), info.Value().c_str(), status)); + } + } + Process::ProcessFlow process; Core::NodeId remoteNode(options.RemoteChannel); @@ -674,15 +685,6 @@ int main(int argc, char** argv) Core::ProcessCurrent().User(string(options.User)); } - for (const auto& info : options.Environments) { - if ((info.key.empty() != true) && (info.value.empty() != true)) { - uint32_t status = Core::SystemInfo::SetEnvironment(info.key, info.value.c_str(), ((info.overriding == RPC::Object::Environment::Scope::GLOBAL) ? true : false)); - if (status != true) { - SYSLOG(Logging::Startup, (_T("Failure in setting Key:Value:[%s]:[%s]\n"), info.key.c_str(), info.value.c_str())); - } - } - } - process.Startup(options.Threads, remoteNode, callsign); // Register an interface to handle incoming requests for interfaces. diff --git a/Source/com/CMakeLists.txt b/Source/com/CMakeLists.txt index 7a12f2b73..10a526a4d 100644 --- a/Source/com/CMakeLists.txt +++ b/Source/com/CMakeLists.txt @@ -45,6 +45,8 @@ set(PUBLIC_HEADERS Module.h ) +target_compile_options(${TARGET} PRIVATE -Wno-psabi) + target_link_libraries(${TARGET} PRIVATE ${NAMESPACE}Core::${NAMESPACE}Core diff --git a/Source/com/Communicator.cpp b/Source/com/Communicator.cpp index e8efcac3d..8e156fc12 100644 --- a/Source/com/Communicator.cpp +++ b/Source/com/Communicator.cpp @@ -576,11 +576,11 @@ namespace RPC { } -ENUM_CONVERSION_BEGIN(RPC::Object::Environment::Scope) +ENUM_CONVERSION_BEGIN(RPC::Environment::scope) - { RPC::Object::Environment::Scope::LOCAL, _TXT("Local") }, - { RPC::Object::Environment::Scope::GLOBAL, _TXT("Global") }, + { RPC::Environment::scope::LOCAL, _TXT("Local") }, + { RPC::Environment::scope::GLOBAL, _TXT("Global") }, -ENUM_CONVERSION_END(RPC::Object::Environment::Scope) +ENUM_CONVERSION_END(RPC::Environment::scope) } diff --git a/Source/com/Communicator.h b/Source/com/Communicator.h index 085e3b237..107053c1d 100644 --- a/Source/com/Communicator.h +++ b/Source/com/Communicator.h @@ -40,26 +40,127 @@ #include "../warningreporting/WarningReportingUnit.h" #endif +#include "IteratorType.h" + namespace Thunder { namespace RPC { + class EXTERNAL Object { public: - static constexpr const TCHAR EnvironmentSeparator = _T('='); enum class HostType { LOCAL, DISTRIBUTED, CONTAINER }; - struct Environment { - enum class Scope { - LOCAL, - GLOBAL - }; - string key; - string value; - Scope overriding; + class EXTERNAL Environment : public RPC::Environment { + public: + static constexpr const TCHAR EnvironmentSeparator = _T('='); + + Environment (const string& keyValue, const RPC::Environment::scope scoping) { + RPC::Environment::Scope = scoping; + RPC::Environment::Key.clear(); + RPC::Environment::Value.clear(); + + size_t start = keyValue.find_first_of(EnvironmentSeparator); + if ((start != string::npos) && (start < keyValue.length())) { + string key = keyValue.substr(0, start); + string value = keyValue.substr(start + 1); + + if ((key.empty() != true) && (value.empty() != true) && (value.length() > 2) && + ((value.at(0) == '\'') && (value.at(value.length() - 1) == '\''))) { + RPC::Environment::Key = std::move(key); + RPC::Environment::Value = value.substr(1, value.length() - 2); + + start = 0; + // Remove all escaping + while ( (start = RPC::Environment::Value.find_first_of('\'', start)) != string::npos ) { + if ( (start > 0) && (RPC::Environment::Value[start-1] == '\\') ) { + if ( (start < 3) || (RPC::Environment::Value[start-2] != '\\') || (RPC::Environment::Value[start-3] != '\\') ) { + // We escaped the single quote (\'), so unescape it.. (') + RPC::Environment::Value.erase(start-1, 1); + } + else { + // We escaped the escaped quote (\\\'), so unescape it.. (\') + RPC::Environment::Value.erase(start-2, 2); + start -= 1; + } + } + else { + // Jump the quote, move on to the next + start += 1; + } + } + } + } + } + Environment (const RPC::Environment& info) { + RPC::Environment::Key = info.Key; + RPC::Environment::Value = info.Value; + RPC::Environment::Scope = info.Scope; + } + Environment (const string& key, const string& value, const RPC::Environment::scope scoping) { + RPC::Environment::Key = key; + RPC::Environment::Value = value; + RPC::Environment::Scope = scoping; + } + Environment(Environment&& move) { + RPC::Environment::Key = std::move(move.RPC::Environment::Key); + RPC::Environment::Value = std::move(move.RPC::Environment::Value); + RPC::Environment::Scope = std::move(move.RPC::Environment::Scope); + } + Environment(const Environment& copy) { + RPC::Environment::Key = copy.Key(); + RPC::Environment::Value = copy.Value(); + RPC::Environment::Scope = copy.Scope(); + } + ~Environment() = default; + + Environment& operator= (Environment&& move) { + RPC::Environment::Key = std::move(move.RPC::Environment::Key); + RPC::Environment::Value = std::move(move.RPC::Environment::Value); + RPC::Environment::Scope = std::move(move.RPC::Environment::Scope); + return (*this); + } + Environment& operator= (const Environment& copy) { + RPC::Environment::Key = copy.Key(); + RPC::Environment::Value = copy.Value(); + RPC::Environment::Scope = copy.Scope(); + return (*this); + } + + public: + const string& Key() const { + return (RPC::Environment::Key); + } + const string& Value() const { + return (RPC::Environment::Value); + } + RPC::Environment::scope Scope() const { + return (RPC::Environment::Scope); + } + string KeyValue() const { + string result = RPC::Environment::Key + EnvironmentSeparator + '\''; + size_t start = result.length();; + result += RPC::Environment::Value; + + // Now see if we need to escape the quotes in the string.. + while ( (start = result.find_first_of('\'', start)) != string::npos ) { + if ((start == 0) || (result[start-1] != '\\')) { + // We need to escape the quote + result.insert(start, 1, '\\'); + start += 2; + } + else { + // We are escaping an escaped quote + result.insert(start, 2, '\\'); + start += 3; + } + } + result += '\''; + return (result); + } }; Object() @@ -126,7 +227,7 @@ namespace RPC { const string& systemRootPath, const string& remoteAddress, const string& configuration, - const std::vector& environments) + std::vector&& environments) : _locator(locator) , _className(className) , _callsign(callsign) @@ -140,29 +241,29 @@ namespace RPC { , _systemRootPath(systemRootPath) , _remoteAddress(remoteAddress) , _configuration(configuration) - , _environments(environments) - { - } - ~Object() + , _environments(std::move(environments)) { } + ~Object() = default; Object& operator=(const Object& RHS) { - _locator = RHS._locator; - _className = RHS._className; - _callsign = RHS._callsign; - _interface = RHS._interface; - _version = RHS._version; - _user = RHS._user; - _group = RHS._group; - _threads = RHS._threads; - _priority = RHS._priority; - _systemRootPath = RHS._systemRootPath; - _type = RHS._type; - _remoteAddress = RHS._remoteAddress; - _configuration = RHS._configuration; - _environments = RHS._environments; + if (this != &RHS) { + _locator = RHS._locator; + _className = RHS._className; + _callsign = RHS._callsign; + _interface = RHS._interface; + _version = RHS._version; + _user = RHS._user; + _group = RHS._group; + _threads = RHS._threads; + _priority = RHS._priority; + _systemRootPath = RHS._systemRootPath; + _type = RHS._type; + _remoteAddress = RHS._remoteAddress; + _configuration = RHS._configuration; + _environments = RHS._environments; + } return (*this); } @@ -542,10 +643,7 @@ namespace RPC { _options.Add(_T("-t")).Add(Core::NumberType(instance.Threads()).Text()); } for (auto const& environment : instance.Environments()) { - string env = environment.key + RPC::Object::EnvironmentSeparator + - "\"" + environment.value + "\""; - string option = (environment.overriding == RPC::Object::Environment::Scope::GLOBAL) ? "-E" : "-e"; - _options.Add(_T(option)).Add('"' + env + '"'); + _options.Add(environment.Scope() == RPC::Environment::scope::GLOBAL ? _T("-E") : _T("-e")).Add(environment.KeyValue()); } _priority = instance.Priority(); } @@ -1908,6 +2006,9 @@ POP_WARNING() Core::Event _announceEvent; uint32_t _connectionId; }; + + using EnvironmentIterator = IteratorType; + } } diff --git a/Source/com/ICOM.h b/Source/com/ICOM.h index 5d58f1cc7..d035b1221 100644 --- a/Source/com/ICOM.h +++ b/Source/com/ICOM.h @@ -27,6 +27,18 @@ namespace Thunder { namespace RPC { + struct EXTERNAL Environment { + + enum scope : uint8_t { + LOCAL, + GLOBAL + }; + + string Key; + string Value; + scope Scope; + }; + struct EXTERNAL IRemoteConnection : virtual public Core::IUnknown { enum { ID = ID_COMCONNECTION }; @@ -65,5 +77,6 @@ namespace Thunder { typedef IIteratorType IStringIterator; typedef IIteratorType IValueIterator; + typedef IIteratorType IEnvironmentIterator; } } diff --git a/Source/com/Ids.h b/Source/com/Ids.h index 6c1a3b07d..b0cbad129 100644 --- a/Source/com/Ids.h +++ b/Source/com/Ids.h @@ -32,67 +32,68 @@ namespace RPC { ID_TRACEITERATOR = (ID_OFFSET_INTERNAL + 0x0003), ID_TRACECONTROLLER = (ID_OFFSET_INTERNAL + 0x0004), ID_STRINGITERATOR = (ID_OFFSET_INTERNAL + 0x0005), - ID_VALUEITERATOR = (ID_OFFSET_INTERNAL + 0x0006), - ID_MONITORABLE_PROCESS = (ID_OFFSET_INTERNAL + 0x0007), + ID_ENVIRONMENTITERATOR = (ID_OFFSET_INTERNAL + 0x0006), + ID_VALUEITERATOR = (ID_OFFSET_INTERNAL + 0x0007), + ID_MONITORABLE_PROCESS = (ID_OFFSET_INTERNAL + 0x0008), - ID_CONTROLLER_SYSTEM = (ID_OFFSET_INTERNAL + 0x0008), - ID_CONTROLLER_CONFIGURATION = (ID_OFFSET_INTERNAL + 0x0009), - ID_CONTROLLER_DISCOVERY = (ID_OFFSET_INTERNAL + 0x000A), - ID_CONTROLLER_DISCOVERY_DISCOVERYRESULTS_ITERATOR = (ID_OFFSET_INTERNAL + 0x000B), - ID_CONTROLLER_LIFETIME = (ID_OFFSET_INTERNAL + 0x000C), - ID_CONTROLLER_LIFETIME_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x000D), - ID_CONTROLLER_SHELLS = (ID_OFFSET_INTERNAL + 0x000E), - ID_CONTROLLER_SHELLS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x000F), - ID_CONTROLLER_SUBSYSTEMS = (ID_OFFSET_INTERNAL + 0x0010), - ID_CONTROLLER_SUBSYSTEMS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0011), - ID_CONTROLLER_SUBSYSTEMS_ITERATOR = (ID_OFFSET_INTERNAL + 0x0012), - ID_CONTROLLER_METADATA = (ID_OFFSET_INTERNAL + 0x0013), - ID_CONTROLLER_METADATA_SERVICES_ITERATOR = (ID_OFFSET_INTERNAL + 0x0014), - ID_CONTROLLER_METADATA_LINKS_ITERATOR = (ID_OFFSET_INTERNAL + 0x0015), - ID_CONTROLLER_METADATA_PROXIES_ITERATOR = (ID_OFFSET_INTERNAL + 0x0016), - ID_CONTROLLER_METADATA_THREADS_ITERATOR = (ID_OFFSET_INTERNAL + 0x0017), - ID_CONTROLLER_METADATA_CALLSTACK_ITERATOR = (ID_OFFSET_INTERNAL + 0x0018), - ID_CONTROLLER_EVENTS = (ID_OFFSET_INTERNAL + 0x0019), - ID_CONTROLLER_EVENTS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0020), + ID_CONTROLLER_SYSTEM = (ID_OFFSET_INTERNAL + 0x0010), + ID_CONTROLLER_CONFIGURATION = (ID_OFFSET_INTERNAL + 0x0011), + ID_CONTROLLER_DISCOVERY = (ID_OFFSET_INTERNAL + 0x0012), + ID_CONTROLLER_DISCOVERY_RESULTS_ITERATOR = (ID_OFFSET_INTERNAL + 0x0013), + ID_CONTROLLER_LIFETIME = (ID_OFFSET_INTERNAL + 0x0014), + ID_CONTROLLER_LIFETIME_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0015), + ID_CONTROLLER_SHELLS = (ID_OFFSET_INTERNAL + 0x0016), + ID_CONTROLLER_SHELLS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0017), + ID_CONTROLLER_SUBSYSTEMS = (ID_OFFSET_INTERNAL + 0x0018), + ID_CONTROLLER_SUBSYSTEMS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0019), + ID_CONTROLLER_SUBSYSTEMS_ITERATOR = (ID_OFFSET_INTERNAL + 0x001A), + ID_CONTROLLER_METADATA = (ID_OFFSET_INTERNAL + 0x001B), + ID_CONTROLLER_METADATA_SERVICES_ITERATOR = (ID_OFFSET_INTERNAL + 0x001C), + ID_CONTROLLER_METADATA_LINKS_ITERATOR = (ID_OFFSET_INTERNAL + 0x001D), + ID_CONTROLLER_METADATA_PROXIES_ITERATOR = (ID_OFFSET_INTERNAL + 0x001E), + ID_CONTROLLER_METADATA_THREADS_ITERATOR = (ID_OFFSET_INTERNAL + 0x0020), + ID_CONTROLLER_METADATA_CALLSTACK_ITERATOR = (ID_OFFSET_INTERNAL + 0x0021), + ID_CONTROLLER_EVENTS = (ID_OFFSET_INTERNAL + 0x0022), + ID_CONTROLLER_EVENTS_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0023), // Plugin module - ID_PLUGIN = (ID_OFFSET_INTERNAL + 0x0021), - ID_CONTROLLER = (ID_OFFSET_INTERNAL + 0x0020), - ID_PLUGIN_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0022), - ID_PLUGINEXTENDED = (ID_OFFSET_INTERNAL + 0x0023), - ID_WEB = (ID_OFFSET_INTERNAL + 0x0024), - ID_WEBSOCKET = (ID_OFFSET_INTERNAL + 0x0025), - ID_DISPATCHER = (ID_OFFSET_INTERNAL + 0x0026), - ID_TEXTSOCKET = (ID_OFFSET_INTERNAL + 0x0027), - ID_CHANNEL = (ID_OFFSET_INTERNAL + 0x0028), - ID_SECURITY = (ID_OFFSET_INTERNAL + 0x0029), - ID_AUTHENTICATE = (ID_OFFSET_INTERNAL + 0x002A), - ID_PLUGIN_LIFETIME = (ID_OFFSET_INTERNAL + 0x002B), - ID_COMPOSIT_PLUGIN = (ID_OFFSET_INTERNAL + 0x002C), - ID_COMPOSIT_PLUGIN_CALLBACK = (ID_OFFSET_INTERNAL + 0x002D), - ID_DISPATCHER_CALLBACK = (ID_OFFSET_INTERNAL + 0x002E), + ID_PLUGIN = (ID_OFFSET_INTERNAL + 0x0030), + ID_PLUGIN_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0031), + ID_PLUGINEXTENDED = (ID_OFFSET_INTERNAL + 0x0032), + ID_WEB = (ID_OFFSET_INTERNAL + 0x0033), + ID_WEBSOCKET = (ID_OFFSET_INTERNAL + 0x0034), + ID_DISPATCHER = (ID_OFFSET_INTERNAL + 0x0035), + ID_TEXTSOCKET = (ID_OFFSET_INTERNAL + 0x0036), + ID_CHANNEL = (ID_OFFSET_INTERNAL + 0x0037), + ID_SECURITY = (ID_OFFSET_INTERNAL + 0x0038), + ID_AUTHENTICATE = (ID_OFFSET_INTERNAL + 0x0039), + ID_PLUGIN_LIFETIME = (ID_OFFSET_INTERNAL + 0x003A), + ID_COMPOSIT_PLUGIN = (ID_OFFSET_INTERNAL + 0x003B), + ID_COMPOSIT_PLUGIN_CALLBACK = (ID_OFFSET_INTERNAL + 0x003C), + ID_DISPATCHER_CALLBACK = (ID_OFFSET_INTERNAL + 0x003D), - ID_SHELL = (ID_OFFSET_INTERNAL + 0x0030), - ID_SHELL_COMLINK = (ID_OFFSET_INTERNAL + 0x0031), - ID_SHELL_CONNECTIONSERVER = (ID_OFFSET_INTERNAL + 0x0032), - ID_SHELL_CONNECTIONSERVER_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0033), - ID_STATECONTROL = (ID_OFFSET_INTERNAL + 0x0034), - ID_STATECONTROL_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0035), - ID_SUBSYSTEM = (ID_OFFSET_INTERNAL + 0x0036), - ID_SUBSYSTEM_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0037), - ID_SUBSYSTEM_INTERNET = (ID_OFFSET_INTERNAL + 0x0038), - ID_SUBSYSTEM_LOCATION = (ID_OFFSET_INTERNAL + 0x0039), - ID_SUBSYSTEM_IDENTIFIER = (ID_OFFSET_INTERNAL + 0x003A), - ID_SUBSYSTEM_TIME = (ID_OFFSET_INTERNAL + 0x003B), - ID_SUBSYSTEM_SECURITY = (ID_OFFSET_INTERNAL + 0x003C), - ID_SUBSYSTEM_PROVISIONING = (ID_OFFSET_INTERNAL + 0x003D), - ID_SUBSYSTEM_DECRYPTION = (ID_OFFSET_INTERNAL + 0x003E), - ID_REMOTE_INSTANTIATION = (ID_OFFSET_INTERNAL + 0x003F), - ID_SYSTEM_METADATA = (ID_OFFSET_INTERNAL + 0x0040), - ID_ENVIRONMENT_ITERATOR = (ID_OFFSET_INTERNAL + 0x0041), + ID_SHELL = (ID_OFFSET_INTERNAL + 0x0040), + ID_SHELL_COMLINK = (ID_OFFSET_INTERNAL + 0x0041), + ID_SHELL_CONNECTIONSERVER = (ID_OFFSET_INTERNAL + 0x0042), + ID_SHELL_CONNECTIONSERVER_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0043), + ID_STATECONTROL = (ID_OFFSET_INTERNAL + 0x0044), + ID_STATECONTROL_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0045), - ID_EXTERNAL_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0x0080), - ID_EXTERNAL_QA_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0xA000) + ID_SUBSYSTEM = (ID_OFFSET_INTERNAL + 0x0050), + ID_SUBSYSTEM_NOTIFICATION = (ID_OFFSET_INTERNAL + 0x0051), + ID_SUBSYSTEM_INTERNET = (ID_OFFSET_INTERNAL + 0x0052), + ID_SUBSYSTEM_LOCATION = (ID_OFFSET_INTERNAL + 0x0053), + ID_SUBSYSTEM_IDENTIFIER = (ID_OFFSET_INTERNAL + 0x0054), + ID_SUBSYSTEM_TIME = (ID_OFFSET_INTERNAL + 0x0055), + ID_SUBSYSTEM_SECURITY = (ID_OFFSET_INTERNAL + 0x0056), + ID_SUBSYSTEM_PROVISIONING = (ID_OFFSET_INTERNAL + 0x0057), + ID_SUBSYSTEM_DECRYPTION = (ID_OFFSET_INTERNAL + 0x0058), + + ID_REMOTE_INSTANTIATION = (ID_OFFSET_INTERNAL + 0x0070), + ID_SYSTEM_METADATA = (ID_OFFSET_INTERNAL + 0x0071), + + ID_EXTERNAL_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0x0080), + ID_EXTERNAL_QA_INTERFACE_OFFSET = (ID_OFFSET_INTERNAL + 0xA000) }; } } diff --git a/Source/extensions/localtracer/example/CMakeLists.txt b/Source/extensions/localtracer/example/CMakeLists.txt index b0d068dd8..399c62b0f 100644 --- a/Source/extensions/localtracer/example/CMakeLists.txt +++ b/Source/extensions/localtracer/example/CMakeLists.txt @@ -2,6 +2,8 @@ add_executable(local_trace_test main.cpp ) +target_compile_options(local_trace_test PRIVATE -Wno-psabi) + target_link_libraries(local_trace_test PRIVATE CompileSettingsDebug::CompileSettingsDebug ${NAMESPACE}Core::${NAMESPACE}Core diff --git a/Source/extensions/privilegedrequest/example/CMakeLists.txt b/Source/extensions/privilegedrequest/example/CMakeLists.txt index 92717190b..d6a10428a 100644 --- a/Source/extensions/privilegedrequest/example/CMakeLists.txt +++ b/Source/extensions/privilegedrequest/example/CMakeLists.txt @@ -17,6 +17,8 @@ add_executable(fdpassing main.cpp) +target_compile_options (fdpassing PRIVATE -Wno-psabi) + target_link_libraries(fdpassing PRIVATE CompileSettingsDebug::CompileSettingsDebug ${NAMESPACE}Core::${NAMESPACE}Core diff --git a/Source/plugins/Configuration.h b/Source/plugins/Configuration.h index 325f6ed6d..b9cc748f7 100644 --- a/Source/plugins/Configuration.h +++ b/Source/plugins/Configuration.h @@ -40,38 +40,39 @@ namespace Plugin { : Core::JSON::Container() , Key() , Value() - , Override(RPC::Object::Environment::Scope::LOCAL) + , Scope(RPC::Environment::scope::LOCAL) { Add(_T("key"), &Key); Add(_T("value"), &Value); - Add(_T("override"), &Override); + Add(_T("scope"), &Scope); } Environment(const Environment& copy) : Core::JSON::Container() , Key(copy.Key) , Value(copy.Value) - , Override(copy.Override) + , Scope(copy.Scope) { Add(_T("key"), &Key); Add(_T("value"), &Value); - Add(_T("override"), &Override); + Add(_T("scope"), &Scope); } Environment(Environment&& move) noexcept : Core::JSON::Container() , Key(std::move(move.Key)) , Value(std::move(move.Value)) - , Override(std::move(move.Override)) + , Scope(std::move(move.Scope)) { Add(_T("key"), &Key); Add(_T("value"), &Value); - Add(_T("override"), &Override); + Add(_T("scope"), &Scope); } ~Environment() override = default; + Environment& operator=(const Environment& RHS) { Key = RHS.Key; Value = RHS.Value; - Override = RHS.Override; + Scope = RHS.Scope; return (*this); } @@ -80,59 +81,17 @@ namespace Plugin { if (this != &move) { Key = std::move(move.Key); Value = std::move(move.Value); - Override = std::move(move.Override); + Scope = std::move(move.Scope); } return (*this); } - static std::vector List(const Core::JSON::ArrayType& environments) - { - std::vector environmentList; - - if (environments.IsSet() == true) { - Core::JSON::ArrayType::ConstIterator index(environments.Elements()); - while (index.Next() == true) { - if ((index.Current().Key.IsSet() == true) && (index.Current().Value.IsSet() == true)) { - RPC::Object::Environment env; - env.key = index.Current().Key.Value(); - env.value = index.Current().Value.Value(); - env.overriding = index.Current().Override.Value(); - environmentList.push_back(env); - } else { - SYSLOG(Logging::Startup, (_T("Failure in Substituting Value of Key:Value:[%s]:[%s]\n"), index.Current().Key.Value().c_str(), index.Current().Value.Value().c_str())); - } - } - } - return environmentList; - } - static RPC::Object::Environment Info(const string& env, RPC::Object::Environment::Scope overriding) - { - RPC::Object::Environment info; - size_t start = env.find_first_of(RPC::Object::EnvironmentSeparator); - if ((start != string::npos) && (start < env.length())) { - string key = env.substr(0, start); - string value = env.substr(start + 1); - - if ((key.empty() != true) && (value.empty() != true) && (value.length() > 2) && - ((value.at(0) == '\"') && (value.at(value.length() - 1) == '\"'))) { - info.key = key; - info.value = value.substr(1, value.length() - 2); - info.overriding = overriding; - } else { - SYSLOG(Logging::Startup, (_T("Environment key:value fromat is invalid :[%s]:[%s]\n"), key.c_str(), value.c_str())); - } - } else { - SYSLOG(Logging::Startup, (_T("Invalid Enviroment value :[%s]\n"), env.c_str())); - } - return info; - } public: Core::JSON::String Key; Core::JSON::String Value; - Core::JSON::EnumType Override; + Core::JSON::EnumType Scope; }; - using EnvironmentList = Core::JSON::ArrayType; class EXTERNAL RootConfig : public Core::JSON::Container { private: @@ -332,7 +291,19 @@ namespace Plugin { } return result; } + std::vector Environment() const + { + std::vector environmentList; + if (Environments.IsSet() == true) { + Core::JSON::ArrayType::ConstIterator index(Environments.Elements()); + while (index.Next() == true) { + environmentList.emplace_back(RPC::Object::Environment(index.Current().Key.Value(), index.Current().Value.Value(), index.Current().Scope.Value())); + } + } + return environmentList; + } + public: Core::JSON::String Locator; Core::JSON::String User; @@ -343,7 +314,7 @@ namespace Plugin { Core::JSON::EnumType Mode; Core::JSON::String RemoteAddress; Core::JSON::String Configuration; - EnvironmentList Environments; + Core::JSON::ArrayType Environments; }; public: diff --git a/Source/plugins/IController.h b/Source/plugins/IController.h index 47c1d705e..e389a55aa 100644 --- a/Source/plugins/IController.h +++ b/Source/plugins/IController.h @@ -70,7 +70,7 @@ namespace Controller { bool Secure /* @brief Secure or not */; }; - using IDiscoveryResultsIterator = RPC::IIteratorType; + using IDiscoveryResultsIterator = RPC::IIteratorType; }; // @brief Starts SSDP network discovery diff --git a/Source/plugins/IShell.h b/Source/plugins/IShell.h index 61f4bd375..8c03770e6 100644 --- a/Source/plugins/IShell.h +++ b/Source/plugins/IShell.h @@ -58,7 +58,7 @@ namespace PluginHost { virtual void Unregister(const INotification* sink) = 0; virtual RPC::IRemoteConnection* RemoteConnection(const uint32_t connectionId) = 0; - virtual void* Instantiate(const RPC::Object& object, const uint32_t waitTime, uint32_t& connectionId) = 0; + virtual void* Instantiate(RPC::Object& object, const uint32_t waitTime, uint32_t& connectionId) = 0; }; struct EXTERNAL IConnectionServer : virtual public Core::IUnknown { diff --git a/Source/plugins/Shell.cpp b/Source/plugins/Shell.cpp index e8d052665..7902186aa 100644 --- a/Source/plugins/Shell.cpp +++ b/Source/plugins/Shell.cpp @@ -82,7 +82,7 @@ namespace PluginHost SystemRootPath(), rootConfig.RemoteAddress.Value(), rootConfig.Configuration.Value(), - Plugin::Config::Environment::List(rootConfig.Environments)); + rootConfig.Environment()); result = handler->Instantiate(definition, waitTime, pid); } From ac3f7d42d22715897ddce76d49ee164d36d5cc10 Mon Sep 17 00:00:00 2001 From: sebaszm <45654185+sebaszm@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:23:35 +0200 Subject: [PATCH 10/18] [ProcessContainers] Have default container provider (#1769) * [ProcessContainers] Have default container provider * Adjust logging * Update ContainerAdministrator.cpp --------- Co-authored-by: MFransen69 <39826971+MFransen69@users.noreply.github.com> --- Source/com/Communicator.h | 2 +- .../ContainerAdministrator.cpp | 71 +++++++++++++++---- .../ContainerAdministrator.h | 4 ++ .../processcontainers/IProcessContainers.h | 1 + 4 files changed, 62 insertions(+), 16 deletions(-) diff --git a/Source/com/Communicator.h b/Source/com/Communicator.h index 107053c1d..655bae973 100644 --- a/Source/com/Communicator.h +++ b/Source/com/Communicator.h @@ -965,7 +965,7 @@ namespace RPC { ContainerConfig() : Core::JSON::Container() - , ContainerType(ProcessContainers::IContainer::LXC) + , ContainerType(ProcessContainers::IContainer::DEFAULT) #ifdef __DEBUG__ , ContainerPath() #endif diff --git a/Source/extensions/processcontainers/ContainerAdministrator.cpp b/Source/extensions/processcontainers/ContainerAdministrator.cpp index 92c1475c8..18d723364 100644 --- a/Source/extensions/processcontainers/ContainerAdministrator.cpp +++ b/Source/extensions/processcontainers/ContainerAdministrator.cpp @@ -1,4 +1,4 @@ -/* +/* * If not stated otherwise in this file or this component's LICENSE file the * following copyright and licenses apply: * @@ -52,10 +52,16 @@ namespace ProcessContainers { container.Add(value.Data(), &specific); container.FromString(configuration); + if ((specific.IsSet() == false) && (_producers.size() == 1)) { + // Looks like the configuration for this provider is not specified, however there is only one container + // system available, so let's forward the entire config (perhaps it's a legacy config file). + specific = configuration; + } + TRACE(Trace::Information, (_T("Initializing container runtime system '%s'..."), value.Data())); // Pass the configuration to the runtime - const uint32_t initResult = runtime.second->Initialize(specific); + const uint32_t initResult = runtime.second->Initialize(specific.Value()); if (initResult != Core::ERROR_NONE) { TRACE(Trace::Error, (_T("Initialization failure"))); @@ -63,6 +69,33 @@ namespace ProcessContainers { } } + if (_producers.empty() == true) { + SYSLOG(Logging::Error, (_T("No container runtime systems are available!"))); + } + else if (_producers.size() == 1) { + // Since there is only one provider available, make it the default one + _default = _producers.cbegin()->first; + } + else { + Core::JSON::EnumType defaultProducer; + Core::JSON::Container config; + config.Add(_T("default"), &defaultProducer); + config.FromString(configuration); + + if (defaultProducer.IsSet() == true) { + _default = defaultProducer.Value(); + ASSERT(_default != IContainer::DEFAULT); + ASSERT(_producers.find(_default) != _producers.end()); + + const Core::EnumerateType value(_default); + DEBUG_VARIABLE(value); + TRACE(Trace::Information, (_T("Default container runtime is '%s'"), value.Data())); + } + else { + TRACE(Trace::Information, (_T("Default container runtime is not set"))); + } + } + _adminLock.Unlock(); return (result); @@ -86,27 +119,35 @@ namespace ProcessContainers { _adminLock.Lock(); - auto it = _producers.find(type); + const IContainer::containertype containerType = (type == IContainer::DEFAULT? _default : type); + + if (containerType != IContainer::DEFAULT) { - const Core::EnumerateType value(type); - DEBUG_VARIABLE(value); + auto it = _producers.find(containerType); - ASSERT(value.IsSet() == true); + const Core::EnumerateType value(containerType); + DEBUG_VARIABLE(value); - if (it != _producers.end()) { + ASSERT(value.IsSet() == true); + + if (it != _producers.end()) { - auto& runtime = (*it).second; - ASSERT(runtime != nullptr); + auto& runtime = (*it).second; + ASSERT(runtime != nullptr); - container = runtime->Container(id, searchPaths, logPath, configuration); + container = runtime->Container(id, searchPaths, logPath, configuration); - if (container.IsValid() == true) { - TRACE(Trace::Information, (_T("Container '%s' created successfully (runtime system '%s')"), - container->Id().c_str(), value.Data())); + if (container.IsValid() == true) { + TRACE(Trace::Information, (_T("Container '%s' created successfully (runtime system '%s')"), + container->Id().c_str(), value.Data())); + } + } + else { + TRACE(Trace::Error, (_T("Container runtime system '%s' is not enabled!"), value.Data())); } } else { - TRACE(Trace::Error, (_T("Container runtime system '%s' is not enabled!"), value.Data())); + TRACE(Trace::Error, (_T("Container runtime system not specified for '%s'"), id.c_str())); } _adminLock.Unlock(); @@ -143,4 +184,4 @@ namespace ProcessContainers { } } // namespace ProcessContainers -} \ No newline at end of file +} diff --git a/Source/extensions/processcontainers/ContainerAdministrator.h b/Source/extensions/processcontainers/ContainerAdministrator.h index d4409a80f..e63c4eac7 100644 --- a/Source/extensions/processcontainers/ContainerAdministrator.h +++ b/Source/extensions/processcontainers/ContainerAdministrator.h @@ -34,6 +34,9 @@ namespace ProcessContainers { ContainerAdministrator() : _adminLock() + , _producers() + , _containers() + , _default(IContainer::DEFAULT) { } @@ -107,6 +110,7 @@ namespace ProcessContainers { mutable Core::CriticalSection _adminLock; std::map _producers; Core::ProxyListType _containers; + IContainer::containertype _default; }; } // namespace ProcessContainers diff --git a/Source/extensions/processcontainers/IProcessContainers.h b/Source/extensions/processcontainers/IProcessContainers.h index 264f279b7..139734cb8 100644 --- a/Source/extensions/processcontainers/IProcessContainers.h +++ b/Source/extensions/processcontainers/IProcessContainers.h @@ -74,6 +74,7 @@ namespace ProcessContainers { struct IContainer { enum containertype : uint8_t { + DEFAULT, LXC, RUNC, CRUN, From f2aec59f11f200c9f58b894eae23ed24409e65e6 Mon Sep 17 00:00:00 2001 From: HaseenaSainul <41037131+HaseenaSainul@users.noreply.github.com> Date: Tue, 8 Oct 2024 01:08:14 +0530 Subject: [PATCH 11/18] ControllerPlugin: update in Documentation (#1750) Co-authored-by: Pierre Wielders Co-authored-by: MFransen69 <39826971+MFransen69@users.noreply.github.com> --- Source/Thunder/doc/ControllerPlugin.md | 144 +++++++++++++++++++++---- 1 file changed, 125 insertions(+), 19 deletions(-) diff --git a/Source/Thunder/doc/ControllerPlugin.md b/Source/Thunder/doc/ControllerPlugin.md index 82c64da5b..b4c026c9e 100644 --- a/Source/Thunder/doc/ControllerPlugin.md +++ b/Source/Thunder/doc/ControllerPlugin.md @@ -77,40 +77,41 @@ The table below lists configuration options of the plugin. This plugin implements the following interfaces: -- Controller::ISystem ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) -- Controller::IDiscovery ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) -- Controller::IConfiguration ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (uncompliant-extended format) -- Controller::ILifeTime ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) -- Controller::ISubsystems ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (uncompliant-collapsed format) -- Controller::IEvents ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) -- Controller::IMetadata ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) +- ISystem ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) +- IDiscovery ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) +- IConfiguration ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (uncompliant-extended format) +- ILifeTime ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) +- ISubsystems ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (uncompliant-collapsed format) +- IEvents ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) +- IMetadata ([IController.h](https://github.com/rdkcentral/ThunderInterfaces/blob/master/interfaces/IController.h)) (version 1.0.0) (compliant format) # Methods The following methods are provided by the Controller plugin: -Controller System interface methods: +System interface methods: | Method | Description | | :-------- | :-------- | | [reboot](#method.reboot) / [harakiri](#method.reboot) | Reboots the device | | [delete](#method.delete) | Removes contents of a directory from the persistent storage | | [clone](#method.clone) | Creates a clone of given plugin with a new callsign | +| [destroy](#method.destroy) | Destroy given plugin | -Controller Discovery interface methods: +Discovery interface methods: | Method | Description | | :-------- | :-------- | | [startdiscovery](#method.startdiscovery) | Starts SSDP network discovery | -Controller Configuration interface methods: +Configuration interface methods: | Method | Description | | :-------- | :-------- | | [persist](#method.persist) / [storeconfig](#method.persist) | Stores all configuration to the persistent memory | -Controller LifeTime interface methods: +LifeTime interface methods: | Method | Description | | :-------- | :-------- | @@ -252,6 +253,49 @@ Creates a clone of given plugin with a new callsign. } ``` + +## *destroy [method](#head.Methods)* + +Destroy given plugin. + +### Parameters + +| Name | Type | Description | +| :-------- | :-------- | :-------- | +| params | object | *...* | +| params.callsign | string | Callsign of the plugin | + +### Result + +| Name | Type | Description | +| :-------- | :-------- | :-------- | +| result | null | Always null | + +### Example + +#### Request + +```json +{ + "jsonrpc": "2.0", + "id": 42, + "method": "Controller.1.destroy", + "params": { + "callsign": "..." + } +} +``` + +#### Response + +```json +{ + "jsonrpc": "2.0", + "id": 42, + "result": null +} +``` + ## *startdiscovery [method](#head.Methods)* @@ -629,31 +673,31 @@ This is a more intelligent method, compared to *activate*, to move a plugin to a The following properties are provided by the Controller plugin: -Controller System interface properties: +System interface properties: | Property | Description | | :-------- | :-------- | | [environment](#property.environment) (read-only) | Environment variable value | -Controller Discovery interface properties: +Discovery interface properties: | Property | Description | | :-------- | :-------- | | [discoveryresults](#property.discoveryresults) (read-only) | SSDP network discovery results | -Controller Configuration interface properties: +Configuration interface properties: | Property | Description | | :-------- | :-------- | | [configuration](#property.configuration) | Service configuration | -Controller Subsystems interface properties: +Subsystems interface properties: | Property | Description | | :-------- | :-------- | | [subsystems](#property.subsystems) (read-only) | Subsystems status | -Controller Metadata interface properties: +Metadata interface properties: | Property | Description | | :-------- | :-------- | @@ -664,6 +708,7 @@ Controller Metadata interface properties: | [threads](#property.threads) (read-only) | Workerpool threads | | [pendingrequests](#property.pendingrequests) (read-only) | Pending requests | | [callstack](#property.callstack) (read-only) | Thread callstack | +| [buildinfo](#property.buildinfo) (read-only) | Build information | ## *environment [property](#head.Properties)* @@ -1276,6 +1321,67 @@ Provides access to the thread callstack. } ``` + +## *buildinfo [property](#head.Properties)* + +Provides access to the build information. + +> This property is **read-only**. + +### Value + +### Result + +| Name | Type | Description | +| :-------- | :-------- | :-------- | +| result | object | Build information | +| result.systemtype | string | System type (must be one of the following: *Linux, MacOS, Windows*) | +| result.buildtype | string | Build type (must be one of the following: *Debug, DebugOptimized, Production, Release, ReleaseWithDebugInfo*) | +| result?.extensions | array | *(optional)* *...* | +| result?.extensions[#] | string | *(optional)* *...* (must be one of the following: *Bluetooth, Hiberbate, ProcessContainers, WarningReporting*) | +| result.messaging | boolean | Denotes whether Messaging is enabled | +| result.exceptioncatching | boolean | Denotes whether there is an exception | +| result.deadlockdetection | boolean | Denotes whether deadlock detection is enabled | +| result.wcharsupport | boolean | *...* | +| result.instanceidbits | integer | Core instance bits | +| result?.tracelevel | integer | *(optional)* Trace level | +| result.threadpoolcount | integer | *...* | + +### Example + +#### Get Request + +```json +{ + "jsonrpc": "2.0", + "id": 42, + "method": "Controller.1.buildinfo" +} +``` + +#### Get Response + +```json +{ + "jsonrpc": "2.0", + "id": 42, + "result": { + "systemtype": "Windows", + "buildtype": "Debug", + "extensions": [ + "WarningReporting" + ], + "messaging": false, + "exceptioncatching": false, + "deadlockdetection": false, + "wcharsupport": false, + "instanceidbits": 0, + "tracelevel": 0, + "threadpoolcount": 0 + } +} +``` + # Notifications @@ -1283,19 +1389,19 @@ Notifications are autonomous events triggered by the internals of the implementa The following events are provided by the Controller plugin: -Controller LifeTime interface events: +LifeTime interface events: | Notification | Description | | :-------- | :-------- | | [statechange](#notification.statechange) | Notifies of a plugin state change | -Controller Subsystems interface events: +Subsystems interface events: | Notification | Description | | :-------- | :-------- | | [subsystemchange](#notification.subsystemchange) | Notifies a subsystem change | -Controller Events interface events: +Events interface events: | Notification | Description | | :-------- | :-------- | From f6ebe63cfa47101dfe6ae28525c00f25b0cecb68 Mon Sep 17 00:00:00 2001 From: MFransen69 <39826971+MFransen69@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:43:34 +0200 Subject: [PATCH 12/18] [ReleaseNotes] Thunder 5.1 (#1770) * [ReleaseNotes] Thunder 5.1 * [ReleaseNotes] Case now correct for some keywords * [ReleaseNotes] small typos fixed --- ReleaseNotes/ThunderReleaseNotes_R5.1.md | 117 +++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 ReleaseNotes/ThunderReleaseNotes_R5.1.md diff --git a/ReleaseNotes/ThunderReleaseNotes_R5.1.md b/ReleaseNotes/ThunderReleaseNotes_R5.1.md new file mode 100644 index 000000000..0dec8af06 --- /dev/null +++ b/ReleaseNotes/ThunderReleaseNotes_R5.1.md @@ -0,0 +1,117 @@ +# Thunder Release Notes R5.1 + +## introduction + +This document describes the new features and changes introduced in Thunder R5.1 (compared to the latest R5.0 release). +See [here](https://github.com/rdkcentral/Thunder/blob/R5_0/ReleaseNotes/ThunderReleaseNotes_R5.0.md) for the release notes of Thunder R5.0. +This document describes the changes in Thunder, ThunderTools and ThunderInterfaces as they are related. + +# Thunder + +## Process Changes and new Features + +### Feature: Github actions extensions + +The Github actions were extended to have better build coverage, they are ran on every Pull Request commit. More components are now included in the builds. Also the 32bit build and Process Containers are now included as target. + +### Change: Unit test improvements + +The existing Thunder unit tests were improved again and new tests were added. These are also triggered from a GitHub action on each commit added to a Thunder Pull Request. + +### Change: Thunder Documentation + +The Thunder documentation was extended with new content, please find the documentation [here](https://rdkcentral.github.io/Thunder/). More content still to come! + +## Changes and new Features + +### New Feature: Whole plugin OOP (out of process) + +It so now possible to run the full plugin out of process (OOP). Before this feature was added a plugin had to be specifically designed to be run OOP and part of it was always in process. The "whole plugin OOP" feature is added mainly as a development/debug option, for example to test leaks and crashes with plugins only designed to run in process. There is also the prerequisite that the plugin must have a JSON-RPC interface, a prerequisite that may be removed in a future Thunder version. +You use this feature by adding a root section including mode="Local" outside the config part of the plugin (like you normally would have inside the plugin config). Any other modes should work as well but have not been tested. + +### New Feature: PluginSmartInterfaceType + +Next to the already existing SmartInterfaceType that can be used to more easily communicate with plugins exposing interfaces over a COM-RPP connection from native code a new PluginSmartInterfaceType was added that enables the same but then from plugin to plugin. Of course this could already be done but the new PluginSmartInterfaceType makes this much easier and requires less boilerplate code. +Find the code [here](https://github.com/rdkcentral/Thunder/blob/ac3f7d42d22715897ddce76d49ee164d36d5cc10/Source/plugins/Types.h#L403) and an example on how to use this feature can be found [here](https://github.com/rdkcentral/ThunderNanoServices/tree/master/examples/PluginSmartInterfaceType). + +### New Feature: Concurrent Process Containers + +On request of an external partner it is now possible to have multiple container flavours (container providers) active at the same time. +This feature was added fully backwards compatible regarding the Thunder and plugin config files, meaning if only one provider is active no changes to the config files are needed. However a change is required to the Container implementation as well and this has been done for runC and LXC. For Dobby and AWC we are discussing this with the owners of these implementations. +To use the feature first enable multiple container providers in your build. Then in the Thunder config file add the following section to set the default container implementation to be used for a plugin: +```json +"processcontainers" : { + "default" : "lxc" + } +``` +Other valid options are: "runc", "crun", "dobby" and "awc". +If in a plugin you want to use another container flavour than the default one you can do so by adding this to the config file of the container (of course the "mode : Container" should always be there when you want to run a plugin inside a container): +```json +"root":{ + "mode":"Container", + "configuration" : { + "containertype":"runc" + } +} +``` +More detailed documentation on this feature and process containers in general will be added to the Thunder documentation in the near future. + +### New Feature: Flowcontrol + +To mitigate issues with plugins that have methods that have a relatively long processing time while at the same time have an interface that for example invite the user of the interface to do a lot of calls to the plugin and/or the plugin emits a lot of events Thunder 5.1 now has a FlowControl feature. +Per channel (e.g. websocket) there can now be only one JSON-RPC call at a time, the next call received in parallel while the first one has not been handled will only be executed once the first one has been completed. Next to the channel flow control also a feature is added that will only allow one JSON-RPC call at a time to a certain plugin, so even if they come from different channels. +In a future Thunder version the number of allowed parallel calls (both for a channel as to a plugin) will be made configurable. + +### New Feature: Destroy plugin option + +Next to the already existing possibility to Clone a plugin it is now also possible to Destroy a (cloned) plugin using the Controller COM-RPC or JSON-RPC interface. + +### Change: General bug fixes + +As Thunder R5.0 was quite a change compared to Thunder R4 a lot of issue found in the mean time in Thunder 5.0 have been fixed in Thunder 5.1. If you are using Thunder 5.0 we would strongly advice to upgrade to Thunder 5.1. As Thunder 5.1 is released relatively short after Thunder 5.0 it does not contain that many new features and changes but more importantly does contain quite some fixes for issues found in Thunder 5.0. + +### Change: Message Engine improvements + +There were quite some improvements to the message engine. UDP support for exporting messages was enhanced greatly. + +## Breaking Changes + +Although Thunder 5.1 is a minor upgrade and therefore should not contain breaking changes there is one however. This because of the request to the Thunder team to include this fix and only after consulting Thunder user representatives. + +### Change: JSON-RPC over HTTP error code + +We were made aware that the HTTP 202 result code that was returned in case there was an error handling a JSON-RPC call over HTTP was not as one would expect. +We did some investigation and although there is no official RFC for JSON-RPC over HTTP the consensus is indeed a 200 code should be returned, also when there is an error handling the JSON-RPC call. So Thunder 5.1 will return a HTTP 200 result code in this case. +Note in case this does cause issues in certain situations the old behaviour can be simply brought back by setting the LEGACY_JSONRPCOVERHTTP_ERRORCODE build flag when building Thunder. + +# Thunder Tools + +## Changes and new Features + +### Case handling improvements + +There were may changes in the code generators to better support the different Case options for the JSON-RPC methods, parameters, POD members etc. + +### camelCase COM-RPC methods + +camelCase COM-RPC methods names are now allowed and supported by the generators (please note this is not the standard used in the Thunder repositories) + +### Enhanced fixed array support + +Fixed arrays are now also allowed as method parameters and POD members in interfaces. Please note only primitive types are allowed as already was the case with dynamic arrays. Thunder documentation will be updated soon with more information on this. + +### OptionalType enhancements + +OptionalType is now also allowed for property indexes and iterators in interfaces. The OptionalType for the property index is for example used in the Thunder Controller interface: if filled it indicates the callsign of the plugin for which info is requested, if omitted (so empty) information for all plugins will be returned. + +### null returned for property setter + +In case a property setter was used with an array or object it might return something else than null in previous versions of Thunder. The generated documentation however did (correctly) specify null would be returned. This is now also actually the case for the generated code. + +# Thunder Interfaces + +## Changes and new Features + +### IDolby + +IDolby was made backwards compatible again (this was broken in ThunderInterfaces R5.0) From 61fe958584165172e5340256d19814f71b10dad4 Mon Sep 17 00:00:00 2001 From: HaseenaSainul <41037131+HaseenaSainul@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:38:07 +0530 Subject: [PATCH 13/18] Handle thunder id based on platform type (#1766) * Use JSON::InstanceID, instead Json::ThreadId thread conversion places * Review comments addressed --------- Co-authored-by: Pierre Wielders --- Source/Thunder/Controller.cpp | 4 ++-- Source/Thunder/Controller.h | 2 +- Source/Thunder/PluginHost.cpp | 11 ++++------- Source/Thunder/PluginServer.h | 7 +------ Source/Thunder/PostMortem.h | 12 ++++++------ Source/core/JSON.h | 1 - Source/core/Portability.cpp | 6 +++--- Source/core/Portability.h | 17 ++++++++-------- Source/core/ProcessInfo.cpp | 12 ++++++------ Source/core/ProcessInfo.h | 6 +++--- Source/core/ResourceMonitor.h | 2 +- Source/core/Thread.cpp | 4 ++-- Source/core/Thread.h | 6 +++--- Source/core/ThreadPool.h | 6 +++--- Source/core/Timer.h | 2 +- Source/core/WorkerPool.h | 8 ++++---- Source/plugins/Metadata.cpp | 4 ++-- Source/plugins/Metadata.h | 30 +++++++++++++++++++++++++++-- Tests/unit/core/test_thread.cpp | 10 +++++----- Tests/unit/core/test_workerpool.cpp | 4 ++-- 20 files changed, 86 insertions(+), 68 deletions(-) diff --git a/Source/Thunder/Controller.cpp b/Source/Thunder/Controller.cpp index f0683b703..04ab088c8 100644 --- a/Source/Thunder/Controller.cpp +++ b/Source/Thunder/Controller.cpp @@ -52,7 +52,7 @@ namespace Plugin { static Core::ProxyPoolType jsonBodyTextFactory(2); - void Controller::Callstack(const ThreadId id, Core::JSON::ArrayType& response) const { + void Controller::Callstack(const Core::thread_id id, Core::JSON::ArrayType& response) const { std::list stackList; ::DumpCallStack(id, stackList); @@ -1256,7 +1256,7 @@ namespace Plugin { while (it.Next() == true) { auto const& entry = it.Current(); - threads.push_back({ entry.Id.Value(), entry.Job.Value(), entry.Runs.Value() }); + threads.push_back({ PluginHost::Metadata::InstanceId(entry.Id.Value()), entry.Job.Value(), entry.Runs.Value() }); } using Iterator = IMetadata::Data::IThreadsIterator; diff --git a/Source/Thunder/Controller.h b/Source/Thunder/Controller.h index bd7ed365a..497e502f4 100644 --- a/Source/Thunder/Controller.h +++ b/Source/Thunder/Controller.h @@ -368,7 +368,7 @@ namespace Plugin { void WorkerPoolMetadata(PluginHost::Metadata::Server& data) const { _pluginServer->WorkerPool().Snapshot(data); } - void Callstack(const ThreadId id, Core::JSON::ArrayType& response) const; + void Callstack(const Core::thread_id id, Core::JSON::ArrayType& response) const; void SubSystems(); uint32_t Clone(const string& basecallsign, const string& newcallsign); void Proxies(Core::JSON::ArrayType& info) const; diff --git a/Source/Thunder/PluginHost.cpp b/Source/Thunder/PluginHost.cpp index be22a0b7a..abbb9c6a7 100644 --- a/Source/Thunder/PluginHost.cpp +++ b/Source/Thunder/PluginHost.cpp @@ -889,11 +889,8 @@ POP_WARNING() printf("Pending: %d\n", static_cast(metaData.Pending.size())); printf("Poolruns:\n"); for (uint8_t index = 0; index < metaData.Slots; index++) { -#ifdef __APPLE__ - printf(" Thread%02d|0x%16" PRIxPTR ": %10d", (index), reinterpret_cast(metaData.Slot[index].WorkerId), metaData.Slot[index].Runs); -#else - printf(" Thread%02d|0x%16lX: %10d", (index), metaData.Slot[index].WorkerId, metaData.Slot[index].Runs); -#endif + printf(" Thread%02d|0x%16" PRIu64 ": %10d", (index), static_cast(Metadata::InstanceId(metaData.Slot[index].WorkerId)), metaData.Slot[index].Runs); + if (metaData.Slot[index].Job.IsSet() == false) { printf("\n"); } @@ -983,10 +980,10 @@ POP_WARNING() case '7': case '8': case '9': { - ThreadId threadId = _dispatcher->WorkerPool().Id(keyPress - '0'); + Core::thread_id threadId = _dispatcher->WorkerPool().Id(keyPress - '0'); printf("\nThreadPool thread[%c] callstack:\n", keyPress); printf("============================================================\n"); - if (threadId != (ThreadId)(~0)) { + if (threadId != (Core::thread_id)(~0)) { uint8_t counter = 0; std::list stackList; ::DumpCallStack(threadId, stackList); diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index df783746f..5614b66b9 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -4600,12 +4600,7 @@ namespace PluginHost { while (index.Next() == true) { std::list stackList; - -#ifdef __APPLE__ - ::DumpCallStack(reinterpret_cast(index.Current().Id.Value()), stackList); -#else - ::DumpCallStack(static_cast(index.Current().Id.Value()), stackList); -#endif + ::DumpCallStack(PluginHost::Metadata::ThreadId(index.Current().Id.Value()), stackList); PostMortemData::Callstack dump; dump.Id = index.Current().Id.Value(); diff --git a/Source/Thunder/PostMortem.h b/Source/Thunder/PostMortem.h index 02d8d6c32..e7167c0bd 100644 --- a/Source/Thunder/PostMortem.h +++ b/Source/Thunder/PostMortem.h @@ -108,10 +108,10 @@ namespace PluginHost { } public: - Core::JSON::Pointer Address; - Core::JSON::String Function; - Core::JSON::String Module; - Core::JSON::DecUInt32 Line; + Core::JSON::InstanceId Address; + Core::JSON::String Function; + Core::JSON::String Module; + Core::JSON::DecUInt32 Line; }; @@ -123,7 +123,7 @@ namespace PluginHost { Callstack() : Core::JSON::Container() - , Id(0) + , Id() , Data() { Add(_T("id"), &Id); Add(_T("stack"), &Data); @@ -145,7 +145,7 @@ namespace PluginHost { ~Callstack() override = default; public: - Core::JSON::Pointer Id; + Core::JSON::InstanceId Id; Core::JSON::ArrayType Data; }; diff --git a/Source/core/JSON.h b/Source/core/JSON.h index 7fd5a4438..d59bfe5c2 100644 --- a/Source/core/JSON.h +++ b/Source/core/JSON.h @@ -1017,7 +1017,6 @@ namespace Core { typedef NumberType OctSInt64; typedef NumberType InstanceId; - typedef InstanceId Pointer; template class FloatType : public IElement, public IMessagePack { diff --git a/Source/core/Portability.cpp b/Source/core/Portability.cpp index f7d5bb83d..49ba11e4f 100644 --- a/Source/core/Portability.cpp +++ b/Source/core/Portability.cpp @@ -147,7 +147,7 @@ static void CallstackSignalHandler(int signr VARIABLE_IS_NOT_USED, siginfo_t* in } } -uint32_t GetCallStack(const ThreadId threadId, void* addresses[], const uint32_t bufferSize) +uint32_t GetCallStack(const Thunder::Core::thread_id threadId, void* addresses[], const uint32_t bufferSize) { uint32_t result = 0; @@ -159,7 +159,7 @@ uint32_t GetCallStack(const ThreadId threadId, void* addresses[], const uint32_t } --result; } - } else if (threadId != (::ThreadId)(~0)) { + } else if (threadId != (Thunder::Core::thread_id)(~0)) { while (std::atomic_exchange_explicit(&g_lock, true, std::memory_order_acquire)) ; // spin until acquired @@ -196,7 +196,7 @@ uint32_t GetCallStack(const ThreadId threadId, void* addresses[], const uint32_t extern "C" { -void DumpCallStack(const ThreadId threadId VARIABLE_IS_NOT_USED, std::list& stackList VARIABLE_IS_NOT_USED) +void DumpCallStack(const Thunder::Core::thread_id threadId VARIABLE_IS_NOT_USED, std::list& stackList VARIABLE_IS_NOT_USED) { #if defined(THUNDER_BACKTRACE) void* callstack[32]; diff --git a/Source/core/Portability.h b/Source/core/Portability.h index add1df3bb..3969ddb11 100644 --- a/Source/core/Portability.h +++ b/Source/core/Portability.h @@ -679,12 +679,6 @@ typedef std::string string; #define STRLEN(STATIC_TEXT) ((sizeof(STATIC_TEXT) / sizeof(TCHAR)) - 1) #define EMPTY_STRING _T("") -#ifdef __LINUX__ -typedef pthread_t ThreadId; -#else -typedef DWORD ThreadId; -#endif - #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) @@ -717,6 +711,12 @@ namespace Core { #endif #endif + #ifdef __LINUX__ + typedef pthread_t thread_id; + #else + typedef DWORD thread_id; + #endif + typedef uint32_t hresult; struct callstack_info { @@ -990,8 +990,9 @@ EXTERNAL extern int inet_aton(const char* cp, struct in_addr* inp); EXTERNAL extern void usleep(const uint32_t value); #endif -EXTERNAL void DumpCallStack(const ThreadId threadId, std::list& stack); -EXTERNAL uint32_t GetCallStack(const ThreadId threadId, void* addresses[], const uint32_t bufferSize); +void EXTERNAL DumpCallStack(const Thunder::Core::thread_id threadId, std::list& stack); +uint32_t EXTERNAL GetCallStack(const Thunder::Core::thread_id threadId, void* addresses[], const uint32_t bufferSize); + } diff --git a/Source/core/ProcessInfo.cpp b/Source/core/ProcessInfo.cpp index c29dc469a..5d1b071d8 100644 --- a/Source/core/ProcessInfo.cpp +++ b/Source/core/ProcessInfo.cpp @@ -686,30 +686,30 @@ namespace Core { EnumerateChildProcesses(processInfo, _processes); } - bool ProcessTree::ContainsProcess(ThreadId pid) const + bool ProcessTree::ContainsProcess(thread_id pid) const { PUSH_WARNING(DISABLE_WARNING_CONVERSION_TO_GREATERSIZE) - auto comparator = [pid](const ProcessInfo& processInfo) { return ((ThreadId)(processInfo.Id()) == pid); }; + auto comparator = [pid](const ProcessInfo& processInfo) { return ((thread_id)(processInfo.Id()) == pid); }; POP_WARNING() std::list::const_iterator i = std::find_if(_processes.cbegin(), _processes.cend(), comparator); return (i != _processes.cend()); } - void ProcessTree::GetProcessIds(std::list& processIds) const + void ProcessTree::GetProcessIds(std::list& processIds) const { processIds.clear(); for (const ProcessInfo& process : _processes) { PUSH_WARNING(DISABLE_WARNING_CONVERSION_TO_GREATERSIZE) - processIds.push_back((ThreadId)(process.Id())); + processIds.push_back((thread_id)(process.Id())); POP_WARNING() } } - ThreadId ProcessTree::RootId() const + thread_id ProcessTree::RootId() const { PUSH_WARNING(DISABLE_WARNING_CONVERSION_TO_GREATERSIZE) - return (ThreadId)(_processes.front().Id()); + return (thread_id)(_processes.front().Id()); POP_WARNING() } diff --git a/Source/core/ProcessInfo.h b/Source/core/ProcessInfo.h index f87843907..f2807e366 100644 --- a/Source/core/ProcessInfo.h +++ b/Source/core/ProcessInfo.h @@ -414,9 +414,9 @@ namespace Core { public: explicit ProcessTree(const ProcessInfo& processInfo); - bool ContainsProcess(ThreadId pid) const; - void GetProcessIds(std::list& processIds) const; - ThreadId RootId() const; + bool ContainsProcess(thread_id pid) const; + void GetProcessIds(std::list& processIds) const; + thread_id RootId() const; private: std::list _processes; diff --git a/Source/core/ResourceMonitor.h b/Source/core/ResourceMonitor.h index acd6abe95..334747b69 100644 --- a/Source/core/ResourceMonitor.h +++ b/Source/core/ResourceMonitor.h @@ -177,7 +177,7 @@ namespace Core { { return (_monitorRuns); } - ::ThreadId Id() const + thread_id Id() const { return (_monitor != nullptr ? _monitor->Id() : 0); } diff --git a/Source/core/Thread.cpp b/Source/core/Thread.cpp index 690a43af7..b635c20f4 100644 --- a/Source/core/Thread.cpp +++ b/Source/core/Thread.cpp @@ -132,14 +132,14 @@ namespace Core { #endif } - ::ThreadId Thread::ThreadId() + thread_id Thread::ThreadId() { #ifdef __WINDOWS__ PUSH_WARNING(DISABLE_WARNING_CONVERSION_TO_GREATERSIZE) return (::GetCurrentThreadId()); POP_WARNING() #else - return static_cast<::ThreadId>(pthread_self()); + return static_cast(pthread_self()); #endif } diff --git a/Source/core/Thread.h b/Source/core/Thread.h index d9f71a00b..f1055352d 100644 --- a/Source/core/Thread.h +++ b/Source/core/Thread.h @@ -218,11 +218,11 @@ namespace Core { int PriorityMin() const; int PriorityMax() const; bool Priority(int priority); - inline ::ThreadId Id() const + inline thread_id Id() const { return (m_ThreadId); } - static ::ThreadId ThreadId(); + static thread_id ThreadId(); template static STORAGETYPE& GetContext() @@ -288,7 +288,7 @@ namespace Core { HANDLE m_hThreadInstance; #endif - ::ThreadId m_ThreadId; + thread_id m_ThreadId; static uint32_t _defaultStackSize; #ifdef __POSIX__ string m_threadName; diff --git a/Source/core/ThreadPool.h b/Source/core/ThreadPool.h index 461122090..ae269908b 100644 --- a/Source/core/ThreadPool.h +++ b/Source/core/ThreadPool.h @@ -51,14 +51,14 @@ namespace Core { virtual void Dispatch(IDispatch*) = 0; }; struct EXTERNAL Metadata { - ::ThreadId WorkerId; + thread_id WorkerId; uint32_t Runs; Core::OptionalType Job; }; #ifdef __CORE_WARNING_REPORTING__ struct EXTERNAL DispatchedJobMetaData { - ::ThreadId WorkerId; + thread_id WorkerId; string CallSign; uint64_t DispatchedTime; uint32_t ReportRunCount; @@ -597,7 +597,7 @@ POP_WARNING() _queue.Unlock(); } - ::ThreadId Id(const uint8_t index) const + thread_id Id(const uint8_t index) const { uint8_t count = 0; std::list::const_iterator ptr = _units.cbegin(); diff --git a/Source/core/Timer.h b/Source/core/Timer.h index 466df1eb8..e496ef5d7 100644 --- a/Source/core/Timer.h +++ b/Source/core/Timer.h @@ -329,7 +329,7 @@ namespace Core { return (static_cast(_pendingQueue.size())); } - ::ThreadId ThreadId() const + thread_id ThreadId() const { return (_timerThread.Id()); } diff --git a/Source/core/WorkerPool.h b/Source/core/WorkerPool.h index c359bbdff..28d9cc0e9 100644 --- a/Source/core/WorkerPool.h +++ b/Source/core/WorkerPool.h @@ -102,7 +102,7 @@ namespace Core { static IWorkerPool& Instance(); static bool IsAvailable(); - virtual ::ThreadId Id(const uint8_t index) const = 0; + virtual thread_id Id(const uint8_t index) const = 0; virtual void Submit(const Core::ProxyType& job) = 0; virtual void Schedule(const Core::Time& time, const Core::ProxyType& job) = 0; virtual bool Reschedule(const Core::Time& time, const Core::ProxyType& job) = 0; @@ -384,9 +384,9 @@ POP_WARNING() _external.Process(); _joined = 0; } - ::ThreadId Id(const uint8_t index) const override + thread_id Id(const uint8_t index) const override { - ::ThreadId result = (::ThreadId)(~0); + thread_id result = (thread_id)(~0); if (index == 0) { result = _timer.ThreadId(); @@ -431,7 +431,7 @@ POP_WARNING() ThreadPool::Minion _external; Core::TimerType _timer; mutable Metadata _metadata; - ::ThreadId _joined; + thread_id _joined; #ifdef __CORE_WARNING_REPORTING__ DispatchedJobMonitor _dispatchedJobMonitor; #endif diff --git a/Source/plugins/Metadata.cpp b/Source/plugins/Metadata.cpp index cee862995..eeae04ba2 100644 --- a/Source/plugins/Metadata.cpp +++ b/Source/plugins/Metadata.cpp @@ -215,7 +215,7 @@ namespace PluginHost Metadata::Server::Minion::Minion() : Core::JSON::Container() - , Id(0) + , Id() , Job() , Runs(0) { Add(_T("id"), &Id); @@ -241,7 +241,7 @@ namespace PluginHost Add(_T("runs"), &Runs); } Metadata::Server::Minion& Metadata::Server::Minion::operator=(const Core::ThreadPool::Metadata& info) { - Id = (Core::instance_id)info.WorkerId; + Id = Metadata::InstanceId(info.WorkerId); Runs = info.Runs; if (info.Job.IsSet() == false) { diff --git a/Source/plugins/Metadata.h b/Source/plugins/Metadata.h index e60c0eef4..6079ad040 100644 --- a/Source/plugins/Metadata.h +++ b/Source/plugins/Metadata.h @@ -140,8 +140,8 @@ namespace PluginHost { public: Core::JSON::InstanceId Id; - Core::JSON::String Job; - Core::JSON::DecUInt32 Runs; + Core::JSON::String Job; + Core::JSON::DecUInt32 Runs; }; public: @@ -255,6 +255,32 @@ namespace PluginHost { AppVersion.Clear(); } + template + static typename std::enable_if<(std::is_same::value), Core::instance_id>::type + InstanceId(TYPE id) + { + return reinterpret_cast(reinterpret_cast(id)); + } + template + static typename std::enable_if::value), Core::instance_id>::type + InstanceId(TYPE id) + { + return static_cast(id); + } + + template + static typename std::enable_if<(std::is_same::value), Core::thread_id>::type + ThreadId(Core::instance_id id) + { + return reinterpret_cast(id); + } + template + static typename std::enable_if::value), Core::thread_id>::type + ThreadId(Core::instance_id id) + { + return static_cast(id); + } + public: SubSystem SubSystems; Core::JSON::ArrayType Plugins; diff --git a/Tests/unit/core/test_thread.cpp b/Tests/unit/core/test_thread.cpp index 02fc96f32..03720cb53 100644 --- a/Tests/unit/core/test_thread.cpp +++ b/Tests/unit/core/test_thread.cpp @@ -38,7 +38,7 @@ namespace Core { ThreadClass(const ThreadClass&) = delete; ThreadClass& operator=(const ThreadClass&) = delete; - ThreadClass(volatile bool& threadDone, std::mutex& mutex, std::condition_variable& cv, ::ThreadId parentTid) + ThreadClass(volatile bool& threadDone, std::mutex& mutex, std::condition_variable& cv, ::Thunder::Core::thread_id parentTid) : ::Thunder::Core::Thread(::Thunder::Core::Thread::DefaultStackSize(), _T("Test")) , _done(threadDone) , _threadMutex(mutex) @@ -66,7 +66,7 @@ namespace Core { volatile bool& _done; std::mutex& _threadMutex; std::condition_variable& _threadCV; - ::ThreadId _parentTid; + ::Thunder::Core::thread_id _parentTid; }; class Job : public ::Thunder::Core::IDispatch { @@ -97,7 +97,7 @@ namespace Core { private: static bool _threadDone; - static ::ThreadId _parentTPid; + static ::Thunder::Core::thread_id _parentTPid; public: static std::mutex _mutex; @@ -107,11 +107,11 @@ namespace Core { bool Job::_threadDone = false; std::mutex Job::_mutex; std::condition_variable Job::_cv; - ::ThreadId Job::_parentTPid = ::Thunder::Core::Thread::ThreadId(); + ::Thunder::Core::thread_id Job::_parentTPid = ::Thunder::Core::Thread::ThreadId(); TEST(Core_Thread, DISABLED_SimpleThread) { - ::ThreadId parentTid = ::Thunder::Core::Thread::ThreadId(); + ::Thunder::Core::thread_id parentTid = ::Thunder::Core::Thread::ThreadId(); volatile bool threadDone = false; std::mutex mutex; std::condition_variable cv; diff --git a/Tests/unit/core/test_workerpool.cpp b/Tests/unit/core/test_workerpool.cpp index deec4df9b..b5e531e4c 100644 --- a/Tests/unit/core/test_workerpool.cpp +++ b/Tests/unit/core/test_workerpool.cpp @@ -260,10 +260,10 @@ namespace Core { const uint8_t MaxSize = 15; bool isPoolId = false; char id[MaxSize]; - sprintf(id, "%x", static_cast<::ThreadId>(pthread_self())); + sprintf(id, "%x", static_cast<::Thunder::Core::thread_id>(pthread_self())); for (uint8_t index = 0; index < _threadsCount + 2; index++) { char workerId[MaxSize]; - sprintf(workerId, "%x", static_cast<::ThreadId>(::Thunder::Core::IWorkerPool::Instance().Id(index))); + sprintf(workerId, "%x", static_cast<::Thunder::Core::thread_id>(::Thunder::Core::IWorkerPool::Instance().Id(index))); if (strcpy(workerId, id)) { isPoolId = true; From ea4a37bb327c48ff62fe864e6e7b47e11e10bef7 Mon Sep 17 00:00:00 2001 From: Pierre Wielders Date: Wed, 9 Oct 2024 15:11:15 +0200 Subject: [PATCH 14/18] Update Portability.h (#1772) Probably a merge issue as the old code came back again. Always make sure thet the Preprocessor are ate the beginning of a function declaration! --- Source/core/Portability.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/core/Portability.h b/Source/core/Portability.h index 3969ddb11..395b252af 100644 --- a/Source/core/Portability.h +++ b/Source/core/Portability.h @@ -990,8 +990,8 @@ EXTERNAL extern int inet_aton(const char* cp, struct in_addr* inp); EXTERNAL extern void usleep(const uint32_t value); #endif -void EXTERNAL DumpCallStack(const Thunder::Core::thread_id threadId, std::list& stack); -uint32_t EXTERNAL GetCallStack(const Thunder::Core::thread_id threadId, void* addresses[], const uint32_t bufferSize); +EXTERNAL void DumpCallStack(const Thunder::Core::thread_id threadId, std::list& stack); +EXTERNAL uint32_t GetCallStack(const Thunder::Core::thread_id threadId, void* addresses[], const uint32_t bufferSize); } From 0161d8edb23ffadac5f49b5eee4ff1a421605916 Mon Sep 17 00:00:00 2001 From: nxtum <94901881+nxtum@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:59:41 +0200 Subject: [PATCH 15/18] Replace SharedBuffer::Semaphore with Core::SharedSemaphore (#1737) * core::binarysemaphore in sharedbuffer * spelling, binairy to binary in sharedbuffer * musl fix, added sem_timedwait * fix by initializing sem * fixed logic of jump * hide sem_t * detect musl * fix windows * sem open support, max method * add unused * errors semopen * create/open sem * sharedbuffer offset calculations * add sizeof to handle * fix alignment for buffer * align buffer 32/64 * align adminstration * add variable unused * change isLocked to Count * create windowsapi for ntquerysemaphore * spelling mistake * name fix * remarks --------- Co-authored-by: Pierre Wielders --- Source/core/Portability.h | 30 ++++ Source/core/Proxy.h | 4 +- Source/core/SharedBuffer.cpp | 150 +++----------------- Source/core/SharedBuffer.h | 40 +----- Source/core/Sync.cpp | 259 +++++++++++++++++++++++++++++++++-- Source/core/Sync.h | 60 ++++++-- Source/core/Thread.cpp | 2 +- 7 files changed, 354 insertions(+), 191 deletions(-) diff --git a/Source/core/Portability.h b/Source/core/Portability.h index 395b252af..b5b44cb90 100644 --- a/Source/core/Portability.h +++ b/Source/core/Portability.h @@ -359,6 +359,20 @@ typedef std::string string; //const std::basic_string::size_type std::basic_string::npos = (std::basic_string::size_type) - 1; //#endif +// NTQuerySemaphore (undocumented) is used to retrieve current count of a semaphore +using NTSTATUS = LONG; +using _NTQuerySemaphore = NTSTATUS(NTAPI*)( + HANDLE SemaphoreHandle, + DWORD SemaphoreInformationClass, + PVOID SemaphoreInformation, + ULONG SemaphoreInformationLength, + PULONG ReturnLength OPTIONAL +); +struct SEMAPHORE_BASIC_INFORMATION { + ULONG CurrentCount; + ULONG MaximumCount; +}; + #define LITTLE_ENDIAN_PLATFORM 1 #undef ERROR #define __WINDOWS__ @@ -385,6 +399,7 @@ typedef std::string string; #include #include #include +#include #include #include @@ -412,9 +427,24 @@ typedef std::string string; #include #include #include // memfd_create in Messaging/ConsoleRedirect.h +#include #include +#ifndef _GNU_SOURCE + #define _GNU_SOURCE + #include + #ifndef __USE_GNU + #define __MUSL__ + #endif + #undef _GNU_SOURCE /* don't contaminate other includes unnecessarily */ +#else + #include + #ifndef __USE_GNU + #define __MUSL__ + #endif +#endif + #ifdef __APPLE__ #include #define SCHED_BATCH 99 diff --git a/Source/core/Proxy.h b/Source/core/Proxy.h index d51cce599..0b2830fe5 100644 --- a/Source/core/Proxy.h +++ b/Source/core/Proxy.h @@ -539,11 +539,11 @@ POP_WARNING() { return (!operator==(a_RHS)); } - inline bool operator==(const nullptr_t&) const + inline bool operator==(const std::nullptr_t&) const { return (_refCount == nullptr); } - inline bool operator!=(const nullptr_t&) const + inline bool operator!=(const std::nullptr_t&) const { return (_refCount != nullptr); } diff --git a/Source/core/SharedBuffer.cpp b/Source/core/SharedBuffer.cpp index c26704943..572b1f7d9 100644 --- a/Source/core/SharedBuffer.cpp +++ b/Source/core/SharedBuffer.cpp @@ -26,148 +26,38 @@ namespace Thunder { namespace Core { -#ifdef __WINDOWS__ - SharedBuffer::Semaphore::Semaphore(const TCHAR sourceName[]) - : _semaphore(::CreateSemaphore(nullptr, 1, 1, sourceName)) - { - } -#else - SharedBuffer::Semaphore::Semaphore(sem_t* storage) - : _semaphore(storage) - { - ASSERT(storage != nullptr); - } -#endif - SharedBuffer::Semaphore::~Semaphore() - { -#ifdef __WINDOWS__ - if (_semaphore != nullptr) { - ::CloseHandle(_semaphore); - } -#else - sem_destroy(_semaphore); -#endif - } - - uint32_t SharedBuffer::Semaphore::Unlock() - { -#ifdef __WINDOWS__ - if (_semaphore != nullptr) { - BOOL result = ::ReleaseSemaphore(_semaphore, 1, nullptr); - - ASSERT(result != FALSE); - } -#else - VARIABLE_IS_NOT_USED int result = sem_post(_semaphore); - - ASSERT((result == 0) || (errno == EOVERFLOW)); -#endif - return ERROR_NONE; - } - - bool SharedBuffer::Semaphore::IsLocked() - { -#ifdef __WINDOWS__ - bool locked = (::WaitForSingleObjectEx(_semaphore, 0, FALSE) != WAIT_OBJECT_0); - - if (locked == false) { - ::ReleaseSemaphore(_semaphore, 1, nullptr); - } - - return (locked); -#else - int semValue = 0; - sem_getvalue(_semaphore, &semValue); - return (semValue == 0); -#endif - } - - uint32_t SharedBuffer::Semaphore::Lock(const uint32_t waitTime) - { - uint32_t result = Core::ERROR_GENERAL; -#ifdef __WINDOWS__ - if (_semaphore != nullptr) { - return (::WaitForSingleObjectEx(_semaphore, waitTime, FALSE) == WAIT_OBJECT_0 ? Core::ERROR_NONE : Core::ERROR_TIMEDOUT); - } -#elif defined(__APPLE__) - - uint32_t timeLeft = waitTime; - int semResult; - while (((semResult = sem_trywait(_semaphore)) != 0) && timeLeft > 0) { - ::SleepMs(100); - if (timeLeft != Core::infinite) { - timeLeft -= (timeLeft > 100 ? 100 : timeLeft); - } - } - result = semResult == 0 ? Core::ERROR_NONE : Core::ERROR_TIMEDOUT; -#else - - struct timespec structTime = {0,0}; - - clock_gettime(CLOCK_MONOTONIC, &structTime); - structTime.tv_nsec += ((waitTime % 1000) * 1000 * 1000); /* remainder, milliseconds to nanoseconds */ - structTime.tv_sec += (waitTime / 1000) + (structTime.tv_nsec / 1000000000); /* milliseconds to seconds */ - structTime.tv_nsec = structTime.tv_nsec % 1000000000; - - do { - if (sem_clockwait(_semaphore, CLOCK_MONOTONIC, &structTime) == 0) { - result = Core::ERROR_NONE; - } - else if ( errno == EINTR ) { - continue; - } - else if ( errno == ETIMEDOUT ) { - result = Core::ERROR_TIMEDOUT; - } - else { - ASSERT(false); - } - break; - } while (true); -#endif - return (result); - } - SharedBuffer::SharedBuffer(const TCHAR name[]) : DataElementFile(name, File::USER_READ | File::USER_WRITE | File::SHAREABLE, 0) , _administrationBuffer((string(name) + ".admin"), File::USER_READ | File::USER_WRITE | File::SHAREABLE, 0) , _administration(reinterpret_cast(PointerAlign(_administrationBuffer.Buffer()))) -#ifdef __WINDOWS__ + #ifdef __WINDOWS__ , _producer((string(name) + ".producer").c_str()) , _consumer((string(name) + ".consumer").c_str()) -#else - , _producer(&(_administration->_producer)) - , _consumer(&(_administration->_consumer)) -#endif - , _customerAdministration(PointerAlign(&(reinterpret_cast(_administration)[sizeof(Administration)]))) + #else + , _producer(reinterpret_cast(_administration) + sizeof(Administration)) + , _consumer(reinterpret_cast(_administration) + sizeof(Administration) + SharedSemaphore::Size()) + #endif + , _customerAdministration(PointerAlign(&(reinterpret_cast(_administration)[sizeof(Administration) + (SharedSemaphore::Size() * 2)]))) { Align(); } SharedBuffer::SharedBuffer(const TCHAR name[], const uint32_t mode, const uint32_t bufferSize, const uint16_t administratorSize) : DataElementFile(name, mode | File::SHAREABLE | File::CREATE, bufferSize) - , _administrationBuffer((string(name) + ".admin"), mode | File::SHAREABLE | File::CREATE, administratorSize + sizeof(Administration) + (2 * sizeof(void*)) + 8 /* Align buffer on 64 bits boundary */) + , _administrationBuffer((string(name) + ".admin"), mode | File::SHAREABLE | File::CREATE, sizeof(Administration) + (SharedSemaphore::Size() * 2) + administratorSize + + (sizeof(void*) * 2) + ((sizeof(Administration) + (SharedSemaphore::Size() * 2) + administratorSize) % sizeof(void*) == 0 ? + 0 : (sizeof(void*) - ((sizeof(Administration) + (SharedSemaphore::Size() * 2) + administratorSize) % sizeof(void*)))) /* Align buffer on 32/64 bits boundary */) , _administration(reinterpret_cast(PointerAlign(_administrationBuffer.Buffer()))) -#ifdef __WINDOWS__ - , _producer((string(name) + ".producer").c_str()) - , _consumer((string(name) + ".consumer").c_str()) -#else - , _producer(&(_administration->_producer)) - , _consumer(&(_administration->_consumer)) -#endif - , _customerAdministration(PointerAlign(&(reinterpret_cast(_administration)[sizeof(Administration)]))) - { - -#ifndef __WINDOWS__ - memset(_administration, 0, sizeof(Administration)); - - sem_init(&(_administration->_producer), 1, 1); /* Initial value is 1. */ - sem_init(&(_administration->_consumer), 1, 0); /* Initial value is 0. */ -#endif + #ifdef __WINDOWS__ + , _producer((string(name) + ".producer").c_str(), 1, 1) + , _consumer((string(name) + ".consumer").c_str(), 0, 1) + #else + , _producer(reinterpret_cast(_administration) + sizeof(Administration), 1, 1) + , _consumer(reinterpret_cast(_administration) + sizeof(Administration) + SharedSemaphore::Size(), 0, 1) +# endif + , _customerAdministration(PointerAlign(&(reinterpret_cast(_administration)[sizeof(Administration) + (SharedSemaphore::Size() * 2)]))) + { + _administration->_bytesWritten = 0; Align(); } - - SharedBuffer::~SharedBuffer() - { - } -} } +} \ No newline at end of file diff --git a/Source/core/SharedBuffer.h b/Source/core/SharedBuffer.h index 664e34d98..77035525d 100644 --- a/Source/core/SharedBuffer.h +++ b/Source/core/SharedBuffer.h @@ -65,46 +65,12 @@ namespace Core { SharedBuffer& operator=(const SharedBuffer&) = delete; private: - class Semaphore { - private: - Semaphore() = delete; - Semaphore(const Semaphore&) = delete; - Semaphore& operator=(const Semaphore&) = delete; - - public: -#ifdef __WINDOWS__ - Semaphore(const TCHAR name[]); -#else - Semaphore(sem_t* storage); - //Semaphore(sem_t* storage, bool initialize) { - //} -#endif - ~Semaphore(); - - public: - uint32_t Lock(const uint32_t waitTime); - uint32_t Unlock(); - bool IsLocked(); - - private: -#ifdef __WINDOWS__ - HANDLE _semaphore; -#else - sem_t* _semaphore; -#endif - }; struct Administration { - uint32_t _bytesWritten; - -#ifndef __WINDOWS__ - sem_t _producer; - sem_t _consumer; -#endif }; public: - virtual ~SharedBuffer(); + ~SharedBuffer() override = default; // This is the consumer constructor. It should always take place, after, the producer // construct. The producer will create the Administration area, and the shared buffer, @@ -215,8 +181,8 @@ namespace Core { private: DataElementFile _administrationBuffer; Administration* _administration; - Semaphore _producer; - Semaphore _consumer; + Core::SharedSemaphore _producer; + Core::SharedSemaphore _consumer; uint8_t* _customerAdministration; }; } diff --git a/Source/core/Sync.cpp b/Source/core/Sync.cpp index e5f04ef3a..545161e0d 100644 --- a/Source/core/Sync.cpp +++ b/Source/core/Sync.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #endif #ifdef __APPLE__ #include @@ -271,7 +272,7 @@ namespace Core { //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- - // BinairySemaphore class + // BinarySemaphore class //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- @@ -283,12 +284,12 @@ namespace Core { // sets the inital count an the maximum count. This way, on platform changes, // only the declaration/definition of the synchronisation object has to be defined // as being Binairy, not the coding. - BinairySemaphore::BinairySemaphore(unsigned int nInitialCount, unsigned int nMaxCount) + BinarySemaphore::BinarySemaphore(unsigned int nInitialCount, unsigned int nMaxCount) { DEBUG_VARIABLE(nMaxCount); ASSERT((nInitialCount == 0) || (nInitialCount == 1)); - TRACE_L5("Constructor BinairySemaphore (int, int) <%p>", (this)); + TRACE_L5("Constructor BinarySemaphore (int, int) <%p>", (this)); #ifdef __POSIX__ m_blLocked = (nInitialCount == 0); @@ -323,12 +324,12 @@ namespace Core { #endif } - BinairySemaphore::BinairySemaphore(bool blLocked) + BinarySemaphore::BinarySemaphore(bool blLocked) #ifdef __POSIX__ : m_blLocked(blLocked) #endif { - TRACE_L5("Constructor BinairySemaphore <%p>", (this)); + TRACE_L5("Constructor BinarySemaphore <%p>", (this)); #ifdef __POSIX__ pthread_condattr_t attr; @@ -360,9 +361,9 @@ namespace Core { #endif } - BinairySemaphore::~BinairySemaphore() + BinarySemaphore::~BinarySemaphore() { - TRACE_L5("Destructor BinairySemaphore <%p>", (this)); + TRACE_L5("Destructor BinarySemaphore <%p>", (this)); #ifdef __POSIX__ // If we really create it, we really have to destroy it. @@ -380,7 +381,7 @@ namespace Core { //---------------------------------------------------------------------------- uint32_t - BinairySemaphore::Lock() + BinarySemaphore::Lock() { #ifdef __WINDOWS__ @@ -423,7 +424,7 @@ namespace Core { } uint32_t - BinairySemaphore::Lock(unsigned int nTime) + BinarySemaphore::Lock(unsigned int nTime) { #ifdef __WINDOWS__ return (::WaitForSingleObjectEx(m_syncMutex, nTime, FALSE) == WAIT_OBJECT_0 ? Core::ERROR_NONE : Core::ERROR_TIMEDOUT); @@ -482,8 +483,8 @@ namespace Core { #endif } - void - BinairySemaphore::Unlock() + uint32_t + BinarySemaphore::Unlock() { #ifdef __POSIX__ @@ -505,6 +506,7 @@ namespace Core { #ifdef __WINDOWS__ ::ReleaseMutex(m_syncMutex); #endif + return ERROR_NONE; } //---------------------------------------------------------------------------- @@ -782,6 +784,241 @@ namespace Core { return (nResult); } + //---------------------------------------------------------------------------- + //---------------------------------------------------------------------------- + // SharedSemaphore class + //---------------------------------------------------------------------------- + //---------------------------------------------------------------------------- + + //---------------------------------------------------------------------------- + // CONSTRUCTOR & DESTRUCTOR + //---------------------------------------------------------------------------- + +#ifdef __WINDOWS__ + class WindowsAPI { + public: + WindowsAPI(WindowsAPI&&) = delete; + WindowsAPI(const WindowsAPI&) = delete; + WindowsAPI& operator=(WindowsAPI&&) = delete; + WindowsAPI& operator=(const WindowsAPI&) = delete; + + ~WindowsAPI() = default; + WindowsAPI() { + _ntQuerySemaphore = reinterpret_cast<_NTQuerySemaphore>(GetProcAddress(GetModuleHandle(_T("ntdll.dll")), "NtQuerySemaphore")); + ASSERT (_ntQuerySemaphore != nullptr); + } + + uint32_t GetSemaphoreCount(HANDLE parameter) const { + SEMAPHORE_BASIC_INFORMATION basicInfo; + NTSTATUS status; + status = _ntQuerySemaphore(parameter, 0, &basicInfo, sizeof(SEMAPHORE_BASIC_INFORMATION), nullptr); + return (status == ERROR_SUCCESS) ? basicInfo.CurrentCount : 0; + } + private: + _NTQuerySemaphore _ntQuerySemaphore; + }; + + static WindowsAPI _windowsAPI; +#endif + + SharedSemaphore::SharedSemaphore(const TCHAR sourceName[], const uint32_t initCount, VARIABLE_IS_NOT_USED const uint32_t maxCount) + { + ASSERT(initCount <= 1); + ASSERT(maxCount == 1); +#ifdef __WINDOWS__ + _semaphore = (::CreateSemaphore(nullptr, initCount, maxCount, sourceName)); + ASSERT(_semaphore != nullptr); +#else + _name = "/" + string(sourceName); + _semaphore = sem_open(_name.c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, initCount); + ASSERT(_semaphore != SEM_FAILED); +#endif + } + + SharedSemaphore::SharedSemaphore(const TCHAR sourceName[]) + { +#ifdef __WINDOWS__ + _semaphore = ::OpenSemaphore(SEMAPHORE_ALL_ACCESS, FALSE, sourceName); + ASSERT(_semaphore != nullptr); +#else + _semaphore = sem_open(sourceName, 0); + ASSERT(_semaphore != nullptr); +#endif + } + +#ifndef __WINDOWS__ + SharedSemaphore::SharedSemaphore(void* storage, const uint32_t initCount, VARIABLE_IS_NOT_USED const uint32_t maxCount) + : _semaphore(storage), _name("") + { + ASSERT(storage != nullptr); + ASSERT(initCount <= 1); + ASSERT(maxCount == 1); + memset(_semaphore, 0, sizeof(sem_t)); + VARIABLE_IS_NOT_USED int result = sem_init(static_cast(_semaphore), 1, initCount); + ASSERT(result != -1); + } + + SharedSemaphore::SharedSemaphore(void* storage) + : _semaphore(storage), _name("") + { + ASSERT(storage != nullptr); + } + +#endif + + SharedSemaphore::~SharedSemaphore() + { +#ifdef __WINDOWS__ + if (_semaphore != nullptr) { + ::CloseHandle(_semaphore); + } +#else + if(_name.size() != 0) { + sem_close(static_cast(_semaphore)); + sem_unlink(_name.c_str()); + } + else { + sem_destroy(static_cast(_semaphore)); + } +#endif + } + + size_t SharedSemaphore::Size() + { +#ifdef __WINDOWS__ + return sizeof(HANDLE); +#else + return sizeof(sem_t); +#endif + } + + uint32_t SharedSemaphore::MaxCount() const + { + // Currently max count is 1, as it is implemented as a binary shared semaphore + return 1; + } + + //---------------------------------------------------------------------------- + // PUBLIC METHODS + //---------------------------------------------------------------------------- + + uint32_t + SharedSemaphore::Unlock() + { +#ifdef __WINDOWS__ + if (_semaphore != nullptr) { + BOOL result = ::ReleaseSemaphore(_semaphore, 1, nullptr); + ASSERT(result != FALSE); + } +#else + VARIABLE_IS_NOT_USED int result = sem_post(static_cast(_semaphore)); + ASSERT((result == 0) || (errno == EOVERFLOW)); +#endif + return ERROR_NONE; + } + + uint32_t + SharedSemaphore::Count() const + { +#ifdef __WINDOWS__ + return (_windowsAPI.GetSemaphoreCount(_semaphore)); +#else + int semValue = 0; + sem_getvalue(static_cast(_semaphore), &semValue); + return semValue; +#endif + } + + uint32_t + SharedSemaphore::Lock(const uint32_t waitTime) + { + uint32_t result = Core::ERROR_GENERAL; +#ifdef __WINDOWS__ + if (_semaphore != nullptr) { + return (::WaitForSingleObjectEx(_semaphore, waitTime, FALSE) == WAIT_OBJECT_0 ? Core::ERROR_NONE : Core::ERROR_TIMEDOUT); + } +#elif defined(__APPLE__) + + uint32_t timeLeft = waitTime; + int semResult; + while (((semResult = sem_trywait(static_cast(_semaphore))) != 0) && timeLeft > 0) { + ::SleepMs(100); + if (timeLeft != Core::infinite) { + timeLeft -= (timeLeft > 100 ? 100 : timeLeft); + } + } + result = semResult == 0 ? Core::ERROR_NONE : Core::ERROR_TIMEDOUT; + +#elif defined(__MUSL__) + struct timespec referenceTime = {0,0}; + clock_gettime(CLOCK_MONOTONIC, &referenceTime); + referenceTime.tv_nsec += ((waitTime % 1000) * 1000 * 1000); /* remainder, milliseconds to nanoseconds */ + referenceTime.tv_sec += (waitTime / 1000) + (referenceTime.tv_nsec / 1000000000); /* milliseconds to seconds */ + referenceTime.tv_nsec = referenceTime.tv_nsec % 1000000000; + do { + + struct timespec structTime = {0,0}; + clock_gettime(CLOCK_REALTIME, &structTime); + structTime.tv_nsec += ((waitTime % 1000) * 1000 * 1000); /* remainder, milliseconds to nanoseconds */ + structTime.tv_sec += (waitTime / 1000) + (structTime.tv_nsec / 1000000000); /* milliseconds to seconds */ + structTime.tv_nsec = structTime.tv_nsec % 1000000000; + + if (sem_timedwait(static_cast(_semaphore), &structTime) == 0) { + result = Core::ERROR_NONE; + } + else if ( errno == EINTR ) { + continue; + } + else if ( errno == ETIMEDOUT ) { + struct timespec currentMonoTime; + clock_gettime(CLOCK_MONOTONIC, ¤tMonoTime); + + struct timespec jumpTime; + jumpTime.tv_sec = currentMonoTime.tv_sec - referenceTime.tv_sec; + + if(jumpTime.tv_sec != 0) { + result = Core::ERROR_TIMEDOUT; + break; + } + + if(referenceTime.tv_sec < currentMonoTime.tv_sec || (referenceTime.tv_sec == currentMonoTime.tv_sec && + referenceTime.tv_nsec < currentMonoTime.tv_nsec)) + { + result = Core::ERROR_TIMEDOUT; + break; + } + } + else { + ASSERT(false); + } + break; + } while (true); +#else + struct timespec structTime = {0,0}; + clock_gettime(CLOCK_MONOTONIC, &structTime); + structTime.tv_nsec += ((waitTime % 1000) * 1000 * 1000); /* remainder, milliseconds to nanoseconds */ + structTime.tv_sec += (waitTime / 1000) + (structTime.tv_nsec / 1000000000); /* milliseconds to seconds */ + structTime.tv_nsec = structTime.tv_nsec % 1000000000; + + do { + if (sem_clockwait(static_cast(_semaphore), CLOCK_MONOTONIC, &structTime) == 0) { + result = Core::ERROR_NONE; + } + else if ( errno == EINTR ) { + continue; + } + else if ( errno == ETIMEDOUT ) { + result = Core::ERROR_TIMEDOUT; + } + else { + ASSERT(false); + } + break; + } while (true); +#endif + return (result); + } + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- // Event class (AVAILABLE WITHIN PROCESS SPACE) diff --git a/Source/core/Sync.h b/Source/core/Sync.h index 63540c773..fbe30a663 100644 --- a/Source/core/Sync.h +++ b/Source/core/Sync.h @@ -29,7 +29,6 @@ #ifdef __LINUX__ #include -#include #endif namespace Thunder { @@ -112,23 +111,22 @@ namespace Core { // =========================================================================== // class BinairySemaphore // =========================================================================== - - class EXTERNAL BinairySemaphore { + class EXTERNAL BinarySemaphore { private: - BinairySemaphore() = delete; - BinairySemaphore(const BinairySemaphore&) = delete; - BinairySemaphore& operator=(const BinairySemaphore&) = delete; + BinarySemaphore() = delete; + BinarySemaphore(const BinarySemaphore&) = delete; + BinarySemaphore& operator=(const BinarySemaphore&) = delete; public: // Methods - BinairySemaphore(unsigned int nInitialCount, unsigned int nMaxCount); - BinairySemaphore(bool blLocked); - ~BinairySemaphore(); + BinarySemaphore(unsigned int nInitialCount, unsigned int nMaxCount); + BinarySemaphore(bool blLocked); + ~BinarySemaphore(); uint32_t Lock(); // Time in milliseconds! uint32_t Lock(unsigned int nSeconds); - void Unlock(); + uint32_t Unlock(); bool Locked() const; protected: // Members @@ -143,6 +141,9 @@ namespace Core { #endif }; + // For compatability reaons with old class name + using BinairySemaphore = BinarySemaphore; + // =========================================================================== // class CountingSemaphore // =========================================================================== @@ -181,6 +182,45 @@ namespace Core { // For Windows platform compatibility typedef CountingSemaphore Semaphore; + // =========================================================================== + // class SharedSemaphore + // =========================================================================== + + class EXTERNAL SharedSemaphore { + public: + SharedSemaphore() = delete; + SharedSemaphore(const Semaphore&) = delete; + SharedSemaphore& operator=(const SharedSemaphore&) = delete; + + SharedSemaphore(const TCHAR name[], const uint32_t initValue, const uint32_t maxValue); + SharedSemaphore(const TCHAR name[]); +#ifndef __WINDOWS__ + /* + If pshared is nonzero, then the semaphore is shared between + processes, and should be located in a region of shared memory + (see shm_open(3), mmap(2), and shmget(2)) + Storage should be at least sizeof(sem_t)! + */ + SharedSemaphore(void *storage, const uint32_t initValue, const uint32_t maxValue); + SharedSemaphore(void *storage); + public: +#endif + ~SharedSemaphore(); + public: + uint32_t Lock(const uint32_t waitTime); + uint32_t Unlock(); + uint32_t Count() const; + static size_t Size(); + uint32_t MaxCount() const; + private: +#ifdef __WINDOWS__ + HANDLE _semaphore; +#else + void* _semaphore; + string _name; +#endif + }; + // =========================================================================== // class Event // =========================================================================== diff --git a/Source/core/Thread.cpp b/Source/core/Thread.cpp index b635c20f4..b5251b310 100644 --- a/Source/core/Thread.cpp +++ b/Source/core/Thread.cpp @@ -469,7 +469,7 @@ POP_WARNING() } #ifdef __DEBUG__ - int Thread::GetCallstack(void** buffer, int size) + int Thread::GetCallstack(VARIABLE_IS_NOT_USED void** buffer, VARIABLE_IS_NOT_USED int size) { #if defined(THUNDER_BACKTRACE) return GetCallStack(m_hThreadInstance, buffer, size); From 456ee8db806f510e03946403b607a0934c433e99 Mon Sep 17 00:00:00 2001 From: Mateusz Daniluk <121170681+VeithMetro@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:29:58 +0200 Subject: [PATCH 16/18] [Actions] Fix errors due to an update of GitHub hosted runners (#1774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Try to install jsonref with pipx in the external env * Use the development template to test the patch * Cannot use pipx on this runner, try to create a virtual env instead * Make sure to activate the virtual enviroment before building * Move the activation after the paths update * Try with sudo * Make sure to only activate the venv when needed * Remove the advanced download as it might casue issues * Make the installation easier * Try pipx in a different wa * Try without sudo * Try venv again * Use pip instead of pipx * Remove installing of jsonref for now to test * Remove venv activation for now * For now install everything apart from jsonref to isolate the issue * Remove deactivation of venv * Add venv again but don't active it yet in the generators * See if activating the venv fixes this single issue with c++ compiler * Activate the venv for the Thunder build as well * Add the libstdc++-multilib to the 32-bit build It looks like the 32-bit C++ compiler is missing... * Install libstdc++-11-dev:i386 instead * Add libc6-dev:i386 * Look for Alternative Package Names * Make sure to install GCC 13 multilib instead of 11 * Add using CHANNEL::Handle * Handle in SocketPort is private so cannot use using * Disable the overloaded virtuals warning * Move the warning push from cpp to the header * Add a PUSH_WARNING(DISABLE_WARNING_OVERLOADED_VIRTUALS) to Handle() * Fix the free() call on a pointer with the nonzero offset FAILED: Source/com/CMakeFiles/ThunderCOM.dir/Communicator.cpp.o /usr/bin/c++ -DBLUEZ_HAS_NO_INCLUSIVE_LANGUAGE -DCOM_EXPORTS -DNO_INCLUSIVE_LANGUAGE -DPROCESSCONTAINERS_ENABLED=1 -DThunderCOM_EXPORTS -DWARNING_REPORTING_ENABLED -D_TRACE_LEVEL=0 -D__CORE_BLUETOOTH_SUPPORT__ -D__CORE_MESSAGING__ -D__CORE_NO_WCHAR_SUPPORT__ -D__CORE_WARNING_REPORTING__ -I/home/runner/work/Thunder/Thunder/Release/build/Thunder/Source/com -I/home/runner/work/Thunder/Thunder/Thunder/Source/com -I/home/runner/work/Thunder/Thunder/Release/build/Thunder/Source/core -I/home/runner/work/Thunder/Thunder/Release/build/Thunder/Source/core/core -I/home/runner/work/Thunder/Thunder/Release/build/Thunder/Source/core/generated/core -I/home/runner/work/Thunder/Thunder/Thunder/Source/core/.. -I/home/runner/work/Thunder/Thunder/Thunder/Source/messaging -I/home/runner/work/Thunder/Thunder/Thunder/Source/messaging/.. -I/home/runner/work/Thunder/Thunder/Thunder/Source/extensions/warningreporting -I/home/runner/work/Thunder/Thunder/Thunder/Source/extensions/warningreporting/.. -I/home/runner/work/Thunder/Thunder/Thunder/Source/extensions/processcontainers -I/home/runner/work/Thunder/Thunder/Thunder/Source/extensions/processcontainers/.. -Wall -Wextra -Wpedantic -Werror -m64 -O3 -DNDEBUG -std=gnu++11 -fPIC -fvisibility=hidden -fvisibility-inlines-hidden -Wno-psabi -MD -MT Source/com/CMakeFiles/ThunderCOM.dir/Communicator.cpp.o -MF Source/com/CMakeFiles/ThunderCOM.dir/Communicator.cpp.o.d -o Source/com/CMakeFiles/ThunderCOM.dir/Communicator.cpp.o -c /home/runner/work/Thunder/Thunder/Thunder/Source/com/Communicator.cpp In file included from /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/Singleton.h:30, from /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/ResourceMonitor.h:24, from /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/DoorBell.h:23, from /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/core.h:35, from /home/runner/work/Thunder/Thunder/Thunder/Source/com/Module.h:26, from /home/runner/work/Thunder/Thunder/Thunder/Source/com/Communicator.h:24, from /home/runner/work/Thunder/Thunder/Thunder/Source/com/Communicator.cpp:20: In static member function ‘static void Thunder::Core::ProxyObject::operator delete(void*) [with CONTEXT = Thunder::Core::IPCMessageType<1, Thunder::RPC::Data::Init, Thunder::RPC::Data::Setup>]’, inlined from ‘Thunder::Core::ProxyObject::~ProxyObject() [with CONTEXT = Thunder::Core::IPCMessageType<1, Thunder::RPC::Data::Init, Thunder::RPC::Data::Setup>]’ at /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/Proxy.h:119:13, inlined from ‘uint32_t Thunder::Core::ProxyObject::Release() const [with CONTEXT = Thunder::Core::IPCMessageType<1, Thunder::RPC::Data::Init, Thunder::RPC::Data::Setup>]’ at /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/Proxy.h:138:21, inlined from ‘Thunder::Core::ProxyType::~ProxyType() [with CONTEXT = Thunder::Core::IPCMessageType<1, Thunder::RPC::Data::Init, Thunder::RPC::Data::Setup>]’ at /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/Proxy.h:439:39, inlined from ‘virtual void Thunder::RPC::CommunicatorClient::StateChange()’ at /home/runner/work/Thunder/Thunder/Thunder/Source/com/Communicator.cpp:524:66: /home/runner/work/Thunder/Thunder/Thunder/Source/core/../core/Proxy.h:97:23: error: ‘void free(void*)’ called on pointer ‘’ with nonzero offset 1056 [-Werror=free-nonheap-object] 97 | ::free(stAllocateBlock); | ~~~~~~^~~~~~~~~~~~~~~~~ cc1plus: all warnings being treated as errors * Don't forget to free the originalPointer * Add using Core::WorkerPool::Snapshot to the WorkerPoolImpl * Remove the changes as they don't fix the issue * Try one more solution * Try one more idea.. * Add DISABLE_WARNING_FREE_NONHEAP_OBJECT * Suppress the warning for now and investigate futher later on * Developnment finished, use the master template again * Install jsonref in venv for the tests as well * Sadly we have to do the merging twice --- .github/workflows/Build Thunder on Linux.yml | 2 +- .github/workflows/Linux build template.yml | 8 ++++++-- .github/workflows/Test Thunder.yml | 7 +++++-- Source/Thunder/PluginServer.h | 2 ++ Source/core/Portability.h | 2 ++ Source/core/Proxy.h | 2 ++ Source/core/SocketPort.h | 2 ++ 7 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Build Thunder on Linux.yml b/.github/workflows/Build Thunder on Linux.yml index 1cbe4b686..6255548b4 100644 --- a/.github/workflows/Build Thunder on Linux.yml +++ b/.github/workflows/Build Thunder on Linux.yml @@ -17,7 +17,7 @@ on: jobs: Thunder: - uses: rdkcentral/Thunder/.github/workflows/Linux build template.yml@master + uses: rdkcentral/Thunder/.github/workflows/Linux build template.yml@development/actions-pep668-error # change back to master after Actions work again ThunderInterfaces: needs: Thunder diff --git a/.github/workflows/Linux build template.yml b/.github/workflows/Linux build template.yml index 81f52d83d..99f918538 100644 --- a/.github/workflows/Linux build template.yml +++ b/.github/workflows/Linux build template.yml @@ -28,8 +28,10 @@ jobs: echo "deb http://archive.ubuntu.com/ubuntu/ jammy-updates main universe restricted multiverse" | sudo tee -a /etc/apt/sources.list ${{matrix.architecture == '32' && 'sudo dpkg --add-architecture i386' || ':'}} sudo apt-get update - sudo apt install python3-pip build-essential cmake ninja-build libusb-1.0-0-dev ${{matrix.architecture == '32' && 'zlib1g-dev:i386 libssl-dev:i386 libsbc-dev:i386 gcc-11-multilib g++-11-multilib' || 'zlib1g-dev libssl-dev libsbc-dev'}} - sudo pip install jsonref + sudo apt install python3-pip build-essential cmake ninja-build libusb-1.0-0-dev ${{matrix.architecture == '32' && 'zlib1g-dev:i386 libssl-dev:i386 libsbc-dev:i386 gcc-13-multilib g++-13-multilib' || 'zlib1g-dev libssl-dev libsbc-dev'}} + python3 -m venv venv + source venv/bin/activate + pip install jsonref # ----- Checkout & DependsOn regex ----- # --------- Thunder ---------- @@ -85,6 +87,7 @@ jobs: # ----- Installing generators & Options regex ----- - name: Install generators run: | + source venv/bin/activate ${{matrix.architecture == '32' && 'export PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig:$PKG_CONFIG_PATH' || 'PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH'}} cmake -G Ninja -S ThunderTools -B ${{matrix.build_type}}/build/ThunderTools \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ @@ -104,6 +107,7 @@ jobs: # ----- Building & uploading artifacts ----- - name: Build Thunder run: | + source venv/bin/activate ${{matrix.architecture == '32' && 'export PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig:$PKG_CONFIG_PATH' || 'PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH'}} cmake -G Ninja -S Thunder -B ${{matrix.build_type}}/build/Thunder \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wpedantic -Werror -m${{matrix.architecture}}" \ diff --git a/.github/workflows/Test Thunder.yml b/.github/workflows/Test Thunder.yml index 1e831f9df..6c589cdbf 100644 --- a/.github/workflows/Test Thunder.yml +++ b/.github/workflows/Test Thunder.yml @@ -40,9 +40,10 @@ jobs: sudo gem install apt-spy2 sudo apt-spy2 fix --commit --launchpad --country=US sudo apt-get update - sudo apt install python3-pip + sudo apt install python3-pip build-essential cmake ninja-build libusb-1.0-0-dev zlib1g-dev libssl-dev libgtest-dev + python3 -m venv venv + source venv/bin/activate pip install jsonref - sudo apt install build-essential cmake ninja-build libusb-1.0-0-dev zlib1g-dev libssl-dev libgtest-dev # ----- Checkout & DependsOn Regex ----- # -------------- Thunder --------------- @@ -98,12 +99,14 @@ jobs: # ----- Building ----- - name: Install generators run: | + source venv/bin/activate cmake -G Ninja -S ThunderTools -B ${{matrix.build_type}}/build/ThunderTools \ -DCMAKE_INSTALL_PREFIX=${{matrix.build_type}}/install/usr cmake --build ${{matrix.build_type}}/build/ThunderTools --target install - name: Build Thunder run: | + source venv/bin/activate cmake -G Ninja -S Thunder -B ${{matrix.build_type}}/build/Thunder \ -DBINDING="127.0.0.1" \ -DCMAKE_BUILD_TYPE=${{matrix.build_type}} \ diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index 5614b66b9..149085ea4 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -148,6 +148,8 @@ namespace PluginHost { }; public: + using Core::WorkerPool::Snapshot; + WorkerPoolImplementation() = delete; WorkerPoolImplementation(WorkerPoolImplementation&&) = delete; WorkerPoolImplementation(const WorkerPoolImplementation&) = delete; diff --git a/Source/core/Portability.h b/Source/core/Portability.h index b5b44cb90..cd419b3a4 100644 --- a/Source/core/Portability.h +++ b/Source/core/Portability.h @@ -190,6 +190,7 @@ #define DISABLE_WARNING_DELETE_INCOMPLETE #define DISABLE_WARNING_INCONSISTENT_MISSING_OVERRIDE #define DISABLE_WARNING_MAYBE_UNINITIALIZED +#define DISABLE_WARNING_FREE_NONHEAP_OBJECT #else #define DISABLE_WARNING_CONDITIONAL_EXPRESSION_IS_CONSTANT @@ -221,6 +222,7 @@ #define DISABLE_WARNING_DELETE_INCOMPLETE PUSH_WARNING_ARG_("-Wdelete-incomplete") #define DISABLE_WARNING_INCONSISTENT_MISSING_OVERRIDE PUSH_WARNING_ARG_("-Winconsistent-missing-override") #define DISABLE_WARNING_MAYBE_UNINITIALIZED PUSH_WARNING_ARG_("-Wmaybe-uninitialized") +#define DISABLE_WARNING_FREE_NONHEAP_OBJECT PUSH_WARNING_ARG_("-Wfree-nonheap-object") #endif #endif diff --git a/Source/core/Proxy.h b/Source/core/Proxy.h index 0b2830fe5..431cb4271 100644 --- a/Source/core/Proxy.h +++ b/Source/core/Proxy.h @@ -94,7 +94,9 @@ namespace Thunder { void* stAllocateBlock) { reinterpret_cast*>(stAllocateBlock)->__Destructed(); +PUSH_WARNING(DISABLE_WARNING_FREE_NONHEAP_OBJECT) ::free(stAllocateBlock); +POP_WARNING() } public: diff --git a/Source/core/SocketPort.h b/Source/core/SocketPort.h index d026b15bc..efadf89bb 100644 --- a/Source/core/SocketPort.h +++ b/Source/core/SocketPort.h @@ -276,7 +276,9 @@ namespace Thunder { return (((m_SocketType == LISTEN) || (m_SocketType == STREAM)) ? SOCK_STREAM : ((m_SocketType == DATAGRAM) ? SOCK_DGRAM : (m_SocketType == SEQUENCED ? SOCK_SEQPACKET : SOCK_RAW))); } uint16_t Events() override; +PUSH_WARNING(DISABLE_WARNING_OVERLOADED_VIRTUALS) void Handle(const uint16_t events) override; +POP_WARNING() bool Closed(); void Opened(); void Accepted(); From 2ab41876b531735620fda0aec0f6d9b180088f72 Mon Sep 17 00:00:00 2001 From: Mateusz Daniluk <121170681+VeithMetro@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:54:42 +0200 Subject: [PATCH 17/18] Use the Linux build template from master again after it was fixed (#1776) --- .github/workflows/Build Thunder on Linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build Thunder on Linux.yml b/.github/workflows/Build Thunder on Linux.yml index 6255548b4..1cbe4b686 100644 --- a/.github/workflows/Build Thunder on Linux.yml +++ b/.github/workflows/Build Thunder on Linux.yml @@ -17,7 +17,7 @@ on: jobs: Thunder: - uses: rdkcentral/Thunder/.github/workflows/Linux build template.yml@development/actions-pep668-error # change back to master after Actions work again + uses: rdkcentral/Thunder/.github/workflows/Linux build template.yml@master ThunderInterfaces: needs: Thunder From f3191fd09f8280d60c165308fe9471c801596f92 Mon Sep 17 00:00:00 2001 From: Mateusz Daniluk <121170681+VeithMetro@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:25:27 +0200 Subject: [PATCH 18/18] Explicitly state the ubuntu version (#1777) Seems like the Ubuntu-latest for some reason can reference to different Ubuntu versions between different repositories, causing serious compatibility issues with installing some packages like GCC mutilib --- .github/workflows/Linux build template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Linux build template.yml b/.github/workflows/Linux build template.yml index 99f918538..0ff63c1e2 100644 --- a/.github/workflows/Linux build template.yml +++ b/.github/workflows/Linux build template.yml @@ -5,7 +5,7 @@ on: jobs: Thunder: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: