From 2d9f8cc3a28de635c1c23e8668d4d7c2e60c7c62 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 24 Jun 2024 18:22:51 +0200 Subject: [PATCH 01/25] Don't auto-log HTTP exceptions. --- libs/httpcl/include/httpcl/http-client.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/httpcl/include/httpcl/http-client.hpp b/libs/httpcl/include/httpcl/http-client.hpp index 7d75b92..95596e4 100644 --- a/libs/httpcl/include/httpcl/http-client.hpp +++ b/libs/httpcl/include/httpcl/http-client.hpp @@ -37,9 +37,7 @@ class IHttpClient Error(Result result, std::string const& message) : std::runtime_error(message) , result(std::move(result)) - { - log().error(message); - } + {} }; virtual ~IHttpClient() = default; From 38b40ca9519f11cf28f5ac7674ca426595676ba6 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 2 Jul 2024 12:35:47 +0200 Subject: [PATCH 02/25] Bump version. --- CMakeLists.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe4e9d3..b4d98a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ project(zswag) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(ZSWAG_VERSION 1.6.7post1) +set(ZSWAG_VERSION 1.7.0) option(ZSWAG_BUILD_WHEELS "Enable zswag whl-output to WHEEL_DEPLOY_DIRECTORY." ON) option(ZSWAG_KEYCHAIN_SUPPORT "Enable zswag keychain support." ON) diff --git a/requirements.txt b/requirements.txt index 55735de..349e900 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ connexion~=2.14.2 requests zserio<3.0.0 pyyaml -pyzswagcl==1.6.7post1 +pyzswagcl==1.7.0 openapi-spec-validator From a6f3f87afeee487dedaecd97a1dd7b633f8cb382 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 4 Jul 2024 19:41:07 +0200 Subject: [PATCH 03/25] Explicitely support zlib. --- CMakeLists.txt | 5 ++++- conanfile.txt | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4d98a7..bf535a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,8 @@ if (NOT TARGET Catch2) endif() if (NOT TARGET httplib) + find_package(ZLIB CONFIG REQUIRED) + set(HTTPLIB_REQUIRE_ZLIB ON CACHE BOOL "Do not use zlib when compiling httplib." FORCE) FetchContent_Declare(httplib GIT_REPOSITORY "https://github.com/yhirose/cpp-httplib.git" GIT_TAG "v0.15.3" @@ -120,7 +122,8 @@ if (NOT TARGET httplib) target_compile_definitions(httplib INTERFACE CPPHTTPLIB_OPENSSL_SUPPORT) - target_link_libraries(httplib INTERFACE OpenSSL::SSL) + target_link_libraries( + httplib INTERFACE OpenSSL::SSL ZLIB::ZLIB) endif() if(ZSWAG_BUILD_WHEELS AND NOT TARGET pybind11) diff --git a/conanfile.txt b/conanfile.txt index 837e863..dc86dd6 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -3,9 +3,10 @@ openssl/3.2.0 keychain/1.2.1 spdlog/1.11.0 pybind11/2.10.4 +zlib/1.2.13 [generators] CMakeDeps [options] -openssl*:shared=True +openssl*:shared=False From cd20e5cc27dbcf64184d8d873c25c3e97af0e37c Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 4 Jul 2024 19:42:04 +0200 Subject: [PATCH 04/25] Simplify SSL support by linking statically. --- CMakeLists.txt | 49 ----------------------------------- libs/pyzswagcl/CMakeLists.txt | 18 +------------ 2 files changed, 1 insertion(+), 66 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf535a4..2d13c55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,55 +163,6 @@ if (ZSWAG_BUILD_WHEELS) add_dependencies(wheel zswag-server-wheel) endif() -############## -# deploy openssl libs - -if (WIN32) - set(OPENSSL_DEPLOY_DIR "${ZSWAG_DEPLOY_DIR}/${CMAKE_BUILD_TYPE}") -else() - set(OPENSSL_DEPLOY_DIR "${ZSWAG_DEPLOY_DIR}") -endif() - -message(STATUS "Deploying to ${OPENSSL_DEPLOY_DIR}") -message(STATUS "OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}") - -if(APPLE) - set(OPENSSL_LIB_DIR "${OPENSSL_INCLUDE_DIR}/../lib") - set(OPENSSL_LIBS - "${OPENSSL_LIB_DIR}/libcrypto.3.dylib" - "${OPENSSL_LIB_DIR}/libssl.3.dylib" - ) -elseif(MSVC) - set(OPENSSL_LIB_DIR "${OPENSSL_INCLUDE_DIR}/../bin") - set(OPENSSL_LIBS - "${OPENSSL_LIB_DIR}/libcrypto-3-x64.dll" - "${OPENSSL_LIB_DIR}/libssl-3-x64.dll" - ) -elseif(UNIX AND NOT APPLE) - set(OPENSSL_LIB_DIR "${OPENSSL_INCLUDE_DIR}/../lib") - set(OPENSSL_LIBS - "${OPENSSL_LIB_DIR}/libcrypto.so.3" - "${OPENSSL_LIB_DIR}/libssl.so.3" - ) -endif() - -foreach(file_i ${OPENSSL_LIBS}) - get_filename_component(filename ${file_i} NAME) - add_custom_command( - OUTPUT "${OPENSSL_DEPLOY_DIR}/${filename}" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${file_i}" - "${OPENSSL_DEPLOY_DIR}/${filename}" - DEPENDS "${file_i}" - COMMENT "Copying ${file_i} to ${OPENSSL_DEPLOY_DIR}" - ) - list(APPEND COPIED_OPENSSL_LIBS "${OPENSSL_DEPLOY_DIR}/${filename}") -endforeach(file_i) - -add_custom_target(copy_openssl_libs ALL DEPENDS ${COPIED_OPENSSL_LIBS}) - -add_dependencies(zswagcl copy_openssl_libs) - diff --git a/libs/pyzswagcl/CMakeLists.txt b/libs/pyzswagcl/CMakeLists.txt index 2c0e8f2..43f8a94 100644 --- a/libs/pyzswagcl/CMakeLists.txt +++ b/libs/pyzswagcl/CMakeLists.txt @@ -19,26 +19,10 @@ target_compile_features(pyzswagcl INTERFACE cxx_std_17) -if (MSVC) - # Required because cpp-httplib speaks https via OpenSSL - set (DEPLOY_FILES - "${OPENSSL_INCLUDE_DIR}/../bin/libcrypto-3-x64.dll" - "${OPENSSL_INCLUDE_DIR}/../bin/libssl-3-x64.dll") -endif() - -if (APPLE) - # Required because cpp-httplib speaks https via OpenSSL - set (DEPLOY_FILES - "${OPENSSL_INCLUDE_DIR}/../lib/libcrypto.3.dylib" - "${OPENSSL_INCLUDE_DIR}/../lib/libssl.3.dylib") -endif() - add_wheel(pyzswagcl AUTHOR "Navigation Data Standard e.V." URL "https://github.com/ndsev/zswag" VERSION "${ZSWAG_VERSION}" DESCRIPTION "Python bindings for the zswag client library." TARGET_DEPENDENCIES - zswagcl speedyj httpcl - DEPLOY_FILES - ${DEPLOY_FILES}) + zswagcl speedyj httpcl) From 28f508723db4e96537cca25c38f1a4d887abb233 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 4 Jul 2024 19:43:25 +0200 Subject: [PATCH 05/25] Support multiple servers in the OpenAPI spec. --- libs/httpcl/src/http-settings.cpp | 4 ++-- libs/pyzswagcl/py-openapi-client.cpp | 9 +++++---- libs/pyzswagcl/py-openapi-client.h | 3 ++- libs/pyzswagcl/py-zswagcl.cpp | 12 +++++++++--- .../zswagcl/private/openapi-client.hpp | 4 +++- .../zswagcl/private/openapi-config.hpp | 2 +- libs/zswagcl/src/openapi-client.cpp | 19 +++++++++++++------ libs/zswagcl/src/openapi-parser.cpp | 19 ++++++++++++------- 8 files changed, 47 insertions(+), 25 deletions(-) diff --git a/libs/httpcl/src/http-settings.cpp b/libs/httpcl/src/http-settings.cpp index 03f0f5b..71dfc97 100644 --- a/libs/httpcl/src/http-settings.cpp +++ b/libs/httpcl/src/http-settings.cpp @@ -389,7 +389,7 @@ void Config::apply(httplib::Client &cl) const // Proxy Settings if (proxy) { - cl.set_proxy(proxy->host.c_str(), proxy->port); + cl.set_proxy(proxy->host, proxy->port); auto password = proxy->password; if (!proxy->keychain.empty()) @@ -397,7 +397,7 @@ void Config::apply(httplib::Client &cl) const if (!proxy->user.empty()) cl.set_proxy_basic_auth( - proxy->user.c_str(), password.c_str()); + proxy->user, password); } cl.set_default_headers(httpLibHeaders); diff --git a/libs/pyzswagcl/py-openapi-client.cpp b/libs/pyzswagcl/py-openapi-client.cpp index 7b66116..62ead3f 100644 --- a/libs/pyzswagcl/py-openapi-client.cpp +++ b/libs/pyzswagcl/py-openapi-client.cpp @@ -65,9 +65,9 @@ namespace void PyOpenApiClient::bind(py::module_& m) { auto serviceClient = py::class_(m, "OAClient") - .def(py::init, std::optional>(), + .def(py::init, std::optional, std::optional>(), "url"_a, "is_local_file"_a = false, "config"_a = httpcl::Config(), - "api_key"_a = std::optional(), "bearer"_a = std::optional()) + "api_key"_a = std::optional(), "bearer"_a = std::optional(), "server_index"_a = std::optional()) // zserio >= 2.3.0 .def("call_method", &PyOpenApiClient::callMethod, "method_name"_a, "request"_a, "unused"_a) @@ -83,7 +83,8 @@ PyOpenApiClient::PyOpenApiClient(std::string const& openApiUrl, bool isLocalFile, httpcl::Config const& config, std::optional apiKey, - std::optional bearer) + std::optional bearer, + std::optional serverIndex) { auto httpConfig = config; // writable copy if (apiKey) @@ -100,7 +101,7 @@ PyOpenApiClient::PyOpenApiClient(std::string const& openApiUrl, return fetchOpenAPIConfig(openApiUrl, *httpClient, httpConfig); }(); - client_ = std::make_unique(openApiConfig, httpConfig, std::move(httpClient)); + client_ = std::make_unique(openApiConfig, httpConfig, std::move(httpClient), *serverIndex); } std::vector PyOpenApiClient::callMethod( diff --git a/libs/pyzswagcl/py-openapi-client.h b/libs/pyzswagcl/py-openapi-client.h index 42dc11e..93ec934 100644 --- a/libs/pyzswagcl/py-openapi-client.h +++ b/libs/pyzswagcl/py-openapi-client.h @@ -18,7 +18,8 @@ class PyOpenApiClient bool isLocalFile, httpcl::Config const& config, std::optional apiKey, - std::optional bearer); + std::optional bearer, + std::optional serverIndex); std::vector callMethod( const std::string& methodName, diff --git a/libs/pyzswagcl/py-zswagcl.cpp b/libs/pyzswagcl/py-zswagcl.cpp index 0a83773..afca395 100644 --- a/libs/pyzswagcl/py-zswagcl.cpp +++ b/libs/pyzswagcl/py-zswagcl.cpp @@ -122,11 +122,17 @@ PYBIND11_MODULE(pyzswagcl, m) auto it = self.methodPath.find(methodName); if (it != self.methodPath.end()) return it->second; - else - throw std::runtime_error( - "Could not find OpenAPI config for method name "s+methodName); + throw std::runtime_error( + "Could not find OpenAPI config for method name "s+methodName); }, py::is_operator(), py::return_value_policy::reference_internal, "method_name"_a) .def_readonly("content", &OpenAPIConfig::content) + .def_property_readonly("servers", [](const OpenAPIConfig& self) -> std::vector + { + std::vector result; + for (auto const& uri : self.servers) + result.emplace_back(uri.build()); + return result; + }, py::return_value_policy::automatic) ; m.def("parse_openapi_config", [](std::string const& path){ diff --git a/libs/zswagcl/include/zswagcl/private/openapi-client.hpp b/libs/zswagcl/include/zswagcl/private/openapi-client.hpp index 212fa83..96a910a 100644 --- a/libs/zswagcl/include/zswagcl/private/openapi-client.hpp +++ b/libs/zswagcl/include/zswagcl/private/openapi-client.hpp @@ -20,7 +20,8 @@ class OpenAPIClient OpenAPIClient(OpenAPIConfig config, httpcl::Config httpConfig, - std::unique_ptr client); + std::unique_ptr client, + uint32_t serverIndex = 0); ~OpenAPIClient(); /** @@ -41,6 +42,7 @@ class OpenAPIClient private: std::unique_ptr client_; httpcl::Settings settings_; + httpcl::URIComponents server_; }; } diff --git a/libs/zswagcl/include/zswagcl/private/openapi-config.hpp b/libs/zswagcl/include/zswagcl/private/openapi-config.hpp index 90902bd..2597f97 100644 --- a/libs/zswagcl/include/zswagcl/private/openapi-config.hpp +++ b/libs/zswagcl/include/zswagcl/private/openapi-config.hpp @@ -183,7 +183,7 @@ struct OpenAPIConfig /** * URI parts. */ - httpcl::URIComponents uri; + std::vector servers; /** * Map from service method name to path configuration. diff --git a/libs/zswagcl/src/openapi-client.cpp b/libs/zswagcl/src/openapi-client.cpp index da6c658..b1ce38e 100644 --- a/libs/zswagcl/src/openapi-client.cpp +++ b/libs/zswagcl/src/openapi-client.cpp @@ -117,17 +117,24 @@ void checkSecurityAlternativesAndApplyApiKey(OpenAPIConfig::SecurityAlternatives OpenAPIClient::OpenAPIClient(OpenAPIConfig config, httpcl::Config httpConfig, - std::unique_ptr client) + std::unique_ptr client, + uint32_t serverIndex) : config_(std::move(config)) + , httpConfig_(std::move(httpConfig)) , client_(std::move(client)) - , httpConfig_(httpConfig) { - httpcl::log().debug("Instantiating OpenApiClient for node at '{}'", config_.uri.build()); + if (serverIndex >= config_.servers.size()) + throw httpcl::logRuntimeError( + fmt::format( + "The server {} index is out of bounds - there are only {}.", + serverIndex, + config_.servers.size())); + server_ = config.servers[serverIndex]; + httpcl::log().debug("Instantiating OpenApiClient for node at '{}'", server_.build()); assert(client_); } -OpenAPIClient::~OpenAPIClient() -{} +OpenAPIClient::~OpenAPIClient() = default; std::string OpenAPIClient::call(const std::string& methodIdent, const std::functionsecond; - httpcl::URIComponents uri(config_.uri); + auto uri = server_; uri.appendPath(resolvePath(method, paramCb)); std::string builtUri = uri.build(); std::string debugContext = stx::format("[{} {}]", method.httpMethod, uri.buildPath()); diff --git a/libs/zswagcl/src/openapi-parser.cpp b/libs/zswagcl/src/openapi-parser.cpp index a81af2b..f29c7aa 100644 --- a/libs/zswagcl/src/openapi-parser.cpp +++ b/libs/zswagcl/src/openapi-parser.cpp @@ -385,9 +385,9 @@ static void parseServer(const YAMLScope& serverNode, if (urlStr.empty()) { // Ignore empty URLs. } else if (urlStr.front() == '/') { - config.uri = httpcl::URIComponents::fromStrPath(urlStr); + config.servers.emplace_back(httpcl::URIComponents::fromStrPath(urlStr)); } else { - config.uri = httpcl::URIComponents::fromStrRfc3986(urlStr); + config.servers.emplace_back(httpcl::URIComponents::fromStrRfc3986(urlStr)); } } } @@ -452,11 +452,16 @@ OpenAPIConfig fetchOpenAPIConfig(const std::string& url, httpcl::log().debug("{} Parsing OpenAPI spec", debugContext); auto config = parseOpenAPIConfig(ss); - if (config.uri.scheme.empty()) - config.uri.scheme = uriParts.scheme; - if (config.uri.host.empty()) { - config.uri.host = uriParts.host; - config.uri.port = uriParts.port; + // Add a default server and add missing server uri parts. + if (config.servers.empty()) + config.servers.emplace_back(); + for (auto& server : config.servers) { + if (server.scheme.empty()) + server.scheme = uriParts.scheme; + if (server.host.empty()) { + server.host = uriParts.host; + server.port = uriParts.port; + } } httpcl::log().debug("{} Parsed spec has {} methods.", debugContext, config.methodPath.size()); From f624b4d69fef613e746c396dd9999294387e7d12 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Thu, 4 Jul 2024 19:43:56 +0200 Subject: [PATCH 06/25] Allow parameters which do not define z-zserio-request-part. --- libs/zswagcl/src/openapi-parser.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libs/zswagcl/src/openapi-parser.cpp b/libs/zswagcl/src/openapi-parser.cpp index f29c7aa..b396104 100644 --- a/libs/zswagcl/src/openapi-parser.cpp +++ b/libs/zswagcl/src/openapi-parser.cpp @@ -235,6 +235,22 @@ static void parseMethodParameter(YAMLScope const& parameterNode, OpenAPIConfig::Path& path) { auto nameNode = parameterNode.mandatoryChild("name"); + + if (!parameterNode[ZSERIO_REQUEST_PART]) { + // Ignore parameters which do not have x-zserio-request-part, but + // output a warning for such parameters if they are not optional. + // By default, OpenAPI treats all request parameters as optional. + // You can add required: true to mark a parameter as required. + if (auto requiredNode = parameterNode["required"]) { + if (requiredNode.as()) + httpcl::log().warn( + "The parameter {} does not have x-zserio-request-part and is not optional." + "Ensure that it is filled by passing additional HTTP settings.", + parameterNode.str()); + } + return; + } + auto& parameter = path.parameters[nameNode.as()]; parameter.ident = nameNode.as(); From c73dca05857d171447625d1f2826055f965e75af Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 5 Jul 2024 16:57:45 +0200 Subject: [PATCH 07/25] Support new flexible YAML settings file format. --- libs/httpcl/include/httpcl/http-settings.hpp | 10 +- libs/httpcl/src/http-settings.cpp | 98 ++++++++++++++++---- 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/libs/httpcl/include/httpcl/http-settings.hpp b/libs/httpcl/include/httpcl/http-settings.hpp index f62a0e5..125a3d6 100644 --- a/libs/httpcl/include/httpcl/http-settings.hpp +++ b/libs/httpcl/include/httpcl/http-settings.hpp @@ -5,6 +5,7 @@ #include #include #include +#include "yaml-cpp/yaml.h" namespace httpcl @@ -41,6 +42,10 @@ struct Config std::string keychain; }; + std::optional scope; + std::regex urlPattern; + std::string urlPatternString; + std::map cookies; std::optional auth; std::optional proxy; @@ -67,7 +72,7 @@ struct Config }; /** - * Loads settings from HTTP_SETTINGS_FILE. + * Loads/stores settings from/to HTTP_SETTINGS_FILE. * Allows returning config for a specific URL. */ struct Settings @@ -85,7 +90,8 @@ struct Settings /** * Map from URL pattern to some config values. */ - std::map settings; + std::vector settings; + YAML::Node document; }; struct secret diff --git a/libs/httpcl/src/http-settings.cpp b/libs/httpcl/src/http-settings.cpp index 71dfc97..4540ddd 100644 --- a/libs/httpcl/src/http-settings.cpp +++ b/libs/httpcl/src/http-settings.cpp @@ -112,9 +112,51 @@ struct convert } namespace { -YAML::Node configToNode(Config const& config, std::string const& url=".*") { + +std::string convertToRegex(const std::string& scope) { + std::string regexPattern = "^"; + for (char c : scope) { + switch (c) { + case '*': + regexPattern += ".*"; + break; + case '.': + regexPattern += "\\."; + break; + case '\\': + regexPattern += "\\\\"; + break; + case '^': + case '$': + case '|': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '?': + case '+': + case '-': + case '!': + regexPattern += '\\'; + regexPattern += c; + break; + default: + regexPattern += c; + break; + } + } + regexPattern += ".*$"; + return regexPattern; +} + +YAML::Node configToNode(Config const& config) { YAML::Node result; - result["url"] = url; + if (config.scope) + result["scope"] = *config.scope; + else + result["url"] = config.urlPatternString; if (!config.cookies.empty()) result["cookies"] = config.cookies; @@ -139,16 +181,21 @@ YAML::Node configToNode(Config const& config, std::string const& url=".*") { return result; } -std::pair configFromNode(YAML::Node const& node) +Config configFromNode(YAML::Node const& node) { - std::string urlPattern; Config conf; - if (auto entryParam = node["url"]) - urlPattern = entryParam.as(); - else - throw std::runtime_error( - "HTTP Settings: Missing 'url' field in: " + YAML::Dump(node)); + if (auto entryParam = node["url"]) { + conf.urlPattern = conf.urlPatternString = entryParam.as(); + } + else { + if (auto entryParamScope = node["scope"]) + conf.scope = entryParamScope.as(); + else + conf.scope = "*"; + conf.urlPatternString = convertToRegex(*conf.scope); + conf.urlPattern = conf.urlPatternString; + } if (auto cookies = node["cookies"]) conf.cookies = cookies.as>(); @@ -172,7 +219,7 @@ std::pair configFromNode(YAML::Node const& node) if (auto apiKey = node["api-key"]) conf.apiKey = apiKey.as(); - return {std::move(conf), std::move(urlPattern)}; + return conf; } } @@ -302,12 +349,20 @@ void Settings::load() try { log().debug("Loading HTTP settings from '{}'...", cookieJar); - auto node = YAML::LoadFile(cookieJar); + document = YAML::LoadFile(cookieJar); + YAML::Node httpSettingsNode; uint32_t idx = 0; - for (auto const& entry : node.as>()) { - auto [conf, urlPattern] = configFromNode(entry); - settings[urlPattern] = std::move(conf); + if (document.IsMap()) { + httpSettingsNode = document["http-settings"]; + } + else { + // Keep supporting the old format, where the root structure is a settings array. + httpSettingsNode = document; + } + + for (auto const& entry : httpSettingsNode.as>()) { + settings.emplace_back(configFromNode(entry)); ++idx; } @@ -330,9 +385,16 @@ void Settings::store() try { auto node = YAML::Node(); - for (const auto& [key, config] : settings) + for (const auto& config : settings) node.push_back(configToNode(config)); + if (document && document.IsMap()) { + document["http-settings"] = node; + } + else { + document = node; + } + log().debug("Saving HTTP settings to '{}'...", cookieJar); std::ofstream os(cookieJar); os << node; @@ -345,16 +407,16 @@ void Settings::store() Config::Config(const std::string& yamlConf) { YAML::Node parsedYaml = YAML::Load(yamlConf); - *this = configFromNode(parsedYaml).first; + *this = configFromNode(parsedYaml); } Config Settings::operator[] (const std::string &url) const { Config result; - for (auto const& [pattern, config] : settings) + for (auto const& config : settings) { - if (!std::regex_match(url, std::regex(pattern))) + if (!std::regex_match(url, config.urlPattern)) continue; result |= config; } From bfce116e866d23bc2338e9452127ffcb18eb0ba6 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Fri, 5 Jul 2024 17:03:13 +0200 Subject: [PATCH 08/25] Fix access-after-move. --- libs/zswagcl/src/openapi-client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/zswagcl/src/openapi-client.cpp b/libs/zswagcl/src/openapi-client.cpp index b1ce38e..9a8b7b7 100644 --- a/libs/zswagcl/src/openapi-client.cpp +++ b/libs/zswagcl/src/openapi-client.cpp @@ -129,7 +129,7 @@ OpenAPIClient::OpenAPIClient(OpenAPIConfig config, "The server {} index is out of bounds - there are only {}.", serverIndex, config_.servers.size())); - server_ = config.servers[serverIndex]; + server_ = config_.servers[serverIndex]; httpcl::log().debug("Instantiating OpenApiClient for node at '{}'", server_.build()); assert(client_); } From 4e6787123ab9d74e9754baef4e42921fee6d7776 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 8 Jul 2024 16:26:56 +0200 Subject: [PATCH 09/25] Fix CI problems. --- .github/workflows/cmake.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5c7be52..b068856 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-latest container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}:2023.2 steps: - - uses: actions/checkout@v2 - with: - submodules: recursive + # Use taiki-e/checkout-action, because the GH checkout actions require + # node20 which can't run on manylinux2014. + - uses: taiki-e/checkout-action@v1 - name: Cache Conan packages uses: actions/cache@v2 with: @@ -23,7 +23,7 @@ jobs: - name: Configure run: | python3 -m venv venv && . ./venv/bin/activate - pip install -U setuptools wheel pip conan==2.2.1 + pip install -U setuptools wheel pip conan==2.5.0 mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. - name: Build working-directory: build @@ -65,7 +65,7 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - run: python -m pip install setuptools wheel conan==2.2.1 + - run: python -m pip install setuptools wheel conan==2.5.0 - run: mkdir build - name: Build (macOS) if: matrix.os == 'macos-13' From f1ad5b4bbf424b90d601767be46ee62a69cee667 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 8 Jul 2024 22:41:45 +0200 Subject: [PATCH 10/25] Use node built into manylinux image instead of the Github version relying on newer glibc. --- .github/workflows/cmake.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index b068856..79d1650 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -8,18 +8,19 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] runs-on: ubuntu-latest - container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}:2023.2 + container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1 steps: - # Use taiki-e/checkout-action, because the GH checkout actions require - # node20 which can't run on manylinux2014. - - uses: taiki-e/checkout-action@v1 - - name: Cache Conan packages - uses: actions/cache@v2 - with: - path: ~/.conan/data - key: ${{ runner.os }}-conan-${{ hashFiles('**/conanfile.txt') }} - restore-keys: | - ${{ runner.os }}-conan- + - name: Symlink Node.js + run: | + # Find the Node.js installation + NODE_DIR=$(dirname $(which node)) + # Create the symlink + mkdir -p /__e && ln -s ${NODE_DIR} /__e/node20 + # Verify the symlink + ls -l /__e + node -v && npm -v + - name: Checkout code + uses: actions/checkout@v2 - name: Configure run: | python3 -m venv venv && . ./venv/bin/activate From ab0c060a90cb188f5b66f391034259d6ed42cff8 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 8 Jul 2024 22:54:56 +0200 Subject: [PATCH 11/25] Add Python 3.12 to CI. --- .github/workflows/cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 79d1650..7a3f8be 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -6,7 +6,7 @@ jobs: build-manylinux: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] runs-on: ubuntu-latest container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1 steps: @@ -50,7 +50,7 @@ jobs: strategy: matrix: os: [macos-13, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 with: From c96ca7f21ba93c5a91fc50687292ecd4f48a6015 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 8 Jul 2024 23:10:09 +0200 Subject: [PATCH 12/25] Correctly symlink node20 --- .github/workflows/cmake.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 7a3f8be..26d74a4 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -15,7 +15,8 @@ jobs: # Find the Node.js installation NODE_DIR=$(dirname $(which node)) # Create the symlink - mkdir -p /__e && ln -s ${NODE_DIR} /__e/node20 + mv -f /__e/node20 /__e/node20.bak + ln -sf "${NODE_DIR}" /__e/node20 # Verify the symlink ls -l /__e node -v && npm -v From 0bca7537d477f2593f9a393590b667a05272d2bf Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 00:14:44 +0200 Subject: [PATCH 13/25] Use Github-provisioned Node after all, no chance to override it. --- .github/workflows/cmake.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 26d74a4..8523c08 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -9,17 +9,12 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] runs-on: ubuntu-latest container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - name: Symlink Node.js + - name: Which Node.js? run: | - # Find the Node.js installation - NODE_DIR=$(dirname $(which node)) - # Create the symlink - mv -f /__e/node20 /__e/node20.bak - ln -sf "${NODE_DIR}" /__e/node20 - # Verify the symlink - ls -l /__e - node -v && npm -v + echo "Node at $(which node): $(node -v); npm: $(npm -v)" - name: Checkout code uses: actions/checkout@v2 - name: Configure From 1cfee19af3315121ec0d7c0754dea49cbf139f29 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 01:14:15 +0200 Subject: [PATCH 14/25] Fix dependency version conflict. --- conanfile.py | 22 ++++++++++++++++++++++ conanfile.txt | 12 ------------ requirements.txt | 1 - setup.py | 2 +- 4 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 conanfile.py delete mode 100644 conanfile.txt diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..36b506a --- /dev/null +++ b/conanfile.py @@ -0,0 +1,22 @@ +from conan import ConanFile +from conan.tools.cmake import CMakeDeps + +class ZswagRecipe(ConanFile): + name = "zswag" + + # Specify the generators + generators = "CMakeDeps" + + # Specify options + default_options = { + "openssl*:shared": False + } + + def requirements(self): + self.requires("openssl/3.2.0") + self.requires("keychain/1.3.0") + self.requires("spdlog/1.11.0") + self.requires("pybind11/2.10.4") + self.requires("zlib/1.2.13") + # keychain and libsecret have a conflict here. + self.requires("glib/2.78.3", override=True) diff --git a/conanfile.txt b/conanfile.txt deleted file mode 100644 index dc86dd6..0000000 --- a/conanfile.txt +++ /dev/null @@ -1,12 +0,0 @@ -[requires] -openssl/3.2.0 -keychain/1.2.1 -spdlog/1.11.0 -pybind11/2.10.4 -zlib/1.2.13 - -[generators] -CMakeDeps - -[options] -openssl*:shared=False diff --git a/requirements.txt b/requirements.txt index 349e900..b7ad7b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,4 @@ connexion~=2.14.2 requests zserio<3.0.0 pyyaml -pyzswagcl==1.7.0 openapi-spec-validator diff --git a/setup.py b/setup.py index a423d15..33a8d5d 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ long_description = fh.read() required_url = [] -required = [] +required = [f"pyzswagcl=={VERSION}"] with open("requirements.txt", "r") as freq: for line in freq.read().split(): if "://" in line: From 36ba66bebf0e76c677400d83ca15131be298b093 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 09:44:23 +0200 Subject: [PATCH 15/25] Update conanfile.py --- conanfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index 36b506a..bc79822 100644 --- a/conanfile.py +++ b/conanfile.py @@ -3,8 +3,7 @@ class ZswagRecipe(ConanFile): name = "zswag" - - # Specify the generators + settings = "os", "arch", "compiler", "build_type" generators = "CMakeDeps" # Specify options From 1afc220e249d9f466e34dd9f67ab873e962eba98 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 10:09:47 +0200 Subject: [PATCH 16/25] Fix ctest no-tests parameter. --- .github/workflows/cmake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8523c08..6a52cb1 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -35,7 +35,7 @@ jobs: working-directory: build run: | . ../venv/bin/activate - ctest -C Release --verbose --no-test=fail + ctest -C Release --verbose --no-tests=error - name: Deploy uses: actions/upload-artifact@v2 with: @@ -95,4 +95,4 @@ jobs: - name: Test working-directory: build run: | - ctest -C Release --verbose --no-test=fail + ctest -C Release --verbose --no-tests=error From 997b0b5f760b99341b79966adf3a4166d28096d7 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 10:21:10 +0200 Subject: [PATCH 17/25] Enforce that testing is enabled. --- .github/workflows/cmake.yml | 7 ++++--- CMakeLists.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 6a52cb1..e68199e 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -21,7 +21,7 @@ jobs: run: | python3 -m venv venv && . ./venv/bin/activate pip install -U setuptools wheel pip conan==2.5.0 - mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release .. + mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DZSWAG_ENABLE_TESTING=ON .. - name: Build working-directory: build run: | @@ -73,7 +73,8 @@ jobs: cmake -DPython3_ROOT_DIR=$pythonLocation \ -DPython3_FIND_FRAMEWORK=LAST \ -DCMAKE_BUILD_TYPE=Release \ - -DHTTPLIB_USE_BROTLI_IF_AVAILABLE=OFF .. + -DHTTPLIB_USE_BROTLI_IF_AVAILABLE=OFF \ + -DZSWAG_ENABLE_TESTING=ON .. cmake --build . mv bin/wheel bin/wheel-auditme # Same as on Linux mkdir bin/wheel && mv bin/wheel-auditme/zswag*.whl bin/wheel @@ -85,7 +86,7 @@ jobs: working-directory: build run: | echo "cmake -DPython3_ROOT_DIR=$env:pythonLocation" - cmake "-DPython3_ROOT_DIR=$env:pythonLocation" -DPython3_FIND_REGISTRY=LAST -DHTTPLIB_USE_ZLIB_IF_AVAILABLE=OFF -DCMAKE_BUILD_TYPE=Release .. + cmake "-DPython3_ROOT_DIR=$env:pythonLocation" -DPython3_FIND_REGISTRY=LAST -DCMAKE_BUILD_TYPE=Release -DZSWAG_ENABLE_TESTING=ON .. cmake --build . --config Release - name: Deploy uses: actions/upload-artifact@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d13c55..a56b906 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ option(ZSWAG_KEYCHAIN_SUPPORT "Enable zswag keychain support." ON) option(ZSWAG_ENABLE_TESTING "Enable testing for the project" OFF) if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + message (STATUS "Testing will be enabled as zswag is the top-level project.") set (ZSWAG_ENABLE_TESTING ON CACHE BOOL "By default, enable testing if this is the main project") endif() From 4efa1326ee59d430db263a11608b35257fe6ddc7 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 10:49:54 +0200 Subject: [PATCH 18/25] Update docs for http-settings. --- README.md | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 09cda10..2bbabd4 100644 --- a/README.md +++ b/README.md @@ -581,26 +581,28 @@ variable. The YAML file contains a list of HTTP-related configs that are applied to HTTP requests based on a regular expression which is matched against the requested URL. -For example, the following entry would match all requests due to the `.*` -url-match-pattern: +For example, the following entry would match all requests due to the `*` +url-match-pattern for the `scope` field: ```yaml -- url: .* - basic-auth: - user: johndoe - keychain: keychain-service-string - proxy: - host: localhost - port: 8888 - user: test - keychain: ... - cookies: - key: value - headers: - key: value - query: - key: value - api-key: value +http-settings: + # Under http-settings, a list of settings is defined for specific URL scopes. + - scope: * # URL scope - e.g. https://*.nds.live/* or *.google.com. + basic-auth: # Basic auth credentials for matching requests. + user: johndoe + keychain: keychain-service-string + proxy: # Proxy settings for matching requests. + host: localhost + port: 8888 + user: test + keychain: ... + cookies: # Additional Cookies for matching requests. + key: value + headers: # Additional Headers for matching requests. + key: value + query: # Additional Query parameters for matching requests. + key: value + api-key: value # API Key as required by OpenAPI config - see description below. ``` **Note:** For `proxy` configs, the credentials are optional. From 64dbd206e5e6a87ec22e3c27881f6a8c9b74a993 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 10:50:13 +0200 Subject: [PATCH 19/25] Fix server index access in HTTP client. --- libs/pyzswagcl/py-openapi-client.cpp | 6 +++++- libs/zswagcl/src/openapi-client.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libs/pyzswagcl/py-openapi-client.cpp b/libs/pyzswagcl/py-openapi-client.cpp index 62ead3f..5b5fe0c 100644 --- a/libs/pyzswagcl/py-openapi-client.cpp +++ b/libs/pyzswagcl/py-openapi-client.cpp @@ -101,7 +101,11 @@ PyOpenApiClient::PyOpenApiClient(std::string const& openApiUrl, return fetchOpenAPIConfig(openApiUrl, *httpClient, httpConfig); }(); - client_ = std::make_unique(openApiConfig, httpConfig, std::move(httpClient), *serverIndex); + client_ = std::make_unique( + openApiConfig, + httpConfig, + std::move(httpClient), + serverIndex ? *serverIndex : 0); } std::vector PyOpenApiClient::callMethod( diff --git a/libs/zswagcl/src/openapi-client.cpp b/libs/zswagcl/src/openapi-client.cpp index 9a8b7b7..d8dce3f 100644 --- a/libs/zswagcl/src/openapi-client.cpp +++ b/libs/zswagcl/src/openapi-client.cpp @@ -126,7 +126,7 @@ OpenAPIClient::OpenAPIClient(OpenAPIConfig config, if (serverIndex >= config_.servers.size()) throw httpcl::logRuntimeError( fmt::format( - "The server {} index is out of bounds - there are only {}.", + "The server index {} is out of bounds (servers.size()={}).", serverIndex, config_.servers.size())); server_ = config_.servers[serverIndex]; From 6b588aab9ecce359ee24e88d6299189929b7ee7e Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Tue, 9 Jul 2024 13:02:13 +0200 Subject: [PATCH 20/25] Disable Python 3.12 support for now, waiting for new zserio release with Enum fix. --- .github/workflows/cmake.yml | 4 ++-- libs/zswag/reflect.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index e68199e..eaed021 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -6,7 +6,7 @@ jobs: build-manylinux: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11"] runs-on: ubuntu-latest container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:2024.1 env: @@ -46,7 +46,7 @@ jobs: strategy: matrix: os: [macos-13, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 with: diff --git a/libs/zswag/reflect.py b/libs/zswag/reflect.py index bf5ffa9..471617d 100644 --- a/libs/zswag/reflect.py +++ b/libs/zswag/reflect.py @@ -176,7 +176,7 @@ def str_to_bytes(s: str, fmt: OAParamFormat) -> bytes: # Convert a single passed parameter value to it's correct type def parse_param_value(param: OAParam, target_type: Type, value: str) -> Any: - # Check if its an enum ... + # Check if it's an enum ... if issubclass(target_type, Enum): return target_type(parse_param_value(param, int, value)) # Check if the parameter format is native string conversion From 2dd3b75943214c1ebdf742e902e37b46db04a3e0 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 15 Jul 2024 17:15:46 +0200 Subject: [PATCH 21/25] Improve ZLIB integration. --- CMakeLists.txt | 8 +++++--- libs/httpcl/include/httpcl/http-settings.hpp | 2 ++ libs/zswag/test/calc/api.yaml | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a56b906..97b19ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,8 +113,10 @@ if (NOT TARGET Catch2) endif() if (NOT TARGET httplib) - find_package(ZLIB CONFIG REQUIRED) - set(HTTPLIB_REQUIRE_ZLIB ON CACHE BOOL "Do not use zlib when compiling httplib." FORCE) + if (NOT TARGET ZLIB::ZLIB) + find_package(ZLIB CONFIG REQUIRED) + endif () + set (HTTPLIB_IS_USING_ZLIB TRUE) FetchContent_Declare(httplib GIT_REPOSITORY "https://github.com/yhirose/cpp-httplib.git" GIT_TAG "v0.15.3" @@ -122,7 +124,7 @@ if (NOT TARGET httplib) FetchContent_MakeAvailable(httplib) target_compile_definitions(httplib INTERFACE - CPPHTTPLIB_OPENSSL_SUPPORT) + CPPHTTPLIB_OPENSSL_SUPPORT) target_link_libraries( httplib INTERFACE OpenSSL::SSL ZLIB::ZLIB) endif() diff --git a/libs/httpcl/include/httpcl/http-settings.hpp b/libs/httpcl/include/httpcl/http-settings.hpp index 125a3d6..014ba27 100644 --- a/libs/httpcl/include/httpcl/http-settings.hpp +++ b/libs/httpcl/include/httpcl/http-settings.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "yaml-cpp/yaml.h" diff --git a/libs/zswag/test/calc/api.yaml b/libs/zswag/test/calc/api.yaml index d7d0ff4..42eea66 100644 --- a/libs/zswag/test/calc/api.yaml +++ b/libs/zswag/test/calc/api.yaml @@ -273,4 +273,5 @@ paths: summary: '' security: - HeaderAuth: [] -servers: [] +servers: + - url: / From bd5f1f24483612a944607dab0fb968c3335851dc Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 15 Jul 2024 17:16:22 +0200 Subject: [PATCH 22/25] Add httpcl::Settings::updateTimestamp --- libs/httpcl/include/httpcl/http-settings.hpp | 9 +++++++++ libs/httpcl/src/http-settings.cpp | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/libs/httpcl/include/httpcl/http-settings.hpp b/libs/httpcl/include/httpcl/http-settings.hpp index 014ba27..14f55bf 100644 --- a/libs/httpcl/include/httpcl/http-settings.hpp +++ b/libs/httpcl/include/httpcl/http-settings.hpp @@ -94,6 +94,15 @@ struct Settings */ std::vector settings; YAML::Node document; + mutable std::shared_mutex mutex; + std::chrono::steady_clock::time_point lastRead; + + /** + * Prompt settings instance to re-parse the HTTP settings file, + * by calling updateTimestamp with std::chrono::steady_clock::now(). + */ + static void updateTimestamp(std::chrono::steady_clock::time_point time); + static std::atomic lastUpdated; }; struct secret diff --git a/libs/httpcl/src/http-settings.cpp b/libs/httpcl/src/http-settings.cpp index 4540ddd..594ea57 100644 --- a/libs/httpcl/src/http-settings.cpp +++ b/libs/httpcl/src/http-settings.cpp @@ -332,8 +332,16 @@ Settings::Settings() load(); } +std::atomic Settings::lastUpdated{std::chrono::steady_clock::now()}; + +void Settings::updateTimestamp(std::chrono::steady_clock::time_point time) { + lastUpdated.store(time, std::memory_order_relaxed); +} + void Settings::load() { + std::unique_lock lock(mutex); + lastRead = std::chrono::steady_clock::now(); settings.clear(); auto cookieJar = std::getenv("HTTP_SETTINGS_FILE"); @@ -412,15 +420,20 @@ Config::Config(const std::string& yamlConf) Config Settings::operator[] (const std::string &url) const { - Config result; + std::shared_lock lock(mutex); + if (lastRead < lastUpdated.load(std::memory_order_relaxed)) { + lock.unlock(); + const_cast(this)->load(); + lock.lock(); + } + Config result; for (auto const& config : settings) { if (!std::regex_match(url, config.urlPattern)) continue; result |= config; } - return result; } From f3edafc569fa5bbe03056be42a66941e34334f6d Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Mon, 15 Jul 2024 17:17:28 +0200 Subject: [PATCH 23/25] Add test for serverIndex. --- libs/pyzswagcl/py-openapi-client.cpp | 3 +-- libs/zswag/test/calc/client.py | 2 +- libs/zswag/test/client.cpp | 30 ++++++++++++----------- libs/zswagcl/include/zswagcl/oaclient.hpp | 5 ++-- libs/zswagcl/src/oaclient.cpp | 5 ++-- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/libs/pyzswagcl/py-openapi-client.cpp b/libs/pyzswagcl/py-openapi-client.cpp index 5b5fe0c..cb06d99 100644 --- a/libs/pyzswagcl/py-openapi-client.cpp +++ b/libs/pyzswagcl/py-openapi-client.cpp @@ -97,8 +97,7 @@ PyOpenApiClient::PyOpenApiClient(std::string const& openApiUrl, std::ifstream fs(openApiUrl); return parseOpenAPIConfig(fs); } - else - return fetchOpenAPIConfig(openApiUrl, *httpClient, httpConfig); + return fetchOpenAPIConfig(openApiUrl, *httpClient, httpConfig); }(); client_ = std::make_unique( diff --git a/libs/zswag/test/calc/client.py b/libs/zswag/test/calc/client.py index e6b984b..d47eac7 100644 --- a/libs/zswag/test/calc/client.py +++ b/libs/zswag/test/calc/client.py @@ -21,7 +21,7 @@ def run_test(aspect, request, fn, expect, auth_args): try: print(f"[py-test-client] Test#{counter}: {aspect}", flush=True) print(f"[py-test-client] -> Instantiating client.", flush=True) - oa_client = OAClient(f"http://{host}:{port}/openapi.json", **auth_args) + oa_client = OAClient(f"http://{host}:{port}/openapi.json", **auth_args, server_index=0) # Just make sure that OpenAPI JSON content is parsable assert oa_client.config().content and json.loads(oa_client.config().content) client = api.Calculator.Client(oa_client) diff --git a/libs/zswag/test/client.cpp b/libs/zswag/test/client.cpp index 8b32d18..7df0032 100644 --- a/libs/zswag/test/client.cpp +++ b/libs/zswag/test/client.cpp @@ -19,7 +19,7 @@ int main (int argc, char* argv[]) { spdlog::info("[cpp-test-client] Starting integration tests with {}", specUrl); - auto runTest = [&] (auto const& fn, auto expect, std::string const& aspect, std::function const& authFun) + auto runTest = [&] (auto const& fn, auto expect, std::string const& aspect, std::function const& authFun) { ++testCounter; spdlog::info("[cpp-test-client] Executing test #{}: {} ...", testCounter, aspect); @@ -28,9 +28,11 @@ int main (int argc, char* argv[]) { spdlog::info("[cpp-test-client] => Instantiating client."); auto httpClient = std::make_unique(); auto openApiConfig = fetchOpenAPIConfig(specUrl, *httpClient); - httpcl::Config authHttpConf; + // See https://github.com/spec-first/connexion/issues/1139 + openApiConfig.servers.insert(openApiConfig.servers.begin(), URIComponents::fromStrPath("/bad/path/we/dont/access")); + Config authHttpConf; authFun(authHttpConf); - auto oaClient = OAClient(openApiConfig, std::move(httpClient), authHttpConf); + auto oaClient = OAClient(openApiConfig, std::move(httpClient), authHttpConf, 1); spdlog::info("[cpp-test-client] => Running request."); calculator::Calculator::Client calcClient(oaClient); auto response = fn(calcClient); @@ -49,13 +51,13 @@ int main (int argc, char* argv[]) { calculator::BaseAndExponent req(calculator::I32(2), calculator::I32(3), 0, "", .0, std::vector{}); return calcClient.powerMethod(req); }, 8., "Pass fields in path and header", - [](httpcl::Config& conf){}); + [](Config& conf){}); runTest([](calculator::Calculator::Client& calcClient){ calculator::Integers req(std::vector{100, -200, 400}); return calcClient.intSumMethod(req); }, 300., "Pass hex-encoded array in query", - [](httpcl::Config& conf){ + [](Config& conf){ conf.headers.insert({"Authorization", "Bearer 123"}); }); @@ -63,8 +65,8 @@ int main (int argc, char* argv[]) { calculator::Bytes req(std::vector{8, 16, 32, 64}); return calcClient.byteSumMethod(req); }, 120., "Pass base64url-encoded byte array in path", - [](httpcl::Config& conf){ - conf.auth = httpcl::Config::BasicAuthentication{ + [](Config& conf){ + conf.auth = Config::BasicAuthentication{ "u", "pw", "" }; }); @@ -73,7 +75,7 @@ int main (int argc, char* argv[]) { calculator::Integers req(std::vector{1, 2, 3, 4}); return calcClient.intMulMethod(req); }, 24., "Pass base64-encoded long array in path", - [](httpcl::Config& conf){ + [](Config& conf){ conf.query.insert({"api-key", "42"}); }); @@ -81,7 +83,7 @@ int main (int argc, char* argv[]) { calculator::Doubles req(std::vector{34.5, 2.}); return calcClient.floatMulMethod(req); }, 69., "Pass float array in query.", - [](httpcl::Config& conf){ + [](Config& conf){ conf.cookies.insert({"api-cookie", "42"}); }); @@ -89,7 +91,7 @@ int main (int argc, char* argv[]) { calculator::Bools req(std::vector{true, false}); return calcClient.bitMulMethod(req); }, false, "Pass bool array in query (expect false).", - [](httpcl::Config& conf){ + [](Config& conf){ conf.apiKey = "42"; }); @@ -97,7 +99,7 @@ int main (int argc, char* argv[]) { calculator::Bools req(std::vector{true, true}); return calcClient.bitMulMethod(req); }, true, "Pass bool array in query (expect true).", - [](httpcl::Config& conf){ + [](Config& conf){ conf.headers.insert({"X-Generic-Token", "42"}); }); @@ -105,7 +107,7 @@ int main (int argc, char* argv[]) { calculator::Double req(1.); return calcClient.identityMethod(req); }, 1., "Pass request as blob in body", - [](httpcl::Config& conf){ + [](Config& conf){ conf.cookies.insert({"api-cookie", "42"}); }); @@ -113,7 +115,7 @@ int main (int argc, char* argv[]) { calculator::Strings req(std::vector{"foo", "bar"}); return calcClient.concatMethod(req); }, std::string("foobar"), "Pass base64-encoded strings.", - [](httpcl::Config& conf){ + [](Config& conf){ conf.headers.insert({"Authorization", "Bearer 123"}); }); @@ -121,7 +123,7 @@ int main (int argc, char* argv[]) { calculator::EnumWrapper req(calculator::Enum::TEST_ENUM_0); return calcClient.nameMethod(req); }, std::string("TEST_ENUM_0"), "Pass enum.", - [](httpcl::Config& conf){ + [](Config& conf){ conf.apiKey = "42"; }); diff --git a/libs/zswagcl/include/zswagcl/oaclient.hpp b/libs/zswagcl/include/zswagcl/oaclient.hpp index 66b6b5b..3b00a33 100644 --- a/libs/zswagcl/include/zswagcl/oaclient.hpp +++ b/libs/zswagcl/include/zswagcl/oaclient.hpp @@ -12,9 +12,10 @@ class OAClient : public ::zserio::IServiceClient { public: OAClient( - zswagcl::OpenAPIConfig config, + OpenAPIConfig config, std::unique_ptr client, - httpcl::Config httpConfig = {}); + httpcl::Config httpConfig = {}, + uint32_t serverIndex = 0); std::vector callMethod( zserio::StringView methodName, diff --git a/libs/zswagcl/src/oaclient.cpp b/libs/zswagcl/src/oaclient.cpp index b539d3d..85a1e1f 100644 --- a/libs/zswagcl/src/oaclient.cpp +++ b/libs/zswagcl/src/oaclient.cpp @@ -9,8 +9,9 @@ namespace zswagcl OAClient::OAClient(zswagcl::OpenAPIConfig config, std::unique_ptr client, - httpcl::Config httpConfig) - : client_(std::move(config), std::move(httpConfig), std::move(client)) + httpcl::Config httpConfig, + uint32_t serverIndex) + : client_(std::move(config), std::move(httpConfig), std::move(client), serverIndex) {} template From b9cea63f75bb9b49665fb78db1c2fb065f62cbc2 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 17 Jul 2024 12:45:30 +0200 Subject: [PATCH 24/25] Catch exceptions from load(). --- libs/httpcl/src/http-settings.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/httpcl/src/http-settings.cpp b/libs/httpcl/src/http-settings.cpp index 594ea57..ffaa723 100644 --- a/libs/httpcl/src/http-settings.cpp +++ b/libs/httpcl/src/http-settings.cpp @@ -423,7 +423,13 @@ Config Settings::operator[] (const std::string &url) const std::shared_lock lock(mutex); if (lastRead < lastUpdated.load(std::memory_order_relaxed)) { lock.unlock(); - const_cast(this)->load(); + try { + const_cast(this)->load(); + } + catch (...) { + // If an exception occurred during load(), it was already + // logged in all likelihood. + } lock.lock(); } From 5e89717ebfad3a132aede702a720e9e6381be758 Mon Sep 17 00:00:00 2001 From: Joseph Birkner Date: Wed, 17 Jul 2024 14:47:09 +0200 Subject: [PATCH 25/25] Fix include guards. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 97b19ba..b440f97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,7 @@ if (ZSWAG_BUILD_WHEELS) endif() endif() -if (NOT TARGET yaml-cpp) +if (NOT TARGET yaml-cpp::yaml-cpp) FetchContent_Declare(yaml-cpp GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git" GIT_TAG "yaml-cpp-0.7.0" @@ -104,7 +104,7 @@ if (NOT TARGET speedyj) FetchContent_MakeAvailable(speedyj) endif() -if (NOT TARGET Catch2) +if (NOT TARGET Catch2::Catch2) FetchContent_Declare(Catch2 GIT_REPOSITORY "https://github.com/catchorg/Catch2.git" GIT_TAG "v3.4.0" @@ -112,7 +112,7 @@ if (NOT TARGET Catch2) FetchContent_MakeAvailable(Catch2) endif() -if (NOT TARGET httplib) +if (NOT TARGET httplib::httplib) if (NOT TARGET ZLIB::ZLIB) find_package(ZLIB CONFIG REQUIRED) endif ()