From 7ab80fb28b9071b386327eaad9198202fc7e50c1 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 12:35:47 -0700 Subject: [PATCH 01/74] Implement easy logging --- CMakeLists.txt | 5 +- cpp/lib/ClientConfigParser.cpp | 289 +++++++++++++++++++++++ include/snowflake/ClientConfigParser.hpp | 93 ++++++++ include/snowflake/client.h | 5 +- lib/client.c | 68 +++++- lib/client_config_parser.h | 44 ++++ lib/logger.c | 4 + tests/test_unit_logger.c | 50 +++- 8 files changed, 549 insertions(+), 9 deletions(-) create mode 100644 cpp/lib/ClientConfigParser.cpp create mode 100644 include/snowflake/ClientConfigParser.hpp create mode 100644 lib/client_config_parser.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dc63924798..bed60b95fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,6 +218,7 @@ set(SOURCE_FILES_CPP_WRAPPER include/snowflake/SFURL.hpp include/snowflake/CurlDesc.hpp include/snowflake/CurlDescPool.hpp + include/snowflake/ClientConfigParser.hpp cpp/lib/Exceptions.cpp cpp/lib/Connection.cpp cpp/lib/Statement.cpp @@ -240,6 +241,7 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ResultSetJson.hpp cpp/lib/Authenticator.cpp cpp/lib/Authenticator.hpp + cpp/lib/ClientConfigParser.cpp cpp/jwt/jwtWrapper.cpp cpp/util/SnowflakeCommon.cpp cpp/util/SFURL.cpp @@ -250,7 +252,8 @@ set(SOURCE_FILES_CPP_WRAPPER lib/result_set_json.h lib/query_context_cache.h lib/curl_desc_pool.h - lib/authenticator.h) + lib/authenticator.h + lib/client_config_parser.h) if (UNIX) if (LINUX) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp new file mode 100644 index 0000000000..78a7964c0c --- /dev/null +++ b/cpp/lib/ClientConfigParser.cpp @@ -0,0 +1,289 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#include "snowflake/ClientConfigParser.hpp" +#include "snowflake/Exceptions.hpp" +#include "../logger/SFLogger.hpp" +#include "client_config_parser.h" +#include "memory.h" + +#include +#include +#include + +#undef snprintf +#include +#include + +using namespace Snowflake::Client; + +namespace +{ + // constants + const std::string SF_CLIENT_CONFIG_FILE_NAME("sf_client_config.json"); + const std::string SF_CLIENT_CONFIG_ENV_NAME("SF_CLIENT_CONFIG_FILE"); + std::set KnownCommonEntries = {"log_level", "log_path"}; +} + +//////////////////////////////////////////////////////////////////////////////// +void load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig) +{ + ClientConfigParser configParser; + ClientConfig clientConfig; + configParser.loadClientConfig(in_configFilePath, clientConfig); + if (!clientConfig.logLevel.empty()) + { + out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); + strcpy(out_clientConfig->logLevel, clientConfig.logLevel.data()); + } + if (!clientConfig.logPath.empty()) + { + out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); + strcpy(out_clientConfig->logPath, clientConfig.logPath.data()); + } +} + +// Public ====================================================================== +//////////////////////////////////////////////////////////////////////////////// +ClientConfigParser::ClientConfigParser() +{ + ; // Do nothing. +} + +//////////////////////////////////////////////////////////////////////////////// +ClientConfigParser::~ClientConfigParser() +{ + ; // Do nothing. +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::loadClientConfig( + const std::string& in_configFilePath, + ClientConfig& out_clientConfig) +{ + char envbuf[MAX_PATH + 1]; + std::string derivedConfigPath = ""; + if (!in_configFilePath.empty()) + { + // 1. Try config file if it was passed in + derivedConfigPath = in_configFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from a connection string: %s", in_configFilePath.c_str()); + } + else if (const char* clientConfigEnv = sf_getenv_s(SF_CLIENT_CONFIG_ENV_NAME.c_str(), envbuf, sizeof(envbuf))) + { + // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME + derivedConfigPath = clientConfigEnv; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from an environment variable: %s", clientConfigEnv); + } + else + { + // 3. Try DLL binary dir + std::string binaryDir = getBinaryPath(); + std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(binaryDirFilePath)) + { + derivedConfigPath = binaryDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); + } + else + { +#if defined(WIN32) || defined(_WIN64) + // 4. Try user home dir + std::string homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); + if (homeDir.empty()) + { + // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH + std::string homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); + std::string homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); + if (!homeDriveEnv.empty() && !homePathEnv.empty()) + { + homeDir = homeDriveEnv + homePathEnv; + } + } + std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(homeDirFilePath)) + { + derivedConfigPath = homeDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + } +#else + // 4. Try user home dir + if (std::string homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + { + std::string homeDirFilePath = homeDir + SF_PATH_SEPARATOR + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(homeDirFilePath)) + { + derivedConfigPath = homeDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + } + } +#endif + } + } + if (!derivedConfigPath.empty()) + { + parseConfigFile(derivedConfigPath, out_clientConfig); + } +} + +// Private ===================================================================== +//////////////////////////////////////////////////////////////////////////////// +bool ClientConfigParser::checkFileExists(const std::string& in_filePath) +{ + FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); + if (file != nullptr) + { + fclose(file); + return true; + } + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::parseConfigFile( + const std::string& in_filePath, + ClientConfig& out_clientConfig) +{ + std::ifstream configFile; + cJSON* jsonConfig; + try + { + configFile.open(in_filePath); + if (!configFile) + { + CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", + "Could not open a file. The file may not exist: %s", + in_filePath.c_str()); + std::string errMsg = "Error finding client configuration file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } +#if !defined(WIN32) && !defined(_WIN64) + checkIfValidPermissions(in_filePath); +#endif + std::stringstream configJSON; + configJSON << configFile.rdbuf(); + jsonConfig = snowflake_cJSON_Parse(configJSON.str().c_str()); + const char* error_ptr = snowflake_cJSON_GetErrorPtr(); + if (error_ptr) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "parseConfigFile", + "Error in parsing JSON: %s, err: %s", in_filePath.c_str(), error_ptr); + std::string errMsg = "Error parsing client configuration file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } + } + catch (std::exception& ex) + { + configFile.close(); + throw; + } + + const cJSON* commonProps = snowflake_cJSON_GetObjectItem(jsonConfig, "common"); + checkUnknownEntries(snowflake_cJSON_Print(commonProps)); + const cJSON* logLevel = snowflake_cJSON_GetObjectItem(commonProps, "log_level"); + if (snowflake_cJSON_IsString(logLevel)) + { + out_clientConfig.logLevel = snowflake_cJSON_GetStringValue(logLevel); + } + const cJSON* logPath = snowflake_cJSON_GetObjectItem(commonProps, "log_path"); + if (snowflake_cJSON_IsString(logPath)) + { + out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); + } + configFile.close(); +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) +{ + boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); + boost::filesystem::perms permissions = fileStatus.permissions(); + if (permissions & boost::filesystem::group_write || + permissions & boost::filesystem::others_write) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "checkIfValidPermissions", + "Error due to other users having permission to modify the config file: %s", + in_filePath.c_str()); + std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) +{ + cJSON* jsonConfig = snowflake_cJSON_Parse(in_jsonString.c_str()); + const char* error_ptr = snowflake_cJSON_GetErrorPtr(); + if (error_ptr) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "checkUnknownEntries", + "Error in parsing JSON: %s, err: %s", in_jsonString.c_str(), error_ptr); + std::string errMsg = "Error parsing json: " + in_jsonString; + throw std::exception(errMsg.c_str()); + } + + if (snowflake_cJSON_IsObject(jsonConfig)) + { + cJSON* entry = NULL; + snowflake_cJSON_ArrayForEach(entry, jsonConfig) + { + std::string key = entry->string; + boost::algorithm::to_lower(key); + if (KnownCommonEntries.find(key) == KnownCommonEntries.end()) + { + std::string warnMsg = + "Unknown configuration entry: " + key + " with value:" + entry->valuestring; + CXX_LOG_WARN( + "sf", + "ClientConfigParser", + "checkUnknownEntries", + warnMsg); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +std::string ClientConfigParser::getBinaryPath() +{ + std::wstring binaryFullPath; + +#if defined(WIN32) || defined(_WIN64) + HMODULE hm = NULL; + wchar_t appName[256]; + GetModuleFileNameW(hm, appName, 256); +#else + Dl_info info; + int result = dladdr((void*)getBinaryPath, &info); + if (result) + { + binaryFullPath = info.dli_fname; + } +#endif + + binaryFullPath = appName; + size_t pos = binaryFullPath.find_last_of(PATH_SEP); + if (pos == std::wstring::npos) + { + return ""; + } + std::wstring binaryPath = binaryFullPath.substr(0, pos + 1); + return std::string(binaryPath.begin(), binaryPath.end()); +} diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp new file mode 100644 index 0000000000..9aebfe1ce0 --- /dev/null +++ b/include/snowflake/ClientConfigParser.hpp @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef PROJECT_CLIENTCONFIG_HPP +#define PROJECT_CLIENTCONFIG_HPP + +#include + +namespace Snowflake +{ +namespace Client +{ + struct ClientConfig + { + // The log level + std::string logLevel = ""; + + // The log path + std::string logPath = ""; + }; + + class ClientConfigParser + { + // Public ================================================================== + public: + /** + * Constructor for client config + */ + ClientConfigParser(); + + /// @brief Destructor. + ~ClientConfigParser(); + + /** + * Construct SFClientConfig from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The SFClientConfig object to be filled. + */ + void loadClientConfig( + const std::string& in_configFilePath, + ClientConfig& out_clientConfig); + + // Private ================================================================= + private: + /** + * @brief Check if the file exists. + * + * @param in_filePath The file path to check. + */ + bool checkFileExists(const std::string& in_filePath); + + /** + * @brief Parse JSON string. + * + * @param in_filePath The filePath of the config file to parse. + * @param out_clientConfig The SFClientConfig object to be filled. + */ + void parseConfigFile( + const std::string& in_filePath, + ClientConfig& out_clientConfig); + + /** + * @ brief Check if other have permission to modify file + * + * @param in_filePath The file path of the config file to check permissions. + */ + void checkIfValidPermissions(const std::string& in_filePath); + + /** + * @ brief Check if there are unknown entries in config file + * + * @param in_jsonString The json object to check in json config file. + */ + void checkUnknownEntries(const std::string& in_jsonString); + + /** + * @ brief Get the path to the binary file + */ + std::string getBinaryPath(); + }; + +} // namespace Client +} // namespace Snowflake + +#endif //PROJECT_CLIENTCONFIG_HPP diff --git a/include/snowflake/client.h b/include/snowflake/client.h index abd332f5be..74e0458540 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -277,7 +277,10 @@ typedef enum SF_GLOBAL_ATTRIBUTE { SF_GLOBAL_CA_BUNDLE_FILE, SF_GLOBAL_SSL_VERSION, SF_GLOBAL_DEBUG, - SF_GLOBAL_OCSP_CHECK + SF_GLOBAL_OCSP_CHECK, + SF_GLOBAL_CLIENT_CONFIG_FILE, + SF_GLOBAL_LOG_LEVEL, + SF_GLOBAL_LOG_PATH } SF_GLOBAL_ATTRIBUTE; /** diff --git a/lib/client.c b/lib/client.c index 90eb215469..e6d2cd60d8 100644 --- a/lib/client.c +++ b/lib/client.c @@ -19,6 +19,7 @@ #include "chunk_downloader.h" #include "authenticator.h" #include "query_context_cache.h" +#include "client_config_parser.h" #ifdef _WIN32 #include @@ -36,6 +37,8 @@ sf_bool DEBUG; sf_bool SF_OCSP_CHECK; char *SF_HEADER_USER_AGENT = NULL; +static char *CLIENT_CONFIG_FILE = NULL; +static char* LOG_LEVEL; static char *LOG_PATH = NULL; static FILE *LOG_FP = NULL; @@ -306,18 +309,34 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { SF_LOG_LEVEL sf_log_level = log_level; char strerror_buf[SF_ERROR_BUFSIZE]; + char client_config_file[MAX_PATH] = {0}; + snowflake_global_get_attribute( + SF_GLOBAL_CLIENT_CONFIG_FILE, client_config_file, sizeof(client_config_file)); + client_config clientConfig = { .logLevel = "", .logPath = "" }; + load_client_config(client_config_file, &clientConfig); + size_t log_path_size = 1; //Start with 1 to include null terminator log_path_size += strlen(time_str); - /* The environment variables takes precedence over the specified parameters */ - sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); - if (sf_log_path == NULL && log_path) { + /* The client config takes precedence over environmental variables */ + if (strlen(clientConfig.logPath) != 0) { + sf_log_path = clientConfig.logPath; + } else { + /* The environment variables takes precedence over the specified parameters */ + sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); + if (sf_log_path == NULL && log_path) { sf_log_path = log_path; + } } - sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); - if (sf_log_level_str != NULL) { + if (strlen(clientConfig.logLevel) != 0) { + sf_log_level = log_from_str_to_level(clientConfig.logLevel); + } else { + /* The client config takes precedence over environment variables */ + sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); + if (sf_log_level_str != NULL) { sf_log_level = log_from_str_to_level(sf_log_level_str); + } } // Set logging level @@ -350,6 +369,9 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { goto cleanup; } + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); + snowflake_global_set_attribute(SF_GLOBAL_LOG_LEVEL, log_from_level_to_str(sf_log_level)); + ret = SF_BOOLEAN_TRUE; cleanup: @@ -536,7 +558,6 @@ _snowflake_check_connection_parameters(SF_CONNECT *sf) { return SF_STATUS_SUCCESS; } - SF_STATUS STDCALL snowflake_global_init( const char *log_path, SF_LOG_LEVEL log_level, SF_USER_MEM_HOOKS *hooks) { SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -552,11 +573,13 @@ SF_STATUS STDCALL snowflake_global_init( _snowflake_memory_hooks_setup(hooks); sf_memory_init(); sf_error_init(); + if (!log_init(log_path, log_level)) { // no way to log error because log_init failed. sf_fprintf(stderr, "Error during log initialization"); goto cleanup; } + CURLcode curl_ret = curl_global_init(CURL_GLOBAL_DEFAULT); if (curl_ret != CURLE_OK) { log_fatal("curl_global_init() failed: %s", @@ -620,6 +643,15 @@ snowflake_global_set_attribute(SF_GLOBAL_ATTRIBUTE type, const void *value) { case SF_GLOBAL_OCSP_CHECK: SF_OCSP_CHECK = *(sf_bool *) value; break; + case SF_GLOBAL_CLIENT_CONFIG_FILE: + alloc_buffer_and_copy(&CLIENT_CONFIG_FILE, value); + break; + case SF_GLOBAL_LOG_LEVEL: + alloc_buffer_and_copy(&LOG_LEVEL, value); + break; + case SF_GLOBAL_LOG_PATH: + alloc_buffer_and_copy(&LOG_PATH, value); + break; default: break; } @@ -649,6 +681,30 @@ snowflake_global_get_attribute(SF_GLOBAL_ATTRIBUTE type, void *value, size_t siz case SF_GLOBAL_OCSP_CHECK: *((sf_bool *) value) = SF_OCSP_CHECK; break; + case SF_GLOBAL_CLIENT_CONFIG_FILE: + if (CLIENT_CONFIG_FILE) { + if (strlen(CLIENT_CONFIG_FILE) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, CLIENT_CONFIG_FILE, size); + } + break; + case SF_GLOBAL_LOG_LEVEL: + if (LOG_LEVEL) { + if (strlen(LOG_LEVEL) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, LOG_LEVEL, size); + } + break; + case SF_GLOBAL_LOG_PATH: + if (LOG_PATH) { + if (strlen(LOG_PATH) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, LOG_PATH, size); + } + break; default: break; } diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h new file mode 100644 index 0000000000..448ce1794b --- /dev/null +++ b/lib/client_config_parser.h @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef SNOWFLAKE_CLIENTCONFIG_H +#define SNOWFLAKE_CLIENTCONFIG_H + +#include "snowflake/client.h" +#include "cJSON.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct client_config + { + // The log level + char *logLevel; + + // The log path + char *logPath; + } client_config; + + /** + * Construct ClientConfig from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The ClientConfig object to be filled. + */ + void load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif //SNOWFLAKE_CLIENTCONFIG_H diff --git a/lib/logger.c b/lib/logger.c index 9d43929aaa..5bb5d6d72a 100644 --- a/lib/logger.c +++ b/lib/logger.c @@ -216,6 +216,10 @@ SF_LOG_LEVEL log_from_str_to_level(const char *level_in_str) { return SF_LOG_FATAL; } +const char* log_from_level_to_str(SF_LOG_LEVEL level) { + return level_names[level]; +} + void log_set_path(const char *path) { L.path = path; } diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 1e756b618d..78aae50fa8 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -4,7 +4,14 @@ #include "utils/test_setup.h" -#ifndef _WIN32 +#include +#include "memory.h" + +#ifdef _WIN32 +inline int access(const char *pathname, int mode) { + return _access(pathname, mode); +} +#else #include #endif @@ -24,6 +31,46 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } +/** + * Tests log settings from client config file + */ +void test_client_config_log(void **unused) { + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE *file; + file = fopen(configFilePath,"w"); + fprintf(file, clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + load_client_config(configFilePath, &clientConfig); + + // Set log name and level + char logname[] = "%s/dummy.log"; + size_t log_path_size = 1 + strlen(logname); + log_path_size += strlen(clientConfig.logPath); + char* LOG_PATH = (char*)SF_CALLOC(1, log_path_size); + sf_sprintf(LOG_PATH, log_path_size, logname, clientConfig.logPath); + log_set_level(log_from_str_to_level(clientConfig.logLevel)); + log_set_path(LOG_PATH); + + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, 0), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, 0), 0); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + #ifndef _WIN32 /** * Tests timing of log file creation @@ -140,6 +187,7 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), + cmocka_unit_test(test_client_config_log), #ifndef _WIN32 cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), From 836a82ab28d294c545a7c408afc936239282fa3f Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 13:08:40 -0700 Subject: [PATCH 02/74] Add missing function definition --- include/snowflake/logger.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/snowflake/logger.h b/include/snowflake/logger.h index b11ec1a14c..b860f8ecd8 100644 --- a/include/snowflake/logger.h +++ b/include/snowflake/logger.h @@ -94,6 +94,8 @@ void log_masked_va_list(FILE* fp, const char *fmt, va_list args); SF_LOG_LEVEL log_from_str_to_level(const char *level_in_str); +const char* log_from_level_to_str(SF_LOG_LEVEL level); + void log_set_path(const char* path); void log_close(); From c1878f332e09ef450fbff4c149d9434c64f4d6f3 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 15:02:14 -0700 Subject: [PATCH 03/74] Fix linux build errors --- cpp/lib/ClientConfigParser.cpp | 19 +++++++++++-------- include/snowflake/ClientConfigParser.hpp | 11 +++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 78a7964c0c..0462c451e5 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -16,6 +16,10 @@ #include #include +#ifndef _WIN32 +#include +#endif + using namespace Snowflake::Client; namespace @@ -115,9 +119,9 @@ void ClientConfigParser::loadClientConfig( } #else // 4. Try user home dir - if (std::string homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { - std::string homeDirFilePath = homeDir + SF_PATH_SEPARATOR + SF_CLIENT_CONFIG_FILE_NAME; + std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (boost::filesystem::exists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; @@ -163,7 +167,7 @@ void ClientConfigParser::parseConfigFile( "Could not open a file. The file may not exist: %s", in_filePath.c_str()); std::string errMsg = "Error finding client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); @@ -180,7 +184,7 @@ void ClientConfigParser::parseConfigFile( "parseConfigFile", "Error in parsing JSON: %s, err: %s", in_filePath.c_str(), error_ptr); std::string errMsg = "Error parsing client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } catch (std::exception& ex) @@ -219,7 +223,7 @@ void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) "Error due to other users having permission to modify the config file: %s", in_filePath.c_str()); std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } @@ -236,7 +240,7 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) "checkUnknownEntries", "Error in parsing JSON: %s, err: %s", in_jsonString.c_str(), error_ptr); std::string errMsg = "Error parsing json: " + in_jsonString; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } if (snowflake_cJSON_IsObject(jsonConfig)) @@ -269,6 +273,7 @@ std::string ClientConfigParser::getBinaryPath() HMODULE hm = NULL; wchar_t appName[256]; GetModuleFileNameW(hm, appName, 256); + binaryFullPath = appName; #else Dl_info info; int result = dladdr((void*)getBinaryPath, &info); @@ -277,8 +282,6 @@ std::string ClientConfigParser::getBinaryPath() binaryFullPath = info.dli_fname; } #endif - - binaryFullPath = appName; size_t pos = binaryFullPath.find_last_of(PATH_SEP); if (pos == std::wstring::npos) { diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 9aebfe1ce0..3b12d970b5 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -20,6 +20,17 @@ namespace Client std::string logPath = ""; }; + struct ClientConfigException : public std::exception + { + ClientConfigException(const std::string& message) : message_(message) {} + const char* what() const noexcept + { + return message_.c_str(); + } + + std::string message_; + }; + class ClientConfigParser { // Public ================================================================== From 21673d47f8c7ab6f6520bf5c33ab86b11b89b7db Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 15:58:02 -0700 Subject: [PATCH 04/74] Fix conversion issue on linux --- cpp/lib/ClientConfigParser.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 0462c451e5..5604838c0b 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -267,26 +267,27 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) //////////////////////////////////////////////////////////////////////////////// std::string ClientConfigParser::getBinaryPath() { - std::wstring binaryFullPath; - + std::string binaryFullPath; #if defined(WIN32) || defined(_WIN64) + std::wstring path; HMODULE hm = NULL; wchar_t appName[256]; GetModuleFileNameW(hm, appName, 256); - binaryFullPath = appName; + path = appName; + binaryFullPath = std::string(path.begin(), path.end()); #else Dl_info info; - int result = dladdr((void*)getBinaryPath, &info); + int result = dladdr((void*)load_client_config, &info); if (result) { - binaryFullPath = info.dli_fname; + binaryFullPath = std::string(info.dli_fname); } #endif size_t pos = binaryFullPath.find_last_of(PATH_SEP); - if (pos == std::wstring::npos) + if (pos == std::string::npos) { return ""; } - std::wstring binaryPath = binaryFullPath.substr(0, pos + 1); - return std::string(binaryPath.begin(), binaryPath.end()); + std::string binaryPath = binaryFullPath.substr(0, pos + 1); + return binaryPath; } From 806fb9c97284f9b6e36a7f19de59eef85b5726cb Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 16:54:33 -0700 Subject: [PATCH 05/74] Fix mac build error and fix logging test failure --- cpp/lib/ClientConfigParser.cpp | 4 ++-- tests/test_unit_logger.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 5604838c0b..270f989587 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -121,7 +121,7 @@ void ClientConfigParser::loadClientConfig( // 4. Try user home dir if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { - std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (boost::filesystem::exists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; @@ -258,7 +258,7 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) "sf", "ClientConfigParser", "checkUnknownEntries", - warnMsg); + warnMsg.c_str()); } } } diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 78aae50fa8..aa49a870ac 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -65,6 +65,7 @@ void test_client_config_log(void **unused) { // Warning log will trigger the log file creation log_warn("dummy warning log"); assert_int_equal(access(LOG_PATH, 0), 0); + log_close(); // Cleanup remove(configFilePath); @@ -94,6 +95,7 @@ void test_log_creation(void **unused) { // warning log will trigger the log file creation log_warn("dummy warning log"); assert_int_equal(access(logname, F_OK), 0); + log_close(); remove(logname); } From 4a5ad67d30f67b24f55b351469526f34460dafb4 Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 3 Oct 2024 10:05:05 -0700 Subject: [PATCH 06/74] Fix mac build error in test case --- tests/test_unit_logger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index aa49a870ac..a87183185f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -39,7 +39,7 @@ void test_client_config_log(void **unused) { char configFilePath[] = "sf_client_config.json"; FILE *file; file = fopen(configFilePath,"w"); - fprintf(file, clientConfigJSON); + fprintf(file, "%s", clientConfigJSON); fclose(file); // Parse client config for log details From 4e05bc716043976ee60de62bc46b558cb18caeb4 Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 3 Oct 2024 16:23:18 -0700 Subject: [PATCH 07/74] Fix setting of global log path --- lib/client.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/client.c b/lib/client.c index e6d2cd60d8..8810f26026 100644 --- a/lib/client.c +++ b/lib/client.c @@ -355,10 +355,12 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "%s/snowflake_%s.txt", sf_log_path, (char *) time_str); + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); } else { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "logs/snowflake_%s.txt", (char *) time_str); + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, "logs/"); } if (LOG_PATH != NULL) { // Set the log path only, the log file will be created when actual log output is needed. @@ -369,7 +371,6 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { goto cleanup; } - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); snowflake_global_set_attribute(SF_GLOBAL_LOG_LEVEL, log_from_level_to_str(sf_log_level)); ret = SF_BOOLEAN_TRUE; From a79370de10c6b949711ff9c5ecbc78e4c86d0ef5 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 4 Oct 2024 10:16:39 -0700 Subject: [PATCH 08/74] Use sf_memcpy instead --- cpp/lib/ClientConfigParser.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 270f989587..3979894030 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -41,12 +41,14 @@ void load_client_config( if (!clientConfig.logLevel.empty()) { out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); - strcpy(out_clientConfig->logLevel, clientConfig.logLevel.data()); + sf_memcpy(out_clientConfig->logLevel, sizeof(out_clientConfig->logLevel), + clientConfig.logLevel.data(), clientConfig.logLevel.size()); } if (!clientConfig.logPath.empty()) { out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); - strcpy(out_clientConfig->logPath, clientConfig.logPath.data()); + sf_memcpy(out_clientConfig->logPath, MAX_PATH, + clientConfig.logPath.data(), clientConfig.logPath.size()); } } From be865c158bb5d3c2ec5d9ce5483bc9f6ff9b7e6c Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 7 Oct 2024 11:38:34 -0700 Subject: [PATCH 09/74] Remove the use of boost algorithm string for compare --- cpp/lib/ClientConfigParser.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 3979894030..b3c5549d2d 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -14,7 +14,6 @@ #undef snprintf #include -#include #ifndef _WIN32 #include @@ -251,8 +250,15 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) snowflake_cJSON_ArrayForEach(entry, jsonConfig) { std::string key = entry->string; - boost::algorithm::to_lower(key); - if (KnownCommonEntries.find(key) == KnownCommonEntries.end()) + bool found = false; + for (std::string knownEntry : KnownCommonEntries) + { + if (sf_strncasecmp(key.c_str(), knownEntry.c_str(), knownEntry.length()) == 0) + { + found = true; + } + } + if (!found) { std::string warnMsg = "Unknown configuration entry: " + key + " with value:" + entry->valuestring; From f90848d0e66b3ae315d90b44913edbcd5cacf4b9 Mon Sep 17 00:00:00 2001 From: norrislee Date: Tue, 8 Oct 2024 16:40:44 -0700 Subject: [PATCH 10/74] Fix test issue --- tests/test_unit_logger.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index a87183185f..b6085ad58f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -7,11 +7,7 @@ #include #include "memory.h" -#ifdef _WIN32 -inline int access(const char *pathname, int mode) { - return _access(pathname, mode); -} -#else +#ifndef _WIN32 #include #endif @@ -31,6 +27,7 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } +#ifndef _WIN32 /** * Tests log settings from client config file */ @@ -60,11 +57,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -72,7 +69,6 @@ void test_client_config_log(void **unused) { remove(LOG_PATH); } -#ifndef _WIN32 /** * Tests timing of log file creation */ From 3823568379b54fe0e2949bf72427fbfd71efc568 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 09:34:17 -0700 Subject: [PATCH 11/74] Fix build issue with test --- tests/test_unit_logger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index b6085ad58f..b79f90b384 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -185,8 +185,8 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), - cmocka_unit_test(test_client_config_log), #ifndef _WIN32 + cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), #endif From a863b0597661130b9922fba41bcf56267cf72efb Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 16:56:41 -0700 Subject: [PATCH 12/74] Change ifstream to FILE due to 32bit debug read issue --- cpp/lib/ClientConfigParser.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index b3c5549d2d..d71104c642 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -157,11 +157,11 @@ void ClientConfigParser::parseConfigFile( const std::string& in_filePath, ClientConfig& out_clientConfig) { - std::ifstream configFile; cJSON* jsonConfig; + FILE* configFile; try { - configFile.open(in_filePath); + configFile = fopen(in_filePath.c_str(), "r"); if (!configFile) { CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", @@ -173,9 +173,15 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - std::stringstream configJSON; - configJSON << configFile.rdbuf(); - jsonConfig = snowflake_cJSON_Parse(configJSON.str().c_str()); + fseek(configFile, 0, SEEK_END); + long length = ftell(configFile); + fseek(configFile, 0, SEEK_SET); + char* buffer = (char*)malloc(length); + if (buffer) + { + fread(buffer, 1, length, configFile); + } + jsonConfig = snowflake_cJSON_Parse(buffer); const char* error_ptr = snowflake_cJSON_GetErrorPtr(); if (error_ptr) { @@ -190,7 +196,7 @@ void ClientConfigParser::parseConfigFile( } catch (std::exception& ex) { - configFile.close(); + fclose(configFile); throw; } @@ -206,7 +212,7 @@ void ClientConfigParser::parseConfigFile( { out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); } - configFile.close(); + fclose(configFile); } //////////////////////////////////////////////////////////////////////////////// From 994228b201708f758669c410d6ea3459f61ce565 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 18:09:09 -0700 Subject: [PATCH 13/74] Fix linux build issue --- cpp/lib/ClientConfigParser.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index d71104c642..2ccf1e170f 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -179,7 +179,15 @@ void ClientConfigParser::parseConfigFile( char* buffer = (char*)malloc(length); if (buffer) { - fread(buffer, 1, length, configFile); + size_t result = fread(buffer, 1, length, configFile); + if (result > 0) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "parseConfigFile", + "Error in reading file: %s", in_filePath.c_str()); + } } jsonConfig = snowflake_cJSON_Parse(buffer); const char* error_ptr = snowflake_cJSON_GetErrorPtr(); From 70bdc6004edc98d700afa17d9a4d177ffa14ab48 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 20:55:58 -0700 Subject: [PATCH 14/74] Fix test issues --- cpp/lib/ClientConfigParser.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 2ccf1e170f..3b3e15e140 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -90,7 +90,7 @@ void ClientConfigParser::loadClientConfig( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(binaryDirFilePath)) + if (checkFileExists(binaryDirFilePath)) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -112,7 +112,7 @@ void ClientConfigParser::loadClientConfig( } } std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -123,7 +123,7 @@ void ClientConfigParser::loadClientConfig( if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -161,7 +161,7 @@ void ClientConfigParser::parseConfigFile( FILE* configFile; try { - configFile = fopen(in_filePath.c_str(), "r"); + configFile = sf_fopen(&configFile, in_filePath.c_str(), "r"); if (!configFile) { CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", @@ -175,12 +175,12 @@ void ClientConfigParser::parseConfigFile( #endif fseek(configFile, 0, SEEK_END); long length = ftell(configFile); - fseek(configFile, 0, SEEK_SET); + rewind(configFile); char* buffer = (char*)malloc(length); if (buffer) { size_t result = fread(buffer, 1, length, configFile); - if (result > 0) + if (result != length) { CXX_LOG_ERROR( "sf", From 309d118ad64d0101bbe4e761fde4741b3bb722ff Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 11 Oct 2024 09:29:29 -0700 Subject: [PATCH 15/74] Rename ifndef for headers to be more consistent --- include/snowflake/ClientConfigParser.hpp | 6 +++--- lib/client_config_parser.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 3b12d970b5..f61ccaa8d9 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -2,8 +2,8 @@ * Copyright (c) 2024 Snowflake Computing */ -#ifndef PROJECT_CLIENTCONFIG_HPP -#define PROJECT_CLIENTCONFIG_HPP +#ifndef SNOWFLAKE_CONFIGPARSER_HPP +#define SNOWFLAKE_CONFIGPARSER_HPP #include @@ -101,4 +101,4 @@ namespace Client } // namespace Client } // namespace Snowflake -#endif //PROJECT_CLIENTCONFIG_HPP +#endif //SNOWFLAKE_CONFIGPARSER_HPP diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h index 448ce1794b..2d5b584802 100644 --- a/lib/client_config_parser.h +++ b/lib/client_config_parser.h @@ -2,8 +2,8 @@ * Copyright (c) 2024 Snowflake Computing */ -#ifndef SNOWFLAKE_CLIENTCONFIG_H -#define SNOWFLAKE_CLIENTCONFIG_H +#ifndef SNOWFLAKE_CONFIGPARSER_H +#define SNOWFLAKE_CONFIGPARSER_H #include "snowflake/client.h" #include "cJSON.h" @@ -41,4 +41,4 @@ extern "C" { } // extern "C" #endif -#endif //SNOWFLAKE_CLIENTCONFIG_H +#endif //SNOWFLAKE_CONFIGPARSER_H From 21a3b114f8d9e56c10a90413c581d5b71b3758d0 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 15:57:45 -0700 Subject: [PATCH 16/74] Replace boost with std::filesystem, fix possible NULL issues, add tests --- CMakeLists.txt | 4 +- cpp/lib/ClientConfigParser.cpp | 116 +++++++++++------------ include/snowflake/ClientConfigParser.hpp | 22 ++--- lib/client.c | 19 +--- lib/client_config_parser.h | 44 --------- tests/test_unit_logger.c | 93 ++++++++++++++++-- 6 files changed, 155 insertions(+), 143 deletions(-) delete mode 100644 lib/client_config_parser.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ec366b43f..b2fba3eed5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,7 @@ set(SOURCE_FILES include/snowflake/logger.h include/snowflake/version.h include/snowflake/platform.h + include/snowflake/client_config_parser.h lib/client.c lib/constants.h lib/cJSON.h @@ -247,8 +248,7 @@ set(SOURCE_FILES_CPP_WRAPPER lib/result_set_json.h lib/query_context_cache.h lib/curl_desc_pool.h - lib/authenticator.h - lib/client_config_parser.h) + lib/authenticator.h) if (UNIX) if (LINUX) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 3b3e15e140..a999bf3c1f 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -4,22 +4,21 @@ #include "snowflake/ClientConfigParser.hpp" #include "snowflake/Exceptions.hpp" +#include "snowflake/client_config_parser.h" #include "../logger/SFLogger.hpp" -#include "client_config_parser.h" #include "memory.h" +#include #include #include #include -#undef snprintf -#include - #ifndef _WIN32 #include #endif using namespace Snowflake::Client; +using namespace std::filesystem; namespace { @@ -30,24 +29,16 @@ namespace } //////////////////////////////////////////////////////////////////////////////// -void load_client_config( +sf_bool load_client_config( const char* in_configFilePath, client_config* out_clientConfig) { - ClientConfigParser configParser; - ClientConfig clientConfig; - configParser.loadClientConfig(in_configFilePath, clientConfig); - if (!clientConfig.logLevel.empty()) - { - out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); - sf_memcpy(out_clientConfig->logLevel, sizeof(out_clientConfig->logLevel), - clientConfig.logLevel.data(), clientConfig.logLevel.size()); - } - if (!clientConfig.logPath.empty()) - { - out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); - sf_memcpy(out_clientConfig->logPath, MAX_PATH, - clientConfig.logPath.data(), clientConfig.logPath.size()); + try { + ClientConfigParser configParser; + configParser.loadClientConfig(in_configFilePath, *out_clientConfig); + } catch (std::exception e) { + sf_fprintf(stderr, e.what()); + return false; } } @@ -67,7 +58,19 @@ ClientConfigParser::~ClientConfigParser() //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::loadClientConfig( const std::string& in_configFilePath, - ClientConfig& out_clientConfig) + client_config& out_clientConfig) +{ + std::string derivedConfigPath = resolveClientConfigPath(in_configFilePath); + + if (!derivedConfigPath.empty()) + { + parseConfigFile(derivedConfigPath, out_clientConfig); + } +} + +//////////////////////////////////////////////////////////////////////////////// +std::string ClientConfigParser::resolveClientConfigPath( + const std::string& in_configFilePath) { char envbuf[MAX_PATH + 1]; std::string derivedConfigPath = ""; @@ -90,7 +93,7 @@ void ClientConfigParser::loadClientConfig( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(binaryDirFilePath)) + if (is_regular_file(binaryDirFilePath)) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -100,19 +103,22 @@ void ClientConfigParser::loadClientConfig( { #if defined(WIN32) || defined(_WIN64) // 4. Try user home dir - std::string homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); - if (homeDir.empty()) + std::string homeDirFilePath; + char* homeDir; + if ((homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf))) && strlen(homeDir) != 0) { + homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + } else { // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH - std::string homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); - std::string homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); - if (!homeDriveEnv.empty() && !homePathEnv.empty()) + char* homeDriveEnv; + char* homePathEnv; + if ((homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf))) && (strlen(homeDriveEnv) != 0) && + (homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf))) && (strlen(homePathEnv) != 0)) { - homeDir = homeDriveEnv + homePathEnv; + homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; } } - std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(homeDirFilePath)) + if (is_regular_file(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -120,10 +126,11 @@ void ClientConfigParser::loadClientConfig( } #else // 4. Try user home dir - if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + char* homeDir; + if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(homeDirFilePath)) + if (is_regular_file(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -133,29 +140,14 @@ void ClientConfigParser::loadClientConfig( #endif } } - if (!derivedConfigPath.empty()) - { - parseConfigFile(derivedConfigPath, out_clientConfig); - } + return derivedConfigPath; } // Private ===================================================================== -//////////////////////////////////////////////////////////////////////////////// -bool ClientConfigParser::checkFileExists(const std::string& in_filePath) -{ - FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); - if (file != nullptr) - { - fclose(file); - return true; - } - return false; -} - //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::parseConfigFile( const std::string& in_filePath, - ClientConfig& out_clientConfig) + client_config& out_clientConfig) { cJSON* jsonConfig; FILE* configFile; @@ -173,14 +165,12 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - fseek(configFile, 0, SEEK_END); - long length = ftell(configFile); - rewind(configFile); - char* buffer = (char*)malloc(length); + const int fileSize = file_size(in_filePath); + char* buffer = (char*)malloc(fileSize); if (buffer) { - size_t result = fread(buffer, 1, length, configFile); - if (result != length) + size_t result = fread(buffer, 1, fileSize, configFile); + if (result != fileSize) { CXX_LOG_ERROR( "sf", @@ -204,7 +194,10 @@ void ClientConfigParser::parseConfigFile( } catch (std::exception& ex) { - fclose(configFile); + if (configFile) + { + fclose(configFile); + } throw; } @@ -220,16 +213,19 @@ void ClientConfigParser::parseConfigFile( { out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); } - fclose(configFile); + if (configFile) + { + fclose(configFile); + } } //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) { - boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); - boost::filesystem::perms permissions = fileStatus.permissions(); - if (permissions & boost::filesystem::group_write || - permissions & boost::filesystem::others_write) + file_status fileStatus = status(in_filePath); + perms permissions = fileStatus.permissions(); + if ((perms::group_write == (permissions & perms::group_write)) || + (perms::others_write == (permissions & perms::others_write))) { CXX_LOG_ERROR( "sf", diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index f61ccaa8d9..263ab0d45c 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -6,20 +6,12 @@ #define SNOWFLAKE_CONFIGPARSER_HPP #include +#include "client_config_parser.h" namespace Snowflake { namespace Client { - struct ClientConfig - { - // The log level - std::string logLevel = ""; - - // The log path - std::string logPath = ""; - }; - struct ClientConfigException : public std::exception { ClientConfigException(const std::string& message) : message_(message) {} @@ -57,16 +49,18 @@ namespace Client */ void loadClientConfig( const std::string& in_configFilePath, - ClientConfig& out_clientConfig); + client_config& out_clientConfig); // Private ================================================================= private: /** - * @brief Check if the file exists. + * @brief Resolve the client config path. * - * @param in_filePath The file path to check. + * @param in_configFilePath The config file path passed in by the user. + * + * @param The client config path */ - bool checkFileExists(const std::string& in_filePath); + std::string resolveClientConfigPath(const std::string& in_configFilePath); /** * @brief Parse JSON string. @@ -76,7 +70,7 @@ namespace Client */ void parseConfigFile( const std::string& in_filePath, - ClientConfig& out_clientConfig); + client_config& out_clientConfig); /** * @ brief Check if other have permission to modify file diff --git a/lib/client.c b/lib/client.c index 8810f26026..5c7cc0fc60 100644 --- a/lib/client.c +++ b/lib/client.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "constants.h" #include "client_int.h" #include "connection.h" @@ -19,7 +20,6 @@ #include "chunk_downloader.h" #include "authenticator.h" #include "query_context_cache.h" -#include "client_config_parser.h" #ifdef _WIN32 #include @@ -38,7 +38,6 @@ sf_bool SF_OCSP_CHECK; char *SF_HEADER_USER_AGENT = NULL; static char *CLIENT_CONFIG_FILE = NULL; -static char* LOG_LEVEL; static char *LOG_PATH = NULL; static FILE *LOG_FP = NULL; @@ -355,12 +354,10 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "%s/snowflake_%s.txt", sf_log_path, (char *) time_str); - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); } else { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "logs/snowflake_%s.txt", (char *) time_str); - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, "logs/"); } if (LOG_PATH != NULL) { // Set the log path only, the log file will be created when actual log output is needed. @@ -647,12 +644,6 @@ snowflake_global_set_attribute(SF_GLOBAL_ATTRIBUTE type, const void *value) { case SF_GLOBAL_CLIENT_CONFIG_FILE: alloc_buffer_and_copy(&CLIENT_CONFIG_FILE, value); break; - case SF_GLOBAL_LOG_LEVEL: - alloc_buffer_and_copy(&LOG_LEVEL, value); - break; - case SF_GLOBAL_LOG_PATH: - alloc_buffer_and_copy(&LOG_PATH, value); - break; default: break; } @@ -691,13 +682,13 @@ snowflake_global_get_attribute(SF_GLOBAL_ATTRIBUTE type, void *value, size_t siz } break; case SF_GLOBAL_LOG_LEVEL: - if (LOG_LEVEL) { - if (strlen(LOG_LEVEL) > size - 1) { + { + if (strlen(log_from_level_to_str(log_get_level())) > size - 1) { return SF_STATUS_ERROR_BUFFER_TOO_SMALL; } - sf_strncpy(value, size, LOG_LEVEL, size); + sf_strncpy(value, size, log_from_level_to_str(log_get_level()), size); + break; } - break; case SF_GLOBAL_LOG_PATH: if (LOG_PATH) { if (strlen(LOG_PATH) > size - 1) { diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h deleted file mode 100644 index 2d5b584802..0000000000 --- a/lib/client_config_parser.h +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) 2024 Snowflake Computing - */ - -#ifndef SNOWFLAKE_CONFIGPARSER_H -#define SNOWFLAKE_CONFIGPARSER_H - -#include "snowflake/client.h" -#include "cJSON.h" - -#ifdef __cplusplus -extern "C" { -#endif - - typedef struct client_config - { - // The log level - char *logLevel; - - // The log path - char *logPath; - } client_config; - - /** - * Construct ClientConfig from client config file passed by user. This method searches the - * config file in following order: 1. configFilePath param which is read from connection URL or - * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to - * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the - * driver directory from where the driver gets loaded. 4. Searches for default config file - * name(sf_client_config.json) under user home directory 5. Searches for default config file - * name(sf_client_config.json) under tmp directory - * - * @param in_configFilePath The config file path passed in by the user. - * @param out_clientConfig The ClientConfig object to be filled. - */ - void load_client_config( - const char* in_configFilePath, - client_config* out_clientConfig); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif //SNOWFLAKE_CONFIGPARSER_H diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index b79f90b384..a4fe7823d2 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -4,13 +4,18 @@ #include "utils/test_setup.h" -#include +#include #include "memory.h" -#ifndef _WIN32 +#ifdef _WIN32 +inline int access(const char* pathname, int mode) { + return _access(pathname, mode); +} +#else #include #endif + /** * Tests converting a string representation of log level to the log level enum */ @@ -27,7 +32,38 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } -#ifndef _WIN32 +/** + * Tests log settings with invalid client config filepath + */ +void test_invalid_client_config_path(void** unused) { + char configFilePath[] = "fakePath.json"; + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); +} + +/** + * Tests log settings from client config file with invalid json + */ +void test_client_config_log_invalid_json(void** unused) { + char clientConfigJSON[] = "{{{\"invalid json\"}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); + + // Cleanup + remove(configFilePath); +} + /** * Tests log settings from client config file */ @@ -57,11 +93,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, F_OK), 0); + assert_int_not_equal(access(LOG_PATH, 0), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, F_OK), 0); + assert_int_equal(access(LOG_PATH, 0), 0); log_close(); // Cleanup @@ -69,6 +105,41 @@ void test_client_config_log(void **unused) { remove(LOG_PATH); } +/** + * Tests log settings from client config file via global init + */ +void test_client_config_log_init(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); + snowflake_global_init("./logs", SF_LOG_TRACE, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, 0), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, 0), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + + /** * Tests timing of log file creation */ @@ -77,7 +148,7 @@ void test_log_creation(void **unused) { // ensure the log file doesn't exist at the beginning remove(logname); - assert_int_not_equal(access(logname, F_OK), 0); + assert_int_not_equal(access(logname, 0), 0); log_set_lock(NULL); log_set_level(SF_LOG_WARN); @@ -86,16 +157,17 @@ void test_log_creation(void **unused) { // info log won't trigger the log file creation since log level is set to warning log_info("dummy info log"); - assert_int_not_equal(access(logname, F_OK), 0); + assert_int_not_equal(access(logname, 0), 0); // warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(logname, F_OK), 0); + assert_int_equal(access(logname, 0), 0); log_close(); remove(logname); } +#ifndef _WIN32 /** * Tests masking secret information in log */ @@ -185,8 +257,11 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), -#ifndef _WIN32 + cmocka_unit_test(test_invalid_client_config_path), + cmocka_unit_test(test_client_config_log_invalid_json), cmocka_unit_test(test_client_config_log), + cmocka_unit_test(test_client_config_log_init), +#ifndef _WIN32 cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), #endif From 8d97f3a3deed0f363189025a1991e7c575643ebe Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 15:58:23 -0700 Subject: [PATCH 17/74] Add client_config_parser to include dir --- include/snowflake/client_config_parser.h | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 include/snowflake/client_config_parser.h diff --git a/include/snowflake/client_config_parser.h b/include/snowflake/client_config_parser.h new file mode 100644 index 0000000000..86ae0683b6 --- /dev/null +++ b/include/snowflake/client_config_parser.h @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef SNOWFLAKE_CONFIGPARSER_H +#define SNOWFLAKE_CONFIGPARSER_H + +#include "snowflake/client.h" +#include "cJSON.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct client_config + { + // The log level + char *logLevel; + + // The log path + char *logPath; + } client_config; + + /** + * Construct client_config from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The client_config object to be filled. + * + * @return true if successful + */ + sf_bool load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif //SNOWFLAKE_CONFIGPARSER_H From 577925a8b399db8ea6744728c764d560e2ceb1f3 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 16:55:25 -0700 Subject: [PATCH 18/74] Fix compilation error --- cpp/lib/ClientConfigParser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index a999bf3c1f..d41b450a8a 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -127,7 +127,7 @@ std::string ClientConfigParser::resolveClientConfigPath( #else // 4. Try user home dir char* homeDir; - if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0) + if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (is_regular_file(homeDirFilePath)) From 6a27fe5d0545d39b7c3cfc7d47d40a05966de006 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 17:24:31 -0700 Subject: [PATCH 19/74] Fix return value error --- cpp/lib/ClientConfigParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index d41b450a8a..13c50a7c66 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -40,6 +40,7 @@ sf_bool load_client_config( sf_fprintf(stderr, e.what()); return false; } + return true; } // Public ====================================================================== From fdb5187e0468853a2ce1576b82f9949780baa405 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 18:14:34 -0700 Subject: [PATCH 20/74] Use experimental filesystem for non-windows --- cpp/lib/ClientConfigParser.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 13c50a7c66..7a1e5083b0 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -8,17 +8,21 @@ #include "../logger/SFLogger.hpp" #include "memory.h" -#include #include #include #include #ifndef _WIN32 #include +#include +using namespace std::experimental::filesystem; +#else +#include +using namespace std::filesystem; #endif using namespace Snowflake::Client; -using namespace std::filesystem; + namespace { From 2d66b2068a93f3112e57cae3f36da902bb7c4e13 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 18:49:51 -0700 Subject: [PATCH 21/74] Add stdc++fs to link libraries for unix builds --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b2fba3eed5..43424000b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -517,11 +517,11 @@ endif() if (LINUX) # Linux - target_link_libraries(snowflakeclient rt dl z) + target_link_libraries(snowflakeclient rt dl z stdc++fs) endif () if (APPLE) # OSX. no librt is required. - target_link_libraries(snowflakeclient dl z) + target_link_libraries(snowflakeclient dl z stdc++fs) endif () if (WIN32) # Windows From e48007493279c3649fcf41f4420e8f71b247ae4e Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 19:41:56 -0700 Subject: [PATCH 22/74] Fix macos filesystem error --- cpp/lib/ClientConfigParser.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 7a1e5083b0..882d8f0580 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -14,11 +14,14 @@ #ifndef _WIN32 #include -#include -using namespace std::experimental::filesystem; -#else +#endif + +#ifdef _WIN32 #include using namespace std::filesystem; +#elif defined(__linux__) +#include +using namespace std::experimental::filesystem; #endif using namespace Snowflake::Client; From 39d1036e5b6aa6d65efec08ef18b05ef9f1ce31a Mon Sep 17 00:00:00 2001 From: SimbaGithub <48035983+SimbaGithub@users.noreply.github.com> Date: Thu, 17 Oct 2024 02:26:08 -0700 Subject: [PATCH 23/74] SNOW-692945: multiple statements support (#727) Co-authored-by: sfc-gh-ext-simba-hx --- CMakeLists.txt | 4 - cpp/lib/ResultSet.cpp | 9 +- cpp/lib/ResultSet.hpp | 26 +- cpp/lib/ResultSetArrow.cpp | 37 ++- cpp/lib/ResultSetArrow.hpp | 14 + cpp/lib/ResultSetJson.cpp | 7 +- cpp/lib/result_set.cpp | 353 +++++++------------- cpp/lib/result_set_arrow.cpp | 480 --------------------------- cpp/lib/result_set_json.cpp | 260 --------------- include/snowflake/client.h | 33 +- include/snowflake/version.h | 3 +- lib/client.c | 547 +++++++++++++++++++++---------- lib/client_int.h | 1 + lib/connection.c | 27 +- lib/connection.h | 7 +- lib/result_set.h | 103 ++---- lib/result_set_arrow.h | 268 --------------- lib/result_set_json.h | 259 --------------- tests/CMakeLists.txt | 1 + tests/test_multiple_statements.c | 307 +++++++++++++++++ 20 files changed, 960 insertions(+), 1786 deletions(-) delete mode 100644 cpp/lib/result_set_arrow.cpp delete mode 100644 cpp/lib/result_set_json.cpp delete mode 100644 lib/result_set_arrow.h delete mode 100644 lib/result_set_json.h create mode 100644 tests/test_multiple_statements.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 529f322af2..b71591988f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,8 +225,6 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ClientQueryContextCache.cpp cpp/lib/ClientQueryContextCache.hpp cpp/lib/result_set.cpp - cpp/lib/result_set_arrow.cpp - cpp/lib/result_set_json.cpp cpp/lib/ResultSet.cpp cpp/lib/ResultSet.hpp cpp/lib/ResultSetArrow.cpp @@ -241,8 +239,6 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/util/CurlDesc.cpp cpp/util/CurlDescPool.cpp lib/result_set.h - lib/result_set_arrow.h - lib/result_set_json.h lib/query_context_cache.h lib/curl_desc_pool.h lib/authenticator.h) diff --git a/cpp/lib/ResultSet.cpp b/cpp/lib/ResultSet.cpp index f304164c3d..bd8484a20f 100644 --- a/cpp/lib/ResultSet.cpp +++ b/cpp/lib/ResultSet.cpp @@ -17,7 +17,7 @@ namespace Snowflake namespace Client { -ResultSet::ResultSet() : +ResultSet::ResultSet(QueryResultFormat format) : m_binaryOutputFormat("HEX"), m_dateOutputFormat("YYYY-MM-DD"), m_timeOutputFormat("HH24:MI:SS"), @@ -28,14 +28,16 @@ ResultSet::ResultSet() : m_currChunkIdx(0), m_currChunkRowIdx(0), m_currColumnIdx(0), - m_currRowIdx(0) + m_currRowIdx(0), + m_queryResultFormat(format) { ; } ResultSet::ResultSet( SF_COLUMN_DESC * metadata, - std::string tzString + std::string tzString, + QueryResultFormat format ) : m_currChunkIdx(0), m_currChunkRowIdx(0), @@ -44,6 +46,7 @@ ResultSet::ResultSet( m_totalChunkCount(0), m_totalColumnCount(0), m_metadata(metadata), + m_queryResultFormat(format), m_isFirstChunk(true), m_tzString(tzString), m_error(SF_STATUS_SUCCESS) diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index 8f3a46f059..157bd9cc99 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -12,21 +12,13 @@ #include "cJSON.h" #include "snowflake/basic_types.h" #include "snowflake/client.h" +#include "result_set.h" namespace Snowflake { namespace Client { -/** - * Enumeration over valid query result formats. - */ -enum QueryResultFormat -{ - ARROW, - JSON -}; - /** * The implementation of a base result set. * @@ -40,7 +32,7 @@ class ResultSet /** * Default constructor. */ - ResultSet(); + ResultSet(QueryResultFormat format); /** * Parameterized constructor. @@ -48,7 +40,7 @@ class ResultSet * @param metadata The metadata of the result set. * @param tzString The time zone. */ - ResultSet(SF_COLUMN_DESC * metadata, std::string tzString); + ResultSet(SF_COLUMN_DESC * metadata, std::string tzString, QueryResultFormat format); /** * Destructor. @@ -194,6 +186,13 @@ class ResultSet */ virtual SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data) = 0; + /** + * Gets the total number of rows in the current chunk being processed. + * + * @return the number of rows in the current chunk. + */ + virtual size_t getRowCountInChunk() = 0; + // Other member getters ======================================================================== SF_STATUS getError() @@ -215,6 +214,11 @@ class ResultSet } } + QueryResultFormat getResultFormat() + { + return m_queryResultFormat; + } + protected: // Protected members =========================================================================== diff --git a/cpp/lib/ResultSetArrow.cpp b/cpp/lib/ResultSetArrow.cpp index 78779bc80a..ef20ded1fe 100644 --- a/cpp/lib/ResultSetArrow.cpp +++ b/cpp/lib/ResultSetArrow.cpp @@ -10,7 +10,6 @@ #include "../logger/SFLogger.hpp" #include "ResultSetArrow.hpp" -#include "result_set_arrow.h" #include "DataConversion.hpp" #include "results.h" @@ -22,9 +21,9 @@ namespace Client ResultSetArrow::ResultSetArrow() : - Snowflake::Client::ResultSet() + Snowflake::Client::ResultSet(SF_ARROW_FORMAT) { - m_queryResultFormat = QueryResultFormat::ARROW; + ; // Do nothing } ResultSetArrow::ResultSetArrow( @@ -32,10 +31,8 @@ ResultSetArrow::ResultSetArrow( SF_COLUMN_DESC * metadata, std::string tzString ) : - ResultSet(metadata, tzString) + ResultSet(metadata, tzString, SF_ARROW_FORMAT) { - m_queryResultFormat = QueryResultFormat::ARROW; - this->appendChunk(initialChunk); // Reset row indices so that they can be re-used by public API. @@ -44,6 +41,34 @@ ResultSetArrow::ResultSetArrow( m_currRowIdx = 0; } +ResultSetArrow::ResultSetArrow( + cJSON * jsonRowset64, + SF_COLUMN_DESC * metadata, + std::string tzString +) : + ResultSet(metadata, tzString, SF_ARROW_FORMAT) +{ + arrow::BufferBuilder* bufferBuilder = NULL; + if (jsonRowset64) + { + const char* base64RowsetStr = snowflake_cJSON_GetStringValue(jsonRowset64); + if (base64RowsetStr && strlen(base64RowsetStr) > 0) + { + // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. + std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); + bufferBuilder = new arrow::BufferBuilder(); + (void)bufferBuilder->Append((void*)decodedRowsetStr.c_str(), decodedRowsetStr.length()); + } + } + + this->appendChunk(bufferBuilder); + + // Reset row indices so that they can be re-used by public API. + m_currChunkIdx = 0; + m_currChunkRowIdx = 0; + m_currRowIdx = 0; +} + ResultSetArrow::~ResultSetArrow() { ; // Do nothing. diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 4c85c27501..1805094b12 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -64,6 +64,20 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, std::string tzString); + + /** + * Parameterized constructor. + * + * This constructor will initialize m_records with the (partial) results + * contained in the initial chunk. It will also initialize m_metadata with + * the metadata in "metadata". + * + * @param jsonRowset64 A pointer to the rowset64 data in json result set. + * @param metadata An array of metadata objects for each column. + * @param tzString The time zone. + */ + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string tzString); + /** * Destructor. */ diff --git a/cpp/lib/ResultSetJson.cpp b/cpp/lib/ResultSetJson.cpp index 9b27947495..a9bfc42490 100644 --- a/cpp/lib/ResultSetJson.cpp +++ b/cpp/lib/ResultSetJson.cpp @@ -18,9 +18,9 @@ namespace Client ResultSetJson::ResultSetJson() : - ResultSet() + ResultSet(SF_JSON_FORMAT) { - m_queryResultFormat = QueryResultFormat::JSON; + ; // Do nothing } ResultSetJson::ResultSetJson( @@ -28,9 +28,8 @@ ResultSetJson::ResultSetJson( SF_COLUMN_DESC * metadata, std::string tzString ) : - ResultSet(metadata, tzString) + ResultSet(metadata, tzString, SF_JSON_FORMAT) { - m_queryResultFormat = QueryResultFormat::JSON; m_chunk = nullptr; appendChunk(rowset); } diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 5cfcb1238a..8923fb77aa 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -4,382 +4,283 @@ #include "result_set.h" #include "ResultSet.hpp" +#include "ResultSetArrow.hpp" +#include "ResultSetJson.hpp" #ifdef __cplusplus extern "C" { #endif - void * rs_create_with_json_result( + result_set_ptr rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string ) { - switch (*query_result_format) + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_create_with_json_result(json_rowset, metadata, tz_string); - case JSON_FORMAT: - return rs_json_create(json_rowset, metadata, tz_string); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, std::string(tz_string)); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson(json_rowset, metadata, std::string(tz_string)); default: return nullptr; } } - void * rs_create_with_chunk( + result_set_ptr rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string ) { - switch (*query_result_format) + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_create_with_chunk((NON_JSON_RESP*)initial_chunk, metadata, tz_string); - case JSON_FORMAT: - return rs_json_create((cJSON*)initial_chunk, metadata, tz_string); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, std::string(tz_string)); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, std::string(tz_string)); default: return nullptr; } } - void rs_destroy(void * rs, QueryResultFormat_t * query_result_format) + void rs_destroy(result_set_ptr rs) { - switch (*query_result_format){ - case ARROW_FORMAT: - rs_arrow_destroy((rs_arrow_t *) rs); + if (!rs) + { + return; + } + QueryResultFormat query_result_format = + static_cast(rs)->getResultFormat(); + switch (query_result_format){ +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + delete static_cast(rs); break; - case JSON_FORMAT: - rs_json_destroy((rs_json_t *) rs); +#endif + case SF_JSON_FORMAT: + delete static_cast(rs); break; default: break; } } +#define ERROR_IF_NULL(ptr) \ +{ \ + if (ptr == NULL) { return SF_STATUS_ERROR_NULL_POINTER; } \ +} + SF_STATUS STDCALL - rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk) + rs_append_chunk(result_set_ptr rs, void * chunk) { - switch (*query_result_format) + ERROR_IF_NULL(rs); + QueryResultFormat query_result_format = + static_cast(rs)->getResultFormat(); + switch (query_result_format) { - case ARROW_FORMAT: - return rs_arrow_append_chunk((rs_arrow_t *) rs, (NON_JSON_RESP*)chunk); - case JSON_FORMAT: - return rs_json_append_chunk((rs_json_t *) rs, (cJSON*)chunk); +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return static_cast(rs)->appendChunk( + (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunk)->buffer)); +#endif + case SF_JSON_FORMAT: + return static_cast(rs)->appendChunk( + (cJSON*)chunk); default: return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; } } - SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format) + SF_STATUS STDCALL rs_next(result_set_ptr rs) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_next((rs_arrow_t *) rs); - case JSON_FORMAT: - return rs_json_next((rs_json_t *) rs); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->next(); } SF_STATUS STDCALL rs_get_cell_as_bool( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_bool((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_bool((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsBool(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int8 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int8((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int8((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt8(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_int64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_int64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_int64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsInt64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint8 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint8((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint8((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint8(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_uint64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_uint64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_uint64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsUint64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_float32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float32 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_float32((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_float32((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsFloat32(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_float64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float64 * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_float64((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_float64((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsFloat64(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_const_string( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, const char ** out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_const_string((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_const_string((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsConstString(idx, out_data); } SF_STATUS STDCALL rs_get_cell_as_timestamp( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, SF_TIMESTAMP * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_as_timestamp((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_as_timestamp((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellAsTimestamp(idx, out_data); } SF_STATUS STDCALL rs_get_cell_strlen( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, size_t * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_cell_strlen((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_get_cell_strlen((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getCellStrlen(idx, out_data); } - size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format) + size_t rs_get_row_count_in_chunk(result_set_ptr rs) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_get_row_count_in_chunk((rs_arrow_t *) rs); - case JSON_FORMAT: - return rs_json_get_row_count_in_chunk((rs_json_t *) rs); - default: - return 0; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->getRowCountInChunk(); } SF_STATUS STDCALL rs_is_cell_null( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data ) { - switch (*query_result_format) - { - case ARROW_FORMAT: - return rs_arrow_is_cell_null((rs_arrow_t *) rs, idx, out_data); - case JSON_FORMAT: - return rs_json_is_cell_null((rs_json_t *) rs, idx, out_data); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + ERROR_IF_NULL(rs); + return static_cast(rs)->isCellNull(idx, out_data); } - SF_STATUS STDCALL rs_get_error( - void * rs, - QueryResultFormat_t * query_result_format - ) + SF_STATUS STDCALL rs_get_error(result_set_ptr rs) { - if (!rs || !query_result_format) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - Snowflake::Client::ResultSet * rs_obj = NULL; - switch (*query_result_format) - { - case ARROW_FORMAT: - rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); - case JSON_FORMAT: - rs_obj = static_cast(((rs_json_t *)rs)->rs_object); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } - - return rs_obj->getError(); + ERROR_IF_NULL(rs); + return static_cast(rs)->getError(); } - const char* rs_get_error_message( - void * rs, - QueryResultFormat_t * query_result_format - ) + const char* rs_get_error_message(result_set_ptr rs) { - if (!rs || !query_result_format) + if (!rs) { return ""; } + return static_cast(rs)->getErrorMessage(); + } - Snowflake::Client::ResultSet * rs_obj = NULL; - switch (*query_result_format) - { - case ARROW_FORMAT: - rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); - case JSON_FORMAT: - rs_obj = static_cast(((rs_json_t *)rs)->rs_object); - default: - return ""; - } +#ifndef SF_WIN32 + size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) + { + size_t data_size = size * nmemb; + arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); + + log_debug("Curl response for arrow chunk size: %zu", data_size); + (void) arrowBufBuilder->Append(ptr, data_size); + return data_size; + } - return rs_obj->getErrorMessage(); + NON_JSON_RESP* callback_create_arrow_resp(void) + { + NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; + arrow_resp->buffer = new arrow::BufferBuilder(); + arrow_resp->write_callback = arrow_write_callback; + return arrow_resp; } +#else + NON_JSON_RESP* callback_create_arrow_resp(void) + { + log_error("Query results were fetched using Arrow"); + return NULL; + } +#endif #ifdef __cplusplus } // extern "C" diff --git a/cpp/lib/result_set_arrow.cpp b/cpp/lib/result_set_arrow.cpp deleted file mode 100644 index 090cbe1788..0000000000 --- a/cpp/lib/result_set_arrow.cpp +++ /dev/null @@ -1,480 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#include "arrowheaders.hpp" - -#include - -#include "memory.h" -#include "result_set_arrow.h" -#include "ResultSetArrow.hpp" - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef SF_WIN32 - rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - arrow::BufferBuilder* bufferBuilder = NULL; - if (json_rowset64) - { - const char * base64RowsetStr = snowflake_cJSON_GetStringValue(json_rowset64); - if (base64RowsetStr && strlen(base64RowsetStr) > 0) - { - // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. - std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); - bufferBuilder = new arrow::BufferBuilder(); - (void) bufferBuilder->Append((void *)decodedRowsetStr.c_str(), decodedRowsetStr.length()); - } - } - - rs_arrow_t * rs_struct = (rs_arrow_t *) SF_MALLOC(sizeof(rs_arrow_t)); - Snowflake::Client::ResultSetArrow * rs_obj = - new Snowflake::Client::ResultSetArrow(bufferBuilder, metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - return rs_struct; - } - - rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - rs_arrow_t * rs_struct = (rs_arrow_t *)SF_MALLOC(sizeof(rs_arrow_t)); - Snowflake::Client::ResultSetArrow * rs_obj = - new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(initial_chunk->buffer), metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - // the buffer is passed in to result set so the response is no longer needed - delete initial_chunk; - - return rs_struct; - } - - void rs_arrow_destroy(rs_arrow_t * rs) - { - if (rs == NULL) - { - return; - } - - delete static_cast(rs->rs_object); - SF_FREE(rs); - } - - SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - - arrow::BufferBuilder * buffer = (arrow::BufferBuilder*)(chunk->buffer); - delete chunk; - return rs_obj->appendChunk(buffer); - } - - SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->next(); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsBool(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt8(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint8(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat32(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat64(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data - ) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsConstString(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data - ) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsTimestamp(idx, out_data); - } - - SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellStrlen(idx, out_data); - } - - size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return 0; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getRowCountInChunk(); - } - - SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetArrow * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->isCellNull(idx, out_data); - } - - size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) - { - size_t data_size = size * nmemb; - arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); - - log_debug("Curl response for arrow chunk size: %zu", data_size); - (void) arrowBufBuilder->Append(ptr, data_size); - return data_size; - } - - NON_JSON_RESP* callback_create_arrow_resp(void) - { - NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; - arrow_resp->buffer = new arrow::BufferBuilder(); - arrow_resp->write_callback = arrow_write_callback; - return arrow_resp; - } - -#else // SF_WIN32 -rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string -) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string -) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -void rs_arrow_destroy(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); -} - -SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_finish_result_set(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_string( - rs_arrow_t * rs, - size_t idx, - char ** out_data, - size_t * io_len, - size_t * io_capacity -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data -) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -size_t rs_arrow_get_total_row_count(rs_arrow_t * rs) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) -{ - log_error("Query results were fetched using Arrow"); - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -} - -size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) -{ - log_error("Query results were fetched using Arrow"); - return 0; -} - -NON_JSON_RESP* callback_create_arrow_resp(void) -{ - log_error("Query results were fetched using Arrow"); - return NULL; -} - -#endif // SF_WIN32 - -#ifdef __cplusplus -} // extern "C" -#endif diff --git a/cpp/lib/result_set_json.cpp b/cpp/lib/result_set_json.cpp deleted file mode 100644 index b276b865f5..0000000000 --- a/cpp/lib/result_set_json.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#include - -#include "memory.h" -#include "result_set_json.h" -#include "ResultSetJson.hpp" - - -#ifdef __cplusplus -extern "C" { -#endif - - rs_json_t * rs_json_create( - cJSON * rowset, - SF_COLUMN_DESC * metadata, - const char * tz_string - ) - { - rs_json_t * rs_struct = (rs_json_t *) SF_MALLOC(sizeof(rs_json_t)); - Snowflake::Client::ResultSetJson * rs_obj = new Snowflake::Client::ResultSetJson( - rowset, metadata, std::string(tz_string)); - rs_struct->rs_object = rs_obj; - - return rs_struct; - } - - void rs_json_destroy(rs_json_t * rs) - { - if (rs == NULL) - { - return; - } - - delete static_cast(rs->rs_object); - SF_FREE(rs); - } - - SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->appendChunk(chunk); - } - - SF_STATUS STDCALL rs_json_next(rs_json_t * rs) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->next(); - } - - SF_STATUS STDCALL rs_json_get_cell_as_bool(rs_json_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsBool(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int8(rs_json_t * rs, size_t idx, int8 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt8(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int32(rs_json_t * rs, size_t idx, int32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_int64(rs_json_t * rs, size_t idx, int64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsInt64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint8(rs_json_t * rs, size_t idx, uint8 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint8(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint32(rs_json_t * rs, size_t idx, uint32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_uint64(rs_json_t * rs, size_t idx, uint64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsUint64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_float32(rs_json_t * rs, size_t idx, float32 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat32(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_float64(rs_json_t * rs, size_t idx, float64 * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsFloat64(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_const_string( - rs_json_t * rs, - size_t idx, - const char ** out_data - ) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsConstString(idx, out_data); - } - - SF_STATUS STDCALL rs_json_get_cell_as_timestamp( - rs_json_t * rs, - size_t idx, - SF_TIMESTAMP * out_data - ) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellAsTimestamp(idx, out_data); - } - - - SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getCellStrlen(idx, out_data); - } - - size_t rs_json_get_row_count_in_chunk(rs_json_t * rs) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return 0; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->getRowCountInChunk(); - } - - SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data) - { - Snowflake::Client::ResultSetJson * rs_obj; - - if (rs == NULL) - { - return SF_STATUS_ERROR_NULL_POINTER; - } - - rs_obj = static_cast (rs->rs_object); - return rs_obj->isCellNull(idx, out_data); - } - -#ifdef __cplusplus -} // extern "C" -#endif diff --git a/include/snowflake/client.h b/include/snowflake/client.h index abd332f5be..0ba8bb0568 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -18,7 +18,7 @@ extern "C" { /** * API Name */ -#define SF_API_NAME "C API" +#define SF_API_NAME "ODBC" /** * SQLState code length @@ -284,8 +284,11 @@ typedef enum SF_GLOBAL_ATTRIBUTE { * Attributes for Snowflake statement context. */ typedef enum SF_STMT_ATTRIBUTE { - SF_STMT_USER_REALLOC_FUNC + SF_STMT_USER_REALLOC_FUNC, + SF_STMT_MULTI_STMT_COUNT } SF_STMT_ATTRIBUTE; +#define SF_MULTI_STMT_COUNT_UNSET (-1) +#define SF_MULTI_STMT_COUNT_UNLIMITED 0 /** * Snowflake Error @@ -442,6 +445,16 @@ typedef struct SF_CHUNK_DOWNLOADER SF_CHUNK_DOWNLOADER; */ typedef struct SF_PUT_GET_RESPONSE SF_PUT_GET_RESPONSE; +typedef void* result_set_ptr; + +/** + * An enumeration over all supported query result formats. + */ +typedef enum QueryResultFormat_e +{ + SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_FORMAT_UNKNOWN +} QueryResultFormat; + /** * Statement context */ @@ -451,9 +464,9 @@ typedef struct SF_STMT { char request_id[SF_UUID4_LEN]; SF_ERROR_STRUCT error; SF_CONNECT *connection; - void *qrf; + QueryResultFormat qrf; char *sql_text; - void *result_set; + result_set_ptr result_set; int64 chunk_rowcount; int64 total_rowcount; int64 total_fieldcount; @@ -465,6 +478,9 @@ typedef struct SF_STMT { SF_STATS *stats; void *stmt_attrs; sf_bool is_dml; + sf_bool is_multi_stmt; + void* multi_stmt_result_ids; + int64 multi_stmt_count; /** * User realloc function used in snowflake_fetch @@ -782,6 +798,15 @@ SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture); +/** + * Determines whether more results are available and, if so, + * initializes processing for the next one. + * @param sfstmt SNOWFLAKE_STMT context. + * + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt); + /** * Fetches the next row for the statement and stores on the bound buffer * if any. Noop if no buffer is bound. diff --git a/include/snowflake/version.h b/include/snowflake/version.h index b21a93dd34..83ddc59f55 100644 --- a/include/snowflake/version.h +++ b/include/snowflake/version.h @@ -5,6 +5,7 @@ #ifndef SNOWFLAKE_CLIENT_VERSION_H #define SNOWFLAKE_CLIENT_VERSION_H -#define SF_API_VERSION "1.0.14" +// TODO: temporary change for testing, will remove +#define SF_API_VERSION "3.0.1" #endif /* SNOWFLAKE_CLIENT_VERSION_H */ diff --git a/lib/client.c b/lib/client.c index 90eb215469..505a582991 100644 --- a/lib/client.c +++ b/lib/client.c @@ -66,6 +66,7 @@ sf_bool validate_application(const char *application); #define _SF_STMT_TYPE_DELETE (_SF_STMT_TYPE_DML + 0x300) #define _SF_STMT_TYPE_MERGE (_SF_STMT_TYPE_DML + 0x400) #define _SF_STMT_TYPE_MULTI_TABLE_INSERT (_SF_STMT_TYPE_DML + 0x500) +#define _SF_STMT_TYPE_MULTI_STMT 0xA000 /** * Detects statement type is DML @@ -1319,6 +1320,315 @@ static void STDCALL _snowflake_stmt_row_metadata_reset(SF_STMT *sfstmt) { sfstmt->stats = NULL; } +/** + * Setup result set from json response. + * could be result of a regular query or one of the results + * in multiple statements + */ +static sf_bool setup_result_with_json_resp(SF_STMT* sfstmt, cJSON* data) +{ + // Set Database info + _mutex_lock(&sfstmt->connection->mutex_parameters); + /* Set other parameters. Ignore the status */ + _set_current_objects(sfstmt, data); + _set_parameters_session_info(sfstmt->connection, data); + qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); + _mutex_unlock(&sfstmt->connection->mutex_parameters); + + // clean up from preivous result + sfstmt->chunk_rowcount = -1; + sfstmt->total_rowcount = -1; + sfstmt->total_fieldcount = -1; + sfstmt->total_row_index = -1; + + // Destroy chunk downloader + chunk_downloader_term(sfstmt->chunk_downloader); + sfstmt->chunk_downloader = NULL; + + int64 stmt_type_id; + if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { + /* failed to get statement type id */ + sfstmt->is_dml = SF_BOOLEAN_FALSE; + } else { + sfstmt->is_dml = detect_stmt_type(stmt_type_id); + } + cJSON* rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); + if (snowflake_cJSON_IsArray(rowtype)) { + _snowflake_stmt_desc_reset(sfstmt); + sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( + rowtype); + sfstmt->desc = set_description(rowtype); + } + cJSON* stats = snowflake_cJSON_GetObjectItem(data, "stats"); + if (snowflake_cJSON_IsObject(stats)) { + _snowflake_stmt_row_metadata_reset(sfstmt); + sfstmt->stats = set_stats(stats); + } else { + sfstmt->stats = NULL; + } + + // Determine query result format and detach rowset object from data. + cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); + char * qrf_str = snowflake_cJSON_GetStringValue(qrf); + cJSON * rowset = NULL; + + if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { +#ifdef SF_WIN32 + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, + "Query results were fetched using Arrow, " + "but the client library does not yet support decoding Arrow results", "", + sfstmt->sfqid); + + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +#endif + sfstmt->qrf = SF_ARROW_FORMAT; + rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); + if (!rowset) + { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + } + else if (strcmp(qrf_str, "json") == 0) { + sfstmt->qrf = SF_JSON_FORMAT; + if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) + { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + } + else { + log_error("Unsupported query result format: %s", qrf_str); + return SF_BOOLEAN_FALSE; + } + // Index starts at 0 and incremented each fetch + sfstmt->total_row_index = 0; + cJSON* chunks = NULL; + cJSON* chunk_headers = NULL; + char* qrmk = NULL; + // When the result set is sufficient large, the server response will contain + // an empty "rowset" object. Instead, it will have a "chunks" object that contains, + // among other fields, a URL from which the result set can be downloaded in chunks. + // In this case, we initialize the chunk downloader, which will download in the + // background as calls to snowflake_fetch() are made. + if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { + // We don't care if there is no qrmk, so ignore return code + json_copy_string(&qrmk, data, "qrmk"); + chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); + NON_JSON_RESP* (*callback_create_resp)(void) = NULL; + if (SF_ARROW_FORMAT == sfstmt->qrf) { + callback_create_resp = callback_create_arrow_resp; + } + sfstmt->chunk_downloader = chunk_downloader_init( + qrmk, + chunk_headers, + chunks, + 2, // thread count + 4, // fetch slot + &sfstmt->error, + sfstmt->connection->insecure_mode, + callback_create_resp, + sfstmt->connection->proxy, + sfstmt->connection->no_proxy, + get_retry_timeout(sfstmt->connection), + sfstmt->connection->retry_count); + SF_FREE(qrmk); + if (!sfstmt->chunk_downloader) { + // Unable to create chunk downloader. + // Error is set in chunk_downloader_init function. + return SF_BOOLEAN_FALSE; + } + // Even when the result set is split into chunks, JSON format will still + // response with the first chunk in "rowset", so be sure to include it. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + sfstmt->qrf, + sfstmt->connection->timezone); + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set); + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } else { + // Create a result set object and update the total rowcount. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + sfstmt->qrf, + sfstmt->connection->timezone); + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set); + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } + + return SF_BOOLEAN_TRUE; +} + +/** + * Setup result set from json response. + * could be result of a regular query or one of the results + * in multiple statements + */ +static sf_bool setup_multi_stmt_result(SF_STMT* sfstmt, cJSON* data) +{ + // Set Database info + _mutex_lock(&sfstmt->connection->mutex_parameters); + /* Set other parameters. Ignore the status */ + _set_current_objects(sfstmt, data); + _set_parameters_session_info(sfstmt->connection, data); + qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); + _mutex_unlock(&sfstmt->connection->mutex_parameters); + + if (sfstmt->multi_stmt_result_ids) + { + snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); + sfstmt->multi_stmt_result_ids = NULL; + } + char* result_ids = NULL; + if (json_copy_string(&result_ids, data, "resultIds")) + { + log_error("No valid resultIds found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_RESPONSE, + "No valid resultIds found in multiple statements response.", + SF_SQLSTATE_GENERAL_ERROR, + sfstmt->sfqid); + return SF_BOOLEAN_FALSE; + } + + // split result ids with comma(,) + cJSON* result_ids_json = snowflake_cJSON_CreateArray(); + char* start = result_ids; + char* end = NULL; + size_t len = strlen(result_ids); + while ((start - result_ids) < len) + { + end = strchr(start, ','); + if (!end) + { + // last part, set to end of the entire string + end = result_ids + len; + } + *end = '\0'; + snowflake_cJSON_AddItemToArray(result_ids_json, snowflake_cJSON_CreateString(start)); + start = end + 1; + } + + sfstmt->multi_stmt_result_ids = result_ids_json; + + return SF_STATUS_SUCCESS == snowflake_next_result(sfstmt); +} + +SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt) +{ + cJSON* result_id_json = NULL; + if (!sfstmt || !sfstmt->is_multi_stmt || !sfstmt->multi_stmt_result_ids || + !snowflake_cJSON_IsArray(sfstmt->multi_stmt_result_ids) || + !(result_id_json = snowflake_cJSON_DetachItemFromArray(sfstmt->multi_stmt_result_ids, 0))) + { + // no more results available. + return SF_STATUS_EOF; + } + + char* result_id = snowflake_cJSON_GetStringValue(result_id_json); + if (!result_id || (strlen(result_id) == 0)) + { + log_error("Empty result id found for multiple statements."); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_RESPONSE, + "Empty result id found in multiple statements response.", + SF_SQLSTATE_GENERAL_ERROR, + sfstmt->sfqid); + snowflake_cJSON_Delete(result_id_json); + return SF_STATUS_ERROR_BAD_RESPONSE; + } + + char* result_url = NULL; + size_t url_size = strlen(QUERY_RESULT_URL_FORMAT) - 2 + strlen(result_id) + 1; + result_url = (char*)SF_CALLOC(1, url_size); + if (!result_url) + { + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_OUT_OF_MEMORY, + "Run out of memory trying to create result url.", + SF_SQLSTATE_MEMORY_ALLOCATION_ERROR, + sfstmt->sfqid); + snowflake_cJSON_Delete(result_id_json); + return SF_STATUS_ERROR_OUT_OF_MEMORY; + } + sf_sprintf(result_url, url_size, QUERY_RESULT_URL_FORMAT, result_id); + snowflake_cJSON_Delete(result_id_json); + + cJSON* result = NULL; + if (!request(sfstmt->connection, &result, result_url, NULL, 0, NULL, NULL, + GET_REQUEST_TYPE, &sfstmt->error, SF_BOOLEAN_FALSE, + 0, sfstmt->connection->retry_count, get_retry_timeout(sfstmt->connection), + NULL, NULL, NULL, SF_BOOLEAN_FALSE)) + { + SF_FREE(result_url); + return sfstmt->error.error_code; + } + SF_FREE(result_url); + + cJSON* data = snowflake_cJSON_GetObjectItem(result, "data"); + + sf_bool success = SF_BOOLEAN_FALSE; + if ((json_copy_bool(&success, result, "success") != SF_JSON_ERROR_NONE) || !success) + { + cJSON *messageJson = NULL; + char *message = NULL; + cJSON *codeJson = NULL; + int64 code = -1; + if (json_copy_string_no_alloc(sfstmt->error.sqlstate, data, + "sqlState", SF_SQLSTATE_LEN)) { + log_debug("No valid sqlstate found in response"); + } + messageJson = snowflake_cJSON_GetObjectItem(result, "message"); + if (messageJson) { + message = messageJson->valuestring; + } + codeJson = snowflake_cJSON_GetObjectItem(result, "code"); + if (codeJson) { + code = (int64) strtol(codeJson->valuestring, NULL, 10); + } else { + log_debug("no code element."); + } + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, code, + message ? message + : "Query was not successful", + NULL, sfstmt->sfqid); + + snowflake_cJSON_Delete(result); + return sfstmt->error.error_code; + } + + setup_result_with_json_resp(sfstmt, data); + snowflake_cJSON_Delete(result); + + return SF_STATUS_SUCCESS; +} + /** * Returns what kind of params are being bound - Named / Positional * based on the first bind input entry. Should be used only if @@ -1443,14 +1753,18 @@ static void STDCALL _snowflake_stmt_reset(SF_STMT *sfstmt) { sfstmt->sql_text = NULL; if (sfstmt->result_set) { - rs_destroy(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); + rs_destroy(sfstmt->result_set); } sfstmt->result_set = NULL; - if (sfstmt->qrf) { - SF_FREE(sfstmt->qrf); + sfstmt->qrf = SF_FORMAT_UNKNOWN; + sfstmt->is_dml = SF_BOOLEAN_FALSE; + sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; + if (sfstmt->multi_stmt_result_ids) + { + snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); } - sfstmt->qrf = NULL; + sfstmt->multi_stmt_result_ids = NULL; if (_snowflake_get_current_param_style(sfstmt) == NAMED) { @@ -1541,7 +1855,7 @@ SF_STMT *STDCALL snowflake_stmt(SF_CONNECT *sf) { if (sfstmt) { _snowflake_stmt_reset(sfstmt); sfstmt->connection = sf; - + sfstmt->multi_stmt_count = SF_MULTI_STMT_COUNT_UNSET; } return sfstmt; } @@ -1771,12 +2085,11 @@ SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { sfstmt->result_set = rs_create_with_chunk( sfstmt->chunk_downloader->queue[index].chunk, sfstmt->desc, - (QueryResultFormat_t *) sfstmt->qrf, + sfstmt->qrf, sfstmt->connection->timezone); } else { rs_append_chunk( sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf, sfstmt->chunk_downloader->queue[index].chunk); } @@ -1948,12 +2261,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, const char *error_msg; cJSON *body = NULL; cJSON *data = NULL; - cJSON *rowtype = NULL; - cJSON *stats = NULL; cJSON *resp = NULL; - cJSON *chunks = NULL; - cJSON *chunk_headers = NULL; - char *qrmk = NULL; char *s_body = NULL; char *s_resp = NULL; sf_bool success = SF_BOOLEAN_FALSE; @@ -2042,7 +2350,8 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, // Create Body body = create_query_json_body(sfstmt->sql_text, sfstmt->sequence_counter, is_string_empty(sfstmt->connection->directURL) ? - NULL : sfstmt->request_id, is_describe_only); + NULL : sfstmt->request_id, is_describe_only, + sfstmt->multi_stmt_count); if (bindings != NULL) { /* binding parameters if exists */ snowflake_cJSON_AddItemToObject(body, "bindings", bindings); @@ -2175,155 +2484,29 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, "localLocation"); } else { - // Set Database info - _mutex_lock(&sfstmt->connection->mutex_parameters); - /* Set other parameters. Ignore the status */ - _set_current_objects(sfstmt, data); - _set_parameters_session_info(sfstmt->connection, data); - qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); - _mutex_unlock(&sfstmt->connection->mutex_parameters); int64 stmt_type_id; if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { - /* failed to get statement type id */ - sfstmt->is_dml = SF_BOOLEAN_FALSE; - } else { - sfstmt->is_dml = detect_stmt_type(stmt_type_id); - } - rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); - if (snowflake_cJSON_IsArray(rowtype)) { - sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( - rowtype); - _snowflake_stmt_desc_reset(sfstmt); - sfstmt->desc = set_description(rowtype); + /* failed to get statement type id */ + sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; } - stats = snowflake_cJSON_GetObjectItem(data, "stats"); - if (snowflake_cJSON_IsObject(stats)) { - _snowflake_stmt_row_metadata_reset(sfstmt); - sfstmt->stats = set_stats(stats); - } else { - sfstmt->stats = NULL; + else { + sfstmt->is_multi_stmt = (_SF_STMT_TYPE_MULTI_STMT == stmt_type_id); } - // Determine query result format and detach rowset object from data. - cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); - char * qrf_str = snowflake_cJSON_GetStringValue(qrf); - sfstmt->qrf = SF_CALLOC(1, sizeof(QueryResultFormat_t)); - cJSON * rowset = NULL; - - if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { -#ifdef SF_WIN32 - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, - "Query results were fetched using Arrow, " - "but the client library does not yet support decoding Arrow results", "", - sfstmt->sfqid); - - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -#endif - *((QueryResultFormat_t *) sfstmt->qrf) = ARROW_FORMAT; - rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); - if (!rowset) + if (sfstmt->is_multi_stmt) + { + if (!setup_multi_stmt_result(sfstmt, data)) { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); goto cleanup; } } - else if (strcmp(qrf_str, "json") == 0) { - *((QueryResultFormat_t *) sfstmt->qrf) = JSON_FORMAT; - if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) + else + { + if (!setup_result_with_json_resp(sfstmt, data)) { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); goto cleanup; } } - else { - log_error("Unsupported query result format: %s", qrf_str); - } - - // Index starts at 0 and incremented each fetch - sfstmt->total_row_index = 0; - - // When the result set is sufficient large, the server response will contain - // an empty "rowset" object. Instead, it will have a "chunks" object that contains, - // among other fields, a URL from which the result set can be downloaded in chunks. - // In this case, we initialize the chunk downloader, which will download in the - // background as calls to snowflake_fetch() are made. - if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { - // We don't care if there is no qrmk, so ignore return code - json_copy_string(&qrmk, data, "qrmk"); - chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); - NON_JSON_RESP* (*callback_create_resp)(void) = NULL; - if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) { - callback_create_resp = callback_create_arrow_resp; - } - - sfstmt->chunk_downloader = chunk_downloader_init( - qrmk, - chunk_headers, - chunks, - 2, // thread count - 4, // fetch slot - &sfstmt->error, - sfstmt->connection->insecure_mode, - callback_create_resp, - sfstmt->connection->proxy, - sfstmt->connection->no_proxy, - get_retry_timeout(sfstmt->connection), - sfstmt->connection->retry_count); - if (!sfstmt->chunk_downloader) { - // Unable to create chunk downloader. - // Error is set in chunk_downloader_init function. - goto cleanup; - } - - // Even when the result set is split into chunks, JSON format will still - // response with the first chunk in "rowset", so be sure to include it. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - (QueryResultFormat_t *)sfstmt->qrf, - sfstmt->connection->timezone); - - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf); - - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } else { - // Create a result set object and update the total rowcount. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - (QueryResultFormat_t *) sfstmt->qrf, - sfstmt->connection->timezone); - - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set, - (QueryResultFormat_t *) sfstmt->qrf); - - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } } } else if (json_error != SF_JSON_ERROR_NONE) { JSON_ERROR_MSG(json_error, error_msg, "Success code"); @@ -2371,7 +2554,6 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, snowflake_cJSON_Delete(body); snowflake_cJSON_Delete(resp); SF_FREE(s_body); - SF_FREE(qrmk); if (result_capture == NULL) { // If no result capture, we always free s_resp SF_FREE(s_resp); @@ -2451,6 +2633,9 @@ SF_STATUS STDCALL snowflake_stmt_get_attr( case SF_STMT_USER_REALLOC_FUNC: *value = sfstmt->user_realloc_func; break; + case SF_STMT_MULTI_STMT_COUNT: + *value = &sfstmt->multi_stmt_count; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2471,6 +2656,9 @@ SF_STATUS STDCALL snowflake_stmt_set_attr( case SF_STMT_USER_REALLOC_FUNC: sfstmt->user_realloc_func = (void*(*)(void*, size_t))value; break; + case SF_STMT_MULTI_STMT_COUNT: + sfstmt->multi_stmt_count = value ? *((int64*)value) : SF_MULTI_STMT_COUNT_UNSET; + break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2525,7 +2713,7 @@ SF_STATUS STDCALL _snowflake_column_null_checks(SF_STMT *sfstmt, void *value_ptr } SF_STATUS STDCALL _snowflake_next(SF_STMT *sfstmt) { - return rs_next(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); + return rs_next(sfstmt->result_set); } SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool *value_ptr) { @@ -2536,9 +2724,9 @@ SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool } if ((status = rs_get_cell_as_bool( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2551,9 +2739,9 @@ SF_STATUS STDCALL snowflake_column_as_uint8(SF_STMT *sfstmt, int idx, uint8 *val } if ((status = rs_get_cell_as_uint8( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2566,9 +2754,9 @@ SF_STATUS STDCALL snowflake_column_as_uint32(SF_STMT *sfstmt, int idx, uint32 *v } if ((status = rs_get_cell_as_uint32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2581,9 +2769,9 @@ SF_STATUS STDCALL snowflake_column_as_uint64(SF_STMT *sfstmt, int idx, uint64 *v } if ((status = rs_get_cell_as_uint64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2596,9 +2784,9 @@ SF_STATUS STDCALL snowflake_column_as_int8(SF_STMT *sfstmt, int idx, int8 *value } if ((status = rs_get_cell_as_int8( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2611,9 +2799,9 @@ SF_STATUS STDCALL snowflake_column_as_int32(SF_STMT *sfstmt, int idx, int32 *val } if ((status = rs_get_cell_as_int32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2626,9 +2814,9 @@ SF_STATUS STDCALL snowflake_column_as_int64(SF_STMT *sfstmt, int idx, int64 *val } if ((status = rs_get_cell_as_int64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2641,9 +2829,9 @@ SF_STATUS STDCALL snowflake_column_as_float32(SF_STMT *sfstmt, int idx, float32 } if ((status = rs_get_cell_as_float32( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2656,9 +2844,9 @@ SF_STATUS STDCALL snowflake_column_as_float64(SF_STMT *sfstmt, int idx, float64 } if ((status = rs_get_cell_as_float64( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2671,9 +2859,9 @@ SF_STATUS STDCALL snowflake_column_as_timestamp(SF_STMT *sfstmt, int idx, SF_TIM } if ((status = rs_get_cell_as_timestamp( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2686,9 +2874,9 @@ SF_STATUS STDCALL snowflake_column_as_const_str(SF_STMT *sfstmt, int idx, const } if ((status = rs_get_cell_as_const_string( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2909,16 +3097,15 @@ SF_STATUS STDCALL snowflake_column_as_str(SF_STMT *sfstmt, int idx, char **value const char* str_val = NULL; if ((status = rs_get_cell_as_const_string( sfstmt->result_set, - sfstmt->qrf, idx, &str_val)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); return status; } - if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) + if (SF_ARROW_FORMAT == sfstmt->qrf) { // For Arrow the const string is formatted already return snowflake_raw_value_to_str_rep(sfstmt, str_val, @@ -2949,9 +3136,9 @@ SF_STATUS STDCALL snowflake_column_strlen(SF_STMT *sfstmt, int idx, size_t *valu } if ((status = rs_get_cell_strlen( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } @@ -2964,9 +3151,9 @@ SF_STATUS STDCALL snowflake_column_is_null(SF_STMT *sfstmt, int idx, sf_bool *va } if ((status = rs_is_cell_null( - sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); } return status; } diff --git a/lib/client_int.h b/lib/client_int.h index ed6b46c5a2..e8b111cd68 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -29,6 +29,7 @@ #define QUERY_URL "/queries/v1/query-request" #define RENEW_SESSION_URL "/session/token-request" #define DELETE_SESSION_URL "/session" +#define QUERY_RESULT_URL_FORMAT "/queries/%s/result" // not used for now but add for URL checking on connection requests #define AUTHENTICATOR_URL "/session/authenticator-request" diff --git a/lib/connection.c b/lib/connection.c index 64f9578c83..3c9d5f2570 100644 --- a/lib/connection.c +++ b/lib/connection.c @@ -201,7 +201,12 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, return body; } -cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only) { +cJSON *STDCALL create_query_json_body(const char *sql_text, + int64 sequence_id, + const char *request_id, + sf_bool is_describe_only, + int64 multi_stmt_count) +{ cJSON *body; double submission_time; // Create body @@ -221,11 +226,27 @@ cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, c snowflake_cJSON_AddStringToObject(body, "requestId", request_id); } + cJSON* parameters = NULL; + if (multi_stmt_count >= 0) + { + parameters = snowflake_cJSON_CreateObject(); + snowflake_cJSON_AddNumberToObject(parameters, "MULTI_STATEMENT_COUNT", (double)multi_stmt_count); + } + #ifdef SF_WIN32 - cJSON * parameters = snowflake_cJSON_CreateObject(); + if (!parameters) + { + parameters = snowflake_cJSON_CreateObject(); + } snowflake_cJSON_AddStringToObject(parameters, "C_API_QUERY_RESULT_FORMAT", "JSON"); - snowflake_cJSON_AddItemToObject(body, "parameters", parameters); + + // temporary code to fake as ODBC to have multiple statements enabled + snowflake_cJSON_AddStringToObject(parameters, "ODBC_QUERY_RESULT_FORMAT", "JSON"); #endif + if (parameters) + { + snowflake_cJSON_AddItemToObject(body, "parameters", parameters); + } return body; } diff --git a/lib/connection.h b/lib/connection.h index af56504750..7dfcfb4688 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -201,9 +201,14 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, const char *application, co * @param sequence_id Sequence ID from the Snowflake Connection object. * @param request_id requestId to be passed as a part of body instead of header. * @param is_describe_only is the query describe only. + * @param multi_stmt_count The value of MULTI_STATEMENT_COUNT set with the query. No setting if < 0. * @return Query cJSON Body. */ -cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only); +cJSON *STDCALL create_query_json_body(const char *sql_text, + int64 sequence_id, + const char *request_id, + sf_bool is_describe_only, + int64 multi_stmt_count); /** * Creates a cJSON blob that is used to renew a session with Snowflake. cJSON blob must be freed by the caller using diff --git a/lib/result_set.h b/lib/result_set.h index 865d8eba69..c5faac33ff 100644 --- a/lib/result_set.h +++ b/lib/result_set.h @@ -6,26 +6,12 @@ #define SNOWFLAKE_RESULTSET_H #include "snowflake/client.h" -#include "result_set_arrow.h" -#include "result_set_json.h" +#include "connection.h" #ifdef __cplusplus extern "C" { #endif - // Utility ===================================================================================== - - /** - * An enumeration over all supported query result formats. - * This is used to help deciding which result set to create. - * - * NOTE: Keep the order consistent with rowset_key_map! - */ - typedef enum QueryResultFormat - { - ARROW_FORMAT, JSON_FORMAT, FORMAT_MAX - } QueryResultFormat_t; - // Result Set API ============================================================================== /** @@ -39,10 +25,10 @@ extern "C" { * * @return the created result set. */ - void * rs_create_with_json_result( + result_set_ptr rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string); /** @@ -56,55 +42,50 @@ extern "C" { * * @return the created result set. */ - void * rs_create_with_chunk( + result_set_ptr rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat_t * query_result_format, + QueryResultFormat query_result_format, const char * tz_string); /** * Destructor. * * @param rs The ResultSet object. - * @param query_result_format The query result format. */ - void rs_destroy(void * rs, QueryResultFormat_t * query_result_format); + void rs_destroy(result_set_ptr rs); /** * Appends the given chunk to the internal result set. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param chunk The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL - rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk); + rs_append_chunk(result_set_ptr rs, void * chunk); /** * Advances to the next row. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format); + SF_STATUS STDCALL rs_next(result_set_ptr rs); /** * Writes the value of the current cell as a boolean to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_bool( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data); @@ -112,15 +93,13 @@ extern "C" { * Writes the value of the current cell as an int8 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int8 * out_data); @@ -128,15 +107,13 @@ extern "C" { * Writes the value of the current cell as an int32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int32 * out_data); @@ -144,15 +121,13 @@ extern "C" { * Writes the value of the current cell as an int64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, int64 * out_data); @@ -160,15 +135,13 @@ extern "C" { * Writes the value of the current cell as a uint8 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint8( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint8 * out_data); @@ -176,15 +149,13 @@ extern "C" { * Writes the value of the current cell as a uint32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint32 * out_data); @@ -192,15 +163,13 @@ extern "C" { * Writes the value of the current cell as a uint64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, uint64 * out_data); @@ -208,15 +177,13 @@ extern "C" { * Writes the value of the current cell as a float32 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float32( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float32 * out_data); @@ -224,15 +191,13 @@ extern "C" { * Writes the value of the current cell as a float64 to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float64( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, float64 * out_data); @@ -240,15 +205,13 @@ extern "C" { * Writes the value of the current cell as a constant C-string to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_const_string( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, const char ** out_data); @@ -256,15 +219,13 @@ extern "C" { * Writes the value of the current cell as a timestamp to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_timestamp( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, SF_TIMESTAMP * out_data); @@ -272,15 +233,13 @@ extern "C" { * Writes the length of the current cell to the provided buffer. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_strlen( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, size_t * out_data); @@ -288,25 +247,22 @@ extern "C" { * Gets the number of rows in the current chunk being processed. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the number of rows in the current chunk. */ - size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format); + size_t rs_get_row_count_in_chunk(result_set_ptr rs); /** * Indiciates whether the current cell is null or not. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_is_cell_null( - void * rs, - QueryResultFormat_t * query_result_format, + result_set_ptr rs, size_t idx, sf_bool * out_data); @@ -314,27 +270,22 @@ extern "C" { * Get the latest error code. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the latest error code. 0 if no error. */ - SF_STATUS STDCALL rs_get_error( - void * rs, - QueryResultFormat_t * query_result_format - ); + SF_STATUS STDCALL rs_get_error(result_set_ptr rs); /** * Get the latest error code. * * @param rs The ResultSet object. - * @param query_result_format The query result format. * * @return the latest error message. empty string if no error. */ - const char* rs_get_error_message( - void * rs, - QueryResultFormat_t * query_result_format - ); + const char* rs_get_error_message(result_set_ptr rs); + + // return callback struct for arrow chunk downloading + NON_JSON_RESP* callback_create_arrow_resp(void); #ifdef __cplusplus } // extern "C" diff --git a/lib/result_set_arrow.h b/lib/result_set_arrow.h deleted file mode 100644 index 3250aad46f..0000000000 --- a/lib/result_set_arrow.h +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef SNOWFLAKE_RESULTSETARROW_H -#define SNOWFLAKE_RESULTSETARROW_H - -#include "cJSON.h" -#include "snowflake/basic_types.h" -#include "snowflake/client.h" -#include "connection.h" - -#ifdef __cplusplus -extern "C" { -#endif - - /** - * A result set interface for Arrow result format. - * - * @see cpp/lib/ResultSet.hpp - * @see cpp/lib/ResultSetArrow.hpp - */ - typedef struct rs_arrow { - void * rs_object; - } rs_arrow_t; - - /** - * Parameterized constructor. - * Initializes the result set with rowset64 data in json result set. - * - * @param json_rowset64 A pointer to the rowset64 data in json result set. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_arrow_t * rs_arrow_create_with_json_result( - cJSON * json_rowset64, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Parameterized constructor. - * Initializes the result set with the first arrow chunk of the result set. - * - * @param initial_chunk A pointer to the first chunk of the result set. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_arrow_t * rs_arrow_create_with_chunk( - NON_JSON_RESP * initial_chunk, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Destructor. - */ - void rs_arrow_destroy(rs_arrow_t * rs); - - /** - * Appends the given chunk to the internal result set. - * - * @param rs The ResultSetArrow object. - * @param chunk The chunk to append. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk); - - /** - * Advances to the next row. - * - * @param rs The ResultSetArrow object. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs); - - /** - * Writes the value of the current cell as a boolean to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_bool( - rs_arrow_t * rs, - size_t idx, - sf_bool * out_data); - - /** - * Writes the value of the current cell as an int8 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int8( - rs_arrow_t * rs, - size_t idx, - int8 * out_data); - - /** - * Writes the value of the current cell as an int32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int32( - rs_arrow_t * rs, - size_t idx, - int32 * out_data); - - /** - * Writes the value of the current cell as an int64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_int64( - rs_arrow_t * rs, - size_t idx, - int64 * out_data); - - /** - * Writes the value of the current cell as a uint8 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint8( - rs_arrow_t * rs, - size_t idx, - uint8 * out_data); - - /** - * Writes the value of the current cell as a uint32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint32( - rs_arrow_t * rs, - size_t idx, - uint32 * out_data); - - /** - * Writes the value of the current cell as a uint64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_uint64( - rs_arrow_t * rs, - size_t idx, - uint64 * out_data); - - /** - * Writes the value of the current cell as a float32 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_float32( - rs_arrow_t * rs, - size_t idx, - float32 * out_data); - - /** - * Writes the value of the current cell as a float64 to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_float64( - rs_arrow_t * rs, - size_t idx, - float64 * out_data); - - /** - * Writes the value of the current cell as a constant C-string to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( - rs_arrow_t * rs, - size_t idx, - const char ** out_data); - - /** - * Writes the value of the current cell as a timestamp to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( - rs_arrow_t * rs, - size_t idx, - SF_TIMESTAMP * out_data); - - /** - * Writes the length of the current cell to the provided buffer. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data); - - /** - * Gets the number of rows in the current chunk being processed. - * - * @param rs The ResultSetArrow object. - * - * @return the number of rows in the current chunk. - */ - size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs); - - /** - * Indiciates whether the current cell is null or not. - * - * @param rs The ResultSetArrow object. - * @param idx The index of the row to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data); - - NON_JSON_RESP* callback_create_arrow_resp(void); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // SNOWFLAKE_RESULTSETARROW_H diff --git a/lib/result_set_json.h b/lib/result_set_json.h deleted file mode 100644 index c2a1d28d20..0000000000 --- a/lib/result_set_json.h +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef SNOWFLAKE_RESULTSETJSON_H -#define SNOWFLAKE_RESULTSETJSON_H - -#include "cJSON.h" -#include "snowflake/basic_types.h" -#include "snowflake/client.h" - -#ifdef __cplusplus -extern "C" { -#endif - - /** - * A result set interface for JSON result format. - * - * @see cpp/lib/ResultSet.hpp - * @see cpp/lib/ResultSetJson.hpp - */ - typedef struct rs_json { - void * rs_object; - } rs_json_t; - - /** - * Parameterized constructor. - * Initializes the result set with required information as well as data. - * - * @param rowset A pointer to the result set data. - * @param metadata A pointer to the metadata for the result set. - * @param tz_string The time zone. - */ - rs_json_t * rs_json_create( - cJSON * rowset, - SF_COLUMN_DESC * metadata, - const char * tz_string); - - /** - * Destructor. - */ - void rs_json_destroy(rs_json_t * rs); - - /** - * Appends the given chunk to the internal result set. - * - * @param rs The ResultSetJson object. - * @param chunk The chunk to append. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk); - - /** - * Advances to next row. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_next(rs_json_t * rs); - - /** - * Writes the value of the current cell as a boolean to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_bool( - rs_json_t * rs, - size_t idx, - sf_bool * out_data); - - /** - * Writes the value of the current cell as an int8 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int8( - rs_json_t * rs, - size_t idx, - int8 * out_data); - - /** - * Writes the value of the current cell as an int32 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int32( - rs_json_t * rs, - size_t idx, - int32 * out_data); - - /** - * Writes the value of the current cell as an int64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_int64( - rs_json_t * rs, - size_t idx, - int64 * out_data); - - /** - * Writes the value of the current cell as a uint8 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint8( - rs_json_t * rs, - size_t idx, - uint8 * out_data); - - /** - * Writes the value of the current cell as a uint32 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint32( - rs_json_t * rs, - size_t idx, - uint32 * out_data); - - /** - * Writes the value of the current cell as a uint64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_uint64( - rs_json_t * rs, - size_t idx, - uint64 * out_data); - - /** - * Writes the value of the current cell as a float32 to the provided buffer. - * - * @param rs The ResultSet object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_float32( - rs_json_t * rs, - size_t idx, - float32 * out_data); - - /** - * Writes the value of the current cell as a float64 to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_float64( - rs_json_t * rs, - size_t idx, - float64 * out_data); - - /** - * Writes the value of the current cell as a constant C-string to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_const_string( - rs_json_t * rs, - size_t idx, - const char ** out_data); - - /** - * Writes the value of the current cell as a timestamp to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_as_timestamp( - rs_json_t * rs, - size_t idx, - SF_TIMESTAMP * out_data); - - /** - * Writes the length of the current cell to the provided buffer. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data); - - /** - * Gets the number of rows in the current chunk being processed. - * - * @param rs The ResultSetJson object. - * - * @return the number of rows in the current chunk. - */ - size_t rs_json_get_row_count_in_chunk(rs_json_t * rs); - - /** - * Gets the total number of rows in the entire result set. - * - * @param rs The ResultSetJson object. - * - * @return the number of rows in the result set. - */ - size_t rs_json_get_total_row_count(rs_json_t * rs); - - /** - * Indiciates whether the current cell is null or not. - * - * @param rs The ResultSetJson object. - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // SNOWFLAKE_RESULTSETJSON_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 20b09d62eb..76e811808b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -42,6 +42,7 @@ SET(TESTS_C test_get_describe_only_query_result test_stmt_functions test_unit_mfa_auth + test_multiple_statements # FEATURE_INCREASED_MAX_LOB_SIZE_IN_MEMORY is internal switch # will enable lob test when the change on server side will be published # test_lob diff --git a/tests/test_multiple_statements.c b/tests/test_multiple_statements.c new file mode 100644 index 0000000000..f4a2cf37e3 --- /dev/null +++ b/tests/test_multiple_statements.c @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ +#include +#include "utils/test_setup.h" + +void test_multi_stmt_transaction(void **unused) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); + assert_int_equal(status, SF_STATUS_SUCCESS); + + int64 multi_stmt_count = 5; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "begin;\n" + "delete from test_multi_txn;\n" + "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" + "commit;\n" + "select count(*) from test_multi_txn", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (delete) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // third statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 2); + + // fourth statement (commit) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // fifth statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + int counter = 0; + int64 out; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &out); + assert_int_equal(out, 2); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, 1); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +void test_multi_stmt_transaction_rollback(void **unused) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); + assert_int_equal(status, SF_STATUS_SUCCESS); + + int64 multi_stmt_count = 5; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "begin;\n" + "delete from test_multi_txn;\n" + "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" + "rollback;\n" + "select count(*) from test_multi_txn", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (delete) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // third statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 2); + + // fourth statement (rollback) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // fifth statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + int counter = 0; + int64 out; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &out); + assert_int_equal(out, 1); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, 1); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +void test_multi_stmt_with_large_result(void **unused) +{ + const int rownum = 100000; + SF_CONNECT *sf = setup_snowflake_connection(); + + // TODO SNOW-1526335 + // Sometime we can't get OCSP response from cache server or responder + // Usually happen on GCP and should be ignored by FAIL_OPEN + // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now + // so we have to disable OCSP validation to around it. + // Will remove this code when adding support for FAIL_OPEN (which is + // the default behavior for all other drivers) + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "GCP", 4)) { + sf_bool insecure_mode = SF_BOOLEAN_TRUE; + snowflake_set_attribute(sf, SF_CON_INSECURE_MODE, &insecure_mode); + } + + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + /* query */ + SF_STMT *sfstmt = snowflake_stmt(sf); + int64 multi_stmt_count = 3; + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + assert_int_equal(status, SF_STATUS_SUCCESS); + + status = snowflake_query(sfstmt, + "create or replace temporary table test_multi_large(c1 number, c2 number);\n" + "insert into test_multi_large select seq4(), TO_VARCHAR(seq4()) from table(generator(rowcount => 100000));\n" + "select * from test_multi_large", + 0); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // first statement (begin) + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), 1); + + // second statement (insert) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), 1); + assert_int_equal(snowflake_affected_rows(sfstmt), rownum); + + // third statement (select) + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); + assert_int_equal(snowflake_num_rows(sfstmt), rownum); + + int counter = 0; + int64 intout; + const char* strout; + char strexp[64]; + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { + snowflake_column_as_int64(sfstmt, 1, &intout); + assert_int_equal(intout, counter); + snowflake_column_as_const_str(sfstmt, 2, &strout); + sprintf(strexp, "%d", counter); + assert_string_equal(strout, strexp); + ++counter; + } + assert_int_equal(status, SF_STATUS_EOF); + assert_int_equal(counter, rownum); + + // no more result + assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); +} + +/* helper function for testing multi_stmt_count, running a query with + * multiple statements of 3. + * @param use_session_param Whethter to set MULTI_STATEMENT_COUNT through + * session parameter or statement attribute. + * @param count The count number to be set + * @return True if the query succeeded, otherwise false. + */ +sf_bool test_multi_stmt_core(sf_bool use_session_param, int count) +{ + SF_CONNECT *sf = setup_snowflake_connection(); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + SF_STMT* sfstmt = snowflake_stmt(sf); + + char query[1024]; + int64 multi_stmt_count = count; + if (SF_BOOLEAN_TRUE == use_session_param) + { + sprintf(query, "alter session set MULTI_STATEMENT_COUNT=%d", count); + status = snowflake_query(sfstmt, query, 0); + } + else + { + status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); + } + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sfstmt->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + sprintf(query, "%s", "select 1; select 2; select 3"); + status = snowflake_query(sfstmt, query, 0); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); + + return (status == SF_STATUS_SUCCESS) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; +} + +void test_multi_stmt_count_session_param_off(void** unused) +{ + // disable multiple statements by setting session parameter to 1 + // the query is expected to fail + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 1), SF_BOOLEAN_FALSE); +} + +void test_multi_stmt_count_session_param_on(void** unused) +{ + // enable multiple statements by setting session parameter to 0 + // the query should work + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 0), SF_BOOLEAN_TRUE); +} + +void test_multi_stmt_count_stmt_attr_match(void** unused) +{ + // set statement attribute with match number + // the query should work + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 3), SF_BOOLEAN_TRUE); +} + +void test_multi_stmt_count_stmt_attr_mismatch(void** unused) +{ + // set statement attribute with mismatch number + // the query is expected to fail + assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 2), SF_BOOLEAN_FALSE); +} + +int main(void) { + initialize_test(SF_BOOLEAN_FALSE); + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_multi_stmt_transaction), + cmocka_unit_test(test_multi_stmt_transaction_rollback), + cmocka_unit_test(test_multi_stmt_with_large_result), + cmocka_unit_test(test_multi_stmt_count_session_param_off), + cmocka_unit_test(test_multi_stmt_count_session_param_on), + cmocka_unit_test(test_multi_stmt_count_stmt_attr_match), + cmocka_unit_test(test_multi_stmt_count_stmt_attr_mismatch), + }; + int ret = cmocka_run_group_tests(tests, NULL, NULL); + snowflake_global_term(); + return ret; +} From f1622329335f96cce0a8ae733c6ccaa2712f0924 Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 17 Oct 2024 13:35:35 -0700 Subject: [PATCH 24/74] Revert from using std::filesystem, add back check file exists due to boost 32-bit windows debug issues --- CMakeLists.txt | 4 +-- cpp/lib/ClientConfigParser.cpp | 40 ++++++++++++++---------- include/snowflake/ClientConfigParser.hpp | 7 +++++ 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43424000b9..b2fba3eed5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -517,11 +517,11 @@ endif() if (LINUX) # Linux - target_link_libraries(snowflakeclient rt dl z stdc++fs) + target_link_libraries(snowflakeclient rt dl z) endif () if (APPLE) # OSX. no librt is required. - target_link_libraries(snowflakeclient dl z stdc++fs) + target_link_libraries(snowflakeclient dl z) endif () if (WIN32) # Windows diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 882d8f0580..4c5c569c9b 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -12,18 +12,12 @@ #include #include +#include + #ifndef _WIN32 #include #endif -#ifdef _WIN32 -#include -using namespace std::filesystem; -#elif defined(__linux__) -#include -using namespace std::experimental::filesystem; -#endif - using namespace Snowflake::Client; @@ -101,7 +95,7 @@ std::string ClientConfigParser::resolveClientConfigPath( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (is_regular_file(binaryDirFilePath)) + if (checkFileExists(binaryDirFilePath.c_str())) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -126,7 +120,7 @@ std::string ClientConfigParser::resolveClientConfigPath( homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; } } - if (is_regular_file(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -138,7 +132,7 @@ std::string ClientConfigParser::resolveClientConfigPath( if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (is_regular_file(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -152,6 +146,18 @@ std::string ClientConfigParser::resolveClientConfigPath( } // Private ===================================================================== +//////////////////////////////////////////////////////////////////////////////// +bool ClientConfigParser::checkFileExists(const std::string& in_filePath) +{ + FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); + if (file != nullptr) + { + fclose(file); + return true; + } + return false; +} + //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::parseConfigFile( const std::string& in_filePath, @@ -173,7 +179,9 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - const int fileSize = file_size(in_filePath); + fseek(configFile, 0, SEEK_END); + long fileSize = ftell(configFile); + fseek(configFile, 0, SEEK_SET); char* buffer = (char*)malloc(fileSize); if (buffer) { @@ -230,10 +238,10 @@ void ClientConfigParser::parseConfigFile( //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) { - file_status fileStatus = status(in_filePath); - perms permissions = fileStatus.permissions(); - if ((perms::group_write == (permissions & perms::group_write)) || - (perms::others_write == (permissions & perms::others_write))) + boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); + boost::filesystem::perms permissions = fileStatus.permissions(); + if (permissions & boost::filesystem::group_write || + permissions & boost::filesystem::others_write) { CXX_LOG_ERROR( "sf", diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 263ab0d45c..87a2cd2e41 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -53,6 +53,13 @@ namespace Client // Private ================================================================= private: + /** + * @brief Check if the file exists. + * + * @param in_filePath The file path to check. + */ + bool checkFileExists(const std::string& in_filePath); + /** * @brief Resolve the client config path. * From 04993e8b1ce34a8164f59f11cd93684f8c7c98d4 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:49:44 -0700 Subject: [PATCH 25/74] Fix boost error --- cpp/lib/ClientConfigParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 4c5c569c9b..335de78594 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -12,6 +12,7 @@ #include #include +#undef snprintf #include #ifndef _WIN32 From 0f494eec4b0ee25205ba145e7c23cc31d9ba1b8a Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:02:04 -0700 Subject: [PATCH 26/74] Fix test cases --- tests/test_unit_logger.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index a4fe7823d2..2980b10943 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -139,7 +139,6 @@ void test_client_config_log_init(void** unused) { remove(LOG_PATH); } - /** * Tests timing of log file creation */ @@ -261,8 +260,8 @@ int main(void) { cmocka_unit_test(test_client_config_log_invalid_json), cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), -#ifndef _WIN32 cmocka_unit_test(test_log_creation), +#ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), #endif }; From 0aaea59f1611be9e0ff8eb301a435d82188f5857 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:05:19 -0700 Subject: [PATCH 27/74] Fix logger unit tests --- tests/test_unit_logger.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 2980b10943..893213420f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -7,11 +7,7 @@ #include #include "memory.h" -#ifdef _WIN32 -inline int access(const char* pathname, int mode) { - return _access(pathname, mode); -} -#else +#ifndef _WIN32 #include #endif @@ -64,6 +60,7 @@ void test_client_config_log_invalid_json(void** unused) { remove(configFilePath); } +#ifndef _WIN32 /** * Tests log settings from client config file */ @@ -93,11 +90,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -127,11 +124,11 @@ void test_client_config_log_init(void** unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -147,7 +144,7 @@ void test_log_creation(void **unused) { // ensure the log file doesn't exist at the beginning remove(logname); - assert_int_not_equal(access(logname, 0), 0); + assert_int_not_equal(access(logname, F_OK), 0); log_set_lock(NULL); log_set_level(SF_LOG_WARN); @@ -156,17 +153,16 @@ void test_log_creation(void **unused) { // info log won't trigger the log file creation since log level is set to warning log_info("dummy info log"); - assert_int_not_equal(access(logname, 0), 0); + assert_int_not_equal(access(logname, F_OK), 0); // warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(logname, 0), 0); + assert_int_equal(access(logname, F_OK), 0); log_close(); remove(logname); } -#ifndef _WIN32 /** * Tests masking secret information in log */ @@ -258,10 +254,10 @@ int main(void) { cmocka_unit_test(test_log_str_to_level), cmocka_unit_test(test_invalid_client_config_path), cmocka_unit_test(test_client_config_log_invalid_json), +#ifndef _WIN32 cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), cmocka_unit_test(test_log_creation), -#ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), #endif }; From 6bc1b928311c56794c045d50edba168bd8f62c39 Mon Sep 17 00:00:00 2001 From: SimbaGithub <48035983+SimbaGithub@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:21:26 -0700 Subject: [PATCH 28/74] SNOW-692945: refactoring for resultset C wrapper (#724) --- cpp/lib/ResultSet.cpp | 40 +++++++++++++++++++++++++- cpp/lib/ResultSet.hpp | 26 ++++++++++++++++- cpp/lib/ResultSetArrow.cpp | 11 +++++--- cpp/lib/ResultSetArrow.hpp | 22 ++++++++++++--- cpp/lib/ResultSetJson.cpp | 5 ++-- cpp/lib/ResultSetJson.hpp | 4 +-- cpp/lib/result_set.cpp | 57 ++++---------------------------------- 7 files changed, 100 insertions(+), 65 deletions(-) diff --git a/cpp/lib/ResultSet.cpp b/cpp/lib/ResultSet.cpp index bd8484a20f..e72769c8f9 100644 --- a/cpp/lib/ResultSet.cpp +++ b/cpp/lib/ResultSet.cpp @@ -11,6 +11,8 @@ #include "snowflake/SF_CRTFunctionSafe.h" #include "DataConversion.hpp" #include "ResultSet.hpp" +#include "ResultSetJson.hpp" +#include "ResultSetArrow.hpp" namespace Snowflake { @@ -36,7 +38,7 @@ ResultSet::ResultSet(QueryResultFormat format) : ResultSet::ResultSet( SF_COLUMN_DESC * metadata, - std::string tzString, + const std::string& tzString, QueryResultFormat format ) : m_currChunkIdx(0), @@ -54,5 +56,41 @@ ResultSet::ResultSet( ; } +ResultSet* ResultSet::CreateResultFromJson(cJSON* json_rowset, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string) +{ + switch (query_result_format) + { +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, tz_string); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson(json_rowset, metadata, tz_string); + default: + return nullptr; + } +} + +ResultSet* ResultSet::CreateResultFromChunk(void* initial_chunk, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string) +{ + switch (query_result_format) + { +#ifndef SF_WIN32 + case SF_ARROW_FORMAT: + return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, tz_string); +#endif + case SF_JSON_FORMAT: + return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, tz_string); + default: + return nullptr; + } +} + } // namespace Client } // namespace Snowflake diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index 157bd9cc99..bd5ee8d0e4 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -40,7 +40,17 @@ class ResultSet * @param metadata The metadata of the result set. * @param tzString The time zone. */ - ResultSet(SF_COLUMN_DESC * metadata, std::string tzString, QueryResultFormat format); + ResultSet(SF_COLUMN_DESC * metadata, const std::string& tzString, QueryResultFormat format); + + static ResultSet* CreateResultFromJson(cJSON* json_rowset, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string); + + static ResultSet* CreateResultFromChunk(void* initial_chunk, + SF_COLUMN_DESC* metadata, + QueryResultFormat query_result_format, + const std::string& tz_string); /** * Destructor. @@ -49,6 +59,20 @@ class ResultSet // API methods ================================================================================= + /** + * Frees the previously held chunk in m_chunk and sets the current chunk to the given chunk. + * Each result type should override this function to handle chunks, + * while some result might not apply (won't be called), return error by default. + * + * @param chunkPtr The chunk to append. + * + * @return 0 if successful, otherwise an error is returned. + */ + virtual SF_STATUS STDCALL appendChunk(void* chunkPtr) + { + return SF_STATUS_ERROR_GENERAL; + } + /** * Advances to the next row. * diff --git a/cpp/lib/ResultSetArrow.cpp b/cpp/lib/ResultSetArrow.cpp index ef20ded1fe..6b20a6b022 100644 --- a/cpp/lib/ResultSetArrow.cpp +++ b/cpp/lib/ResultSetArrow.cpp @@ -29,7 +29,7 @@ ResultSetArrow::ResultSetArrow() : ResultSetArrow::ResultSetArrow( arrow::BufferBuilder* initialChunk, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_ARROW_FORMAT) { @@ -44,10 +44,11 @@ ResultSetArrow::ResultSetArrow( ResultSetArrow::ResultSetArrow( cJSON * jsonRowset64, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_ARROW_FORMAT) { + NON_JSON_RESP resp; arrow::BufferBuilder* bufferBuilder = NULL; if (jsonRowset64) { @@ -60,8 +61,9 @@ ResultSetArrow::ResultSetArrow( (void)bufferBuilder->Append((void*)decodedRowsetStr.c_str(), decodedRowsetStr.length()); } } + resp.buffer = bufferBuilder; - this->appendChunk(bufferBuilder); + this->appendChunk(&resp); // Reset row indices so that they can be re-used by public API. m_currChunkIdx = 0; @@ -76,8 +78,9 @@ ResultSetArrow::~ResultSetArrow() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetArrow::appendChunk(arrow::BufferBuilder * chunk) +SF_STATUS STDCALL ResultSetArrow::appendChunk(void* chunkPtr) { + arrow::BufferBuilder * chunk = (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunkPtr)->buffer); if (chunk == nullptr) { if (m_isFirstChunk) diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 1805094b12..3ad409dd12 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -62,7 +62,7 @@ class ResultSetArrow : public Snowflake::Client::ResultSet * @param metadata An array of metadata objects for each column. * @param tzString The time zone. */ - ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, std::string tzString); + ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, const std::string& tzString); /** @@ -76,7 +76,21 @@ class ResultSetArrow : public Snowflake::Client::ResultSet * @param metadata An array of metadata objects for each column. * @param tzString The time zone. */ - ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string tzString); + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, const std::string& tzString); + + + /** + * Parameterized constructor. + * + * This constructor will initialize m_records with the (partial) results + * contained in the initial chunk. It will also initialize m_metadata with + * the metadata in "metadata". + * + * @param jsonRowset64 A pointer to the rowset64 data in json result set. + * @param metadata An array of metadata objects for each column. + * @param tzString The time zone. + */ + ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string& tzString); /** * Destructor. @@ -88,11 +102,11 @@ class ResultSetArrow : public Snowflake::Client::ResultSet /** * Appends the given chunk to the internal result set. * - * @param chunk The chunk to append. + * @param chunkPtr The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(arrow::BufferBuilder * chunk); + SF_STATUS STDCALL appendChunk(void* chunkPtr); /** * Advances the internal iterator to the next row. diff --git a/cpp/lib/ResultSetJson.cpp b/cpp/lib/ResultSetJson.cpp index a9bfc42490..a3e5510990 100644 --- a/cpp/lib/ResultSetJson.cpp +++ b/cpp/lib/ResultSetJson.cpp @@ -26,7 +26,7 @@ ResultSetJson::ResultSetJson() : ResultSetJson::ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - std::string tzString + const std::string& tzString ) : ResultSet(metadata, tzString, SF_JSON_FORMAT) { @@ -42,8 +42,9 @@ ResultSetJson::~ResultSetJson() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetJson::appendChunk(cJSON * chunk) +SF_STATUS STDCALL ResultSetJson::appendChunk(void* chunkPtr) { + cJSON * chunk = (cJSON *)chunkPtr; if (chunk == nullptr) { CXX_LOG_ERROR("appendChunk -- Received a null chunk to append."); diff --git a/cpp/lib/ResultSetJson.hpp b/cpp/lib/ResultSetJson.hpp index 0c853e7ee2..5dc292dcbd 100644 --- a/cpp/lib/ResultSetJson.hpp +++ b/cpp/lib/ResultSetJson.hpp @@ -52,7 +52,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - std::string tzString); + const std::string& tzString); /** * Destructor. @@ -66,7 +66,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(cJSON * chunk); + SF_STATUS STDCALL appendChunk(void * chunkPtr); /** * Advances the internal iterator to the next row. If there are no more rows to consume, diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 8923fb77aa..1c5e83c9ee 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -18,17 +18,8 @@ extern "C" { const char * tz_string ) { - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, std::string(tz_string)); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson(json_rowset, metadata, std::string(tz_string)); - default: - return nullptr; - } + return Snowflake::Client::ResultSet::CreateResultFromJson( + json_rowset, metadata, query_result_format, std::string(tz_string)); } result_set_ptr rs_create_with_chunk( @@ -38,17 +29,8 @@ extern "C" { const char * tz_string ) { - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, std::string(tz_string)); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, std::string(tz_string)); - default: - return nullptr; - } + return Snowflake::Client::ResultSet::CreateResultFromChunk( + initial_chunk, metadata, query_result_format, std::string(tz_string)); } void rs_destroy(result_set_ptr rs) @@ -57,20 +39,7 @@ extern "C" { { return; } - QueryResultFormat query_result_format = - static_cast(rs)->getResultFormat(); - switch (query_result_format){ -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - delete static_cast(rs); - break; -#endif - case SF_JSON_FORMAT: - delete static_cast(rs); - break; - default: - break; - } + delete static_cast(rs); } #define ERROR_IF_NULL(ptr) \ @@ -82,21 +51,7 @@ extern "C" { rs_append_chunk(result_set_ptr rs, void * chunk) { ERROR_IF_NULL(rs); - QueryResultFormat query_result_format = - static_cast(rs)->getResultFormat(); - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return static_cast(rs)->appendChunk( - (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunk)->buffer)); -#endif - case SF_JSON_FORMAT: - return static_cast(rs)->appendChunk( - (cJSON*)chunk); - default: - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; - } + return static_cast(rs)->appendChunk(chunk); } SF_STATUS STDCALL rs_next(result_set_ptr rs) From e3e1b88d52f2df47a1ce4a417e2baf0ae64a7b79 Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Tue, 22 Oct 2024 09:31:10 -0700 Subject: [PATCH 29/74] SNOW-1524269: support put/get for GCP (#738) --- cpp/StatementPutGet.cpp | 153 ++++++++++++++++++++++++++++++++++++ cpp/StatementPutGet.hpp | 33 ++++++++ include/snowflake/client.h | 5 ++ include/snowflake/version.h | 6 +- lib/chunk_downloader.c | 4 +- lib/client.c | 3 + lib/client_int.h | 1 + lib/connection.c | 19 +++-- lib/connection.h | 32 ++++++-- lib/http_perform.c | 76 +++++++++++++++++- lib/mock_http_perform.h | 2 +- tests/test_simple_put.cpp | 32 ++++---- 12 files changed, 322 insertions(+), 44 deletions(-) diff --git a/cpp/StatementPutGet.cpp b/cpp/StatementPutGet.cpp index 989affbf89..883a129a98 100755 --- a/cpp/StatementPutGet.cpp +++ b/cpp/StatementPutGet.cpp @@ -3,11 +3,35 @@ */ #include +#include "connection.h" #include "snowflake/PutGetParseResponse.hpp" #include "StatementPutGet.hpp" +#include "curl_desc_pool.h" using namespace Snowflake::Client; +static size_t file_get_write_callback(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + size_t data_size = size * nmemb; + std::basic_iostream* recvStream = (std::basic_iostream*)(userdata); + if (recvStream) + { + recvStream->write(static_cast(ptr), data_size); + } + + return data_size; +} + +static size_t file_put_read_callback(void* ptr, size_t size, size_t nmemb, void* userdata) +{ + std::basic_iostream* payload = (std::basic_iostream*)(userdata); + size_t data_size = size * nmemb; + + payload->read(static_cast(ptr), data_size); + size_t ret = payload->gcount(); + return payload->gcount(); +} + StatementPutGet::StatementPutGet(SF_STMT *stmt) : m_stmt(stmt), m_useProxy(false) { @@ -104,6 +128,14 @@ bool StatementPutGet::parsePutGetCommand(std::string *sql, }; putGetParseResponse->stageInfo.endPoint = response->stage_info->endPoint; + } + else if (sf_strncasecmp(response->stage_info->location_type, "gcs", 3) == 0) + { + putGetParseResponse->stageInfo.stageType = StageType::GCS; + putGetParseResponse->stageInfo.credentials = { + {"GCS_ACCESS_TOKEN", response->stage_info->stage_cred->gcs_access_token} + }; + } else if (sf_strncasecmp(response->stage_info->location_type, "local_fs", 8) == 0) { @@ -123,3 +155,124 @@ Util::Proxy* StatementPutGet::get_proxy() return &m_proxy; } } + +bool StatementPutGet::http_put(std::string const& url, + std::vector const& headers, + std::basic_iostream& payload, + size_t payloadLen, + std::string& responseHeaders) +{ + if (!m_stmt || !m_stmt->connection) + { + return false; + } + SF_CONNECT* sf = m_stmt->connection; + void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); + CURL* curl = get_curl_from_desc(curl_desc); + if (!curl) + { + return false; + } + + char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); + sf_strcpy(urlbuf, url.length() + 1, url.c_str()); + + SF_HEADER reqHeaders; + reqHeaders.header = NULL; + for (auto itr = headers.begin(); itr != headers.end(); itr++) + { + reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); + } + + PUT_PAYLOAD putPayload; + putPayload.buffer = &payload; + putPayload.length = payloadLen; + putPayload.read_callback = file_put_read_callback; + + char* respHeaders = NULL; + sf_bool success = SF_BOOLEAN_FALSE; + + success = http_perform(curl, PUT_REQUEST_TYPE, urlbuf, &reqHeaders, NULL, &putPayload, NULL, + NULL, &respHeaders, get_retry_timeout(sf), + SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode,sf->ocsp_fail_open, + sf->retry_on_curle_couldnt_connect_count, + 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, + sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); + + free_curl_desc(curl_desc); + SF_FREE(urlbuf); + curl_slist_free_all(reqHeaders.header); + if (respHeaders) + { + responseHeaders = std::string(respHeaders); + SF_FREE(respHeaders); + } + + return success; +} + +bool StatementPutGet::http_get(std::string const& url, + std::vector const& headers, + std::basic_iostream* payload, + std::string& responseHeaders, + bool headerOnly) +{ + SF_REQUEST_TYPE reqType = GET_REQUEST_TYPE; + if (headerOnly) + { + reqType = HEAD_REQUEST_TYPE; + } + + if (!m_stmt || !m_stmt->connection) + { + return false; + } + SF_CONNECT* sf = m_stmt->connection; + + void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); + CURL* curl = get_curl_from_desc(curl_desc); + if (!curl) + { + return false; + } + + char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); + sf_strcpy(urlbuf, url.length() + 1, url.c_str()); + + SF_HEADER reqHeaders; + reqHeaders.header = NULL; + for (auto itr = headers.begin(); itr != headers.end(); itr++) + { + reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); + } + + NON_JSON_RESP resp; + resp.buffer = payload; + resp.write_callback = file_get_write_callback; + + char* respHeaders = NULL; + sf_bool success = SF_BOOLEAN_FALSE; + + success = http_perform(curl, reqType, urlbuf, &reqHeaders, NULL, NULL, NULL, + &resp, &respHeaders, get_retry_timeout(sf), + SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode, sf->ocsp_fail_open, + sf->retry_on_curle_couldnt_connect_count, + 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, + sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); + + free_curl_desc(curl_desc); + SF_FREE(urlbuf); + curl_slist_free_all(reqHeaders.header); + if (respHeaders) + { + responseHeaders = respHeaders; + SF_FREE(respHeaders); + } + + if (payload) + { + payload->flush(); + } + + return success; +} diff --git a/cpp/StatementPutGet.hpp b/cpp/StatementPutGet.hpp index 94321fac73..19c00df0fd 100644 --- a/cpp/StatementPutGet.hpp +++ b/cpp/StatementPutGet.hpp @@ -28,6 +28,39 @@ class StatementPutGet : public Snowflake::Client::IStatementPutGet virtual Util::Proxy* get_proxy(); + /** + * PUT/GET on GCS use this interface to perform put request. + * Not implemented by default. + * @param url The url of the request. + * @param headers The headers of the request. + * @param payload The upload data. + * @param responseHeaders The headers of the response. + * + * return true if succeed otherwise false + */ + virtual bool http_put(std::string const& url, + std::vector const& headers, + std::basic_iostream& payload, + size_t payloadLen, + std::string& responseHeaders); + + /** + * PUT/GET on GCS use this interface to perform put request. + * Not implemented by default. + * @param url The url of the request. + * @param headers The headers of the request. + * @param payload The upload data. + * @param responseHeaders The headers of the response. + * @param headerOnly True if get response header only without payload body. + * + * return true if succeed otherwise false + */ + virtual bool http_get(std::string const& url, + std::vector const& headers, + std::basic_iostream* payload, + std::string& responseHeaders, + bool headerOnly); + private: SF_STMT *m_stmt; Util::Proxy m_proxy; diff --git a/include/snowflake/client.h b/include/snowflake/client.h index e97ef7e373..756a20a892 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -18,6 +18,11 @@ extern "C" { /** * API Name */ +/* TODO: Temporarily change to ODBC for now to pass the test before + * features (PUT for GCP, multiple statements etc.) unblocked + * on server side. + * Need to revert to C_API when merging to master. + */ #define SF_API_NAME "ODBC" /** diff --git a/include/snowflake/version.h b/include/snowflake/version.h index 83ddc59f55..437ac8e5fb 100644 --- a/include/snowflake/version.h +++ b/include/snowflake/version.h @@ -5,7 +5,11 @@ #ifndef SNOWFLAKE_CLIENT_VERSION_H #define SNOWFLAKE_CLIENT_VERSION_H -// TODO: temporary change for testing, will remove +/* TODO: Temporarily change to ODBC version for now to pass the test before + * features (PUT for GCP, multiple statements etc.) unblocked + * on server side. + * Need to revert to libsfclient version when merging to master. + */ #define SF_API_VERSION "3.0.1" #endif /* SNOWFLAKE_CLIENT_VERSION_H */ diff --git a/lib/chunk_downloader.c b/lib/chunk_downloader.c index 6f6e923cb2..5657cb4465 100644 --- a/lib/chunk_downloader.c +++ b/lib/chunk_downloader.c @@ -217,8 +217,8 @@ sf_bool STDCALL download_chunk(char *url, SF_HEADER *headers, CURL *curl = get_curl_from_desc(curl_desc); if (!curl || - !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, chunk, - non_json_resp, network_timeout, + !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, NULL, chunk, + non_json_resp, NULL, network_timeout, SF_BOOLEAN_TRUE, error, insecure_mode, fail_open, 0, 0, retry_max_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, proxy, no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE)) { diff --git a/lib/client.c b/lib/client.c index 83607e06db..9880113bab 100644 --- a/lib/client.c +++ b/lib/client.c @@ -2488,6 +2488,9 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, json_copy_string( &sfstmt->put_get_response->stage_info->stage_cred->azure_sas_token, stage_cred, "AZURE_SAS_TOKEN"); + json_copy_string( + &sfstmt->put_get_response->stage_info->stage_cred->gcs_access_token, + stage_cred, "GCS_ACCESS_TOKEN"); json_copy_string( &sfstmt->put_get_response->localLocation, data, "localLocation"); diff --git a/lib/client_int.h b/lib/client_int.h index e8b111cd68..632856f031 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -86,6 +86,7 @@ typedef struct SF_STAGE_CRED { char *aws_secret_key; char *aws_token; char *azure_sas_token; + char* gcs_access_token; } SF_STAGE_CRED; typedef struct SF_STAGE_INFO { diff --git a/lib/connection.c b/lib/connection.c index 4ca5a12ca4..cf1916cbdf 100644 --- a/lib/connection.c +++ b/lib/connection.c @@ -239,7 +239,6 @@ cJSON *STDCALL create_query_json_body(const char *sql_text, parameters = snowflake_cJSON_CreateObject(); } snowflake_cJSON_AddStringToObject(parameters, "C_API_QUERY_RESULT_FORMAT", "JSON"); - // temporary code to fake as ODBC to have multiple statements enabled snowflake_cJSON_AddStringToObject(parameters, "ODBC_QUERY_RESULT_FORMAT", "JSON"); #endif @@ -376,7 +375,7 @@ sf_bool STDCALL curl_post_call(SF_CONNECT *sf, } do { - if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, json, NULL, + if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, NULL, json, NULL, NULL, retry_timeout, SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -503,7 +502,7 @@ sf_bool STDCALL curl_get_call(SF_CONNECT *sf, memset(query_code, 0, QUERYCODE_LEN); do { - if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, json, NULL, + if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, NULL, json, NULL, NULL, get_retry_timeout(sf), SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -906,16 +905,16 @@ ARRAY_LIST *json_get_object_keys(const cJSON *item) { } size_t -json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json) { +char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf) { size_t data_size = size * nmemb; log_debug("Curl response size: %zu", data_size); - raw_json->buffer = (char *) SF_REALLOC(raw_json->buffer, - raw_json->size + data_size + 1); + raw_buf->buffer = (char *) SF_REALLOC(raw_buf->buffer, + raw_buf->size + data_size + 1); // Start copying where last null terminator existed - sf_memcpy(&raw_json->buffer[raw_json->size], data_size, data, data_size); - raw_json->size += data_size; - // Set null terminator - raw_json->buffer[raw_json->size] = '\0'; + sf_memcpy(&raw_buf->buffer[raw_buf->size], data_size, data, data_size); + raw_buf->size += data_size; + // Set null raw_buf + raw_buf->buffer[raw_buf->size] = '\0'; return data_size; } diff --git a/lib/connection.h b/lib/connection.h index c536aa9b45..04ef521ee8 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -56,6 +56,9 @@ typedef enum SF_REQUEST_TYPE { /** we are doing a http delete */ DELETE_REQUEST_TYPE, + + /** we are doing a http head */ + HEAD_REQUEST_TYPE, } SF_REQUEST_TYPE; /** @@ -81,12 +84,12 @@ typedef enum SF_JSON_ERROR { /** * Dynamically growing char buffer to hold retrieved in cURL call. */ -typedef struct RAW_JSON_BUFFER { +typedef struct RAW_CHAR_BUFFER { // Char buffer char *buffer; // Number of characters in char buffer size_t size; -} RAW_JSON_BUFFER; +} RAW_CHAR_BUFFER; /** * URL Parameter struct used to construct an encoded URL. @@ -167,6 +170,15 @@ typedef struct non_json_response { void * buffer; } NON_JSON_RESP; +/** +* payload struct for put request +*/ +typedef struct put_payload { + size_t (*read_callback)(void* ptr, size_t size, size_t nmemb, void* userdata); + void * buffer; + size_t length; +} PUT_PAYLOAD; + /** * Macro to get a custom error message to pass to the Snowflake Error object. */ @@ -398,16 +410,16 @@ SF_JSON_ERROR STDCALL json_detach_object_from_array(cJSON **dest, cJSON *data, i ARRAY_LIST *json_get_object_keys(const cJSON *item); /** - * A write callback function to use to write the response text received from the cURL response. The raw JSON buffer + * A write callback function to use to write the response text received from the cURL response. The raw CHAR buffer * will grow in size until * * @param data The data to copy in the buffer. * @param size The size (in bytes) of each data member. * @param nmemb The number of data members. - * @param raw_json The Raw JSON Buffer object that grows in size to copy multiple writes for a single cURL call. + * @param raw_buf The Raw CHAR Buffer object that grows in size to copy multiple writes for a single cURL call. * @return The number of bytes copied into the buffer. */ -size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json); +size_t char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf); /** * Performs an HTTP request with retry. @@ -416,10 +428,13 @@ size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_ * @param request_type The type of HTTP request. * @param url The fully qualified URL to use for the HTTP request. * @param header The header to use for the HTTP request. - * @param body The body to send over the HTTP request. If running GET request, set this to NULL. + * @param body The body to send over the HTTP request. If running GET/PUT request, set this to NULL. + * @param put_payload The payload to send over the PUT HTTP request. If not running PUT request, set this to NULL. * @param json A reference to a cJSON pointer where we should store a successful request. * @param non_json_resp A reference to a non-json response to retrieve response in non-json format. * Used only when json is set to NULL. + * @param resp_headers A reference to retrieve response headers. Needs to be freed with SF_FREE. + * Set to NULL if it's not needed. * @param network_timeout The network request timeout to use for each request try. * @param chunk_downloader A boolean value determining whether or not we are running this request from the chunk * downloader. Each chunk that we download from AWS is invalid JSON so we need to add an @@ -450,8 +465,9 @@ size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_ * @return Success/failure status of http request call. 1 = Success; 0 = Failure/renew timeout */ sf_bool STDCALL http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, cJSON **json, NON_JSON_RESP* non_json_resp, int64 network_timeout, sf_bool chunk_downloader, - SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open, + char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP* non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, + SF_ERROR_STRUCT* error, sf_bool insecure_mode, sf_bool fail_open, int8 retry_on_curle_couldnt_connect_count, int64 renew_timeout, int8 retry_max_count, int64 *elapsed_time, int8 *retried_count, diff --git a/lib/http_perform.c b/lib/http_perform.c index b7291e2034..56d093967d 100644 --- a/lib/http_perform.c +++ b/lib/http_perform.c @@ -143,8 +143,10 @@ sf_bool STDCALL http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, + PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, @@ -190,7 +192,8 @@ sf_bool STDCALL http_perform(CURL *curl, sf_get_current_time_millis() // start time }; time_t elapsedRetryTime = time(NULL); - RAW_JSON_BUFFER buffer = {NULL, 0}; + RAW_CHAR_BUFFER buffer = {NULL, 0}; + RAW_CHAR_BUFFER headerBuffer = { NULL, 0 }; struct data config; config.trace_ascii = 1; @@ -204,6 +207,8 @@ sf_bool STDCALL http_perform(CURL *curl, // Reset buffer since this may not be our first rodeo SF_FREE(buffer.buffer); buffer.size = 0; + SF_FREE(headerBuffer.buffer); + headerBuffer.size = 0; // Generate new request guid, if request guid exists in url if (SF_BOOLEAN_TRUE != retry_ctx_update_url(&curl_retry_ctx, url, include_retry_reason)) { @@ -269,6 +274,46 @@ sf_bool STDCALL http_perform(CURL *curl, break; } } + else if (request_type == HEAD_REQUEST_TYPE) + { + /** we want response header only */ + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + } + else if (request_type == PUT_REQUEST_TYPE) + { + // we need to upload the data + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + if (!put_payload) + { + log_error("Invalid payload for put request"); + break; + } + /** set read callback function */ + res = curl_easy_setopt(curl, CURLOPT_READFUNCTION, put_payload->read_callback); + if (res != CURLE_OK) { + log_error("Failed to set read function [%s]", curl_easy_strerror(res)); + break; + } + + /** set data object to pass to callback function */ + res = curl_easy_setopt(curl, CURLOPT_READDATA, put_payload->buffer); + if (res != CURLE_OK) { + log_error("Failed to set read data [%s]", curl_easy_strerror(res)); + break; + } + + /** set size of put */ + if (put_payload->length <= SF_INT32_MAX) + { + res = curl_easy_setopt(curl, CURLOPT_INFILESIZE, (long)put_payload->length); + } + else + { + res = curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)put_payload->length); + } + } + if (!json && non_json_resp) { @@ -276,7 +321,7 @@ sf_bool STDCALL http_perform(CURL *curl, } else { - res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&json_resp_cb); + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&char_resp_cb); } if (res != CURLE_OK) { log_error("Failed to set writer [%s]", curl_easy_strerror(res)); @@ -292,8 +337,20 @@ sf_bool STDCALL http_perform(CURL *curl, res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buffer); } if (res != CURLE_OK) { - log_error("Failed to set write data [%s]", curl_easy_strerror(res)); - break; + log_error("Failed to set write data [%s]", curl_easy_strerror(res)); + break; + } + + res = curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)&headerBuffer); + if (res != CURLE_OK) { + log_error("Failed to set header data [%s]", curl_easy_strerror(res)); + break; + } + + res = curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (void*)&char_resp_cb); + if (res != CURLE_OK) { + log_error("Failed to set header function [%s]", curl_easy_strerror(res)); + break; } if (DISABLE_VERIFY_PEER) { @@ -501,6 +558,15 @@ sf_bool STDCALL http_perform(CURL *curl, SF_FREE(buffer.buffer); + if (resp_headers) + { + *resp_headers = headerBuffer.buffer; + } + else + { + SF_FREE(headerBuffer.buffer); + } + return ret; } @@ -511,8 +577,10 @@ sf_bool STDCALL __wrap_http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, + PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, + char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, diff --git a/lib/mock_http_perform.h b/lib/mock_http_perform.h index ce6cad20c3..71ea0c8b79 100644 --- a/lib/mock_http_perform.h +++ b/lib/mock_http_perform.h @@ -17,7 +17,7 @@ extern "C" { // The parameters for this are identical to http_perform located in connection.h // This is just the mock interface sf_bool STDCALL __wrap_http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, cJSON **json, NON_JSON_RESP *non_json_resp, int64 network_timeout, sf_bool chunk_downloader, + char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open); #endif diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index e592371e46..c16a3f98c2 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -856,6 +856,19 @@ static int gr_setup(void **unused) { initialize_test(SF_BOOLEAN_FALSE); + // TODO SNOW-1526335 + // Sometime we can't get OCSP response from cache server or responder + // Usually happen on GCP and should be ignored by FAIL_OPEN + // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now + // so we have to disable OCSP validation to around it. + // Will remove this code when adding support for FAIL_OPEN (which is + // the default behavior for all other drivers) + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "GCP", 4)) { + sf_bool value = SF_BOOLEAN_FALSE; + snowflake_global_set_attribute(SF_GLOBAL_OCSP_CHECK, &value); + } + if(!setup_random_database()) { std::cout << "Failed to setup random database, fallback to use regular one." << std::endl; } @@ -1231,12 +1244,6 @@ void test_2GBlarge_put(void **unused) return; } } - // put/get for GCP is not supported in libsnowflakeclient - // will test that in odbc. - if (cenv && !strncmp(cenv, "GCP", 4)) { - errno = 0; - return; - } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1270,12 +1277,6 @@ void test_2GBlarge_get(void **unused) return; } } - // put/get for GCP is not supported in libsnowflakeclient - // will test that in odbc. - if (cenv && !strncmp(cenv, "GCP", 4)) { - errno = 0; - return; - } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1637,7 +1638,7 @@ int main(void) { }); if(testAccount.find("GCP") != std::string::npos) { - setenv("CLOUD_PROVIDER", "GCP", 1); + setenv("CLOUD_PROVIDER", "GCP", 1); } else if(testAccount.find("AZURE") != std::string::npos) { @@ -1651,11 +1652,6 @@ int main(void) { char *cp = getenv("CLOUD_PROVIDER"); std::cout << "Cloud provider is " << cp << std::endl; #endif - const char *cloud_provider = std::getenv("CLOUD_PROVIDER"); - if(cloud_provider && ( strcmp(cloud_provider, "GCP") == 0 ) ) { - std::cout << "GCP put/get feature is not available in libsnowflakeclient." << std::endl; - return 0; - } const struct CMUnitTest tests[] = { cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), From 4b2cc1642b6758df053e0660f0c7fee54fd9c6dc Mon Sep 17 00:00:00 2001 From: Harry Xi Date: Wed, 23 Oct 2024 11:12:30 -0700 Subject: [PATCH 30/74] SNOW-1524269 native put get support (#745) --- CMakeLists.txt | 2 + ci/build_linux.sh | 4 +- cpp/FileTransferAgent.cpp | 76 ++++- cpp/StatementPutGet.cpp | 2 +- cpp/lib/ResultSet.hpp | 8 + cpp/lib/ResultSetArrow.hpp | 15 - cpp/lib/ResultSetPutGet.cpp | 336 +++++++++++++++++++ cpp/lib/ResultSetPutGet.hpp | 205 ++++++++++++ cpp/lib/result_set.cpp | 1 + include/snowflake/client.h | 24 +- lib/client.c | 122 ++++++- lib/client_int.h | 42 ++- tests/CMakeLists.txt | 8 +- tests/test_simple_put.cpp | 635 +++++++++++++++++++++++++++--------- 14 files changed, 1304 insertions(+), 176 deletions(-) create mode 100644 cpp/lib/ResultSetPutGet.cpp create mode 100644 cpp/lib/ResultSetPutGet.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b71591988f..82ccd84aa5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,8 @@ set (SOURCE_FILES_PUT_GET cpp/logger/SFLogger.hpp cpp/logger/SecretDetector.cpp cpp/logger/SecretDetector.hpp + cpp/lib/ResultSetPutGet.cpp + cpp/lib/ResultSetPutGet.hpp include/snowflake/IFileTransferAgent.hpp include/snowflake/ISFLogger.hpp include/snowflake/IStatementPutGet.hpp diff --git a/ci/build_linux.sh b/ci/build_linux.sh index d42ec7c84b..6e58b1a987 100755 --- a/ci/build_linux.sh +++ b/ci/build_linux.sh @@ -51,5 +51,7 @@ docker run \ #remove image to save disk space on github if [[ -n "$GITHUB_ACTIONS" ]]; then docker rm -vf $(docker ps -aq --filter ancestor=${BUILD_IMAGE_NAME}) - docker rmi -f "${BUILD_IMAGE_NAME}" + if [[ $CLIENT_CODE_COVERAGE -ne 1 ]] && [[ "$BUILD_TYPE" != "Debug" ]]; then + docker rmi -f "${BUILD_IMAGE_NAME}" + fi fi diff --git a/cpp/FileTransferAgent.cpp b/cpp/FileTransferAgent.cpp index 80066899a1..cc0fbaee90 100755 --- a/cpp/FileTransferAgent.cpp +++ b/cpp/FileTransferAgent.cpp @@ -9,8 +9,9 @@ #include "FileTransferAgent.hpp" #include "snowflake/SnowflakeTransferException.hpp" #include "snowflake/IStatementPutGet.hpp" +#include "StatementPutGet.hpp" +#include "lib/ResultSetPutGet.hpp" #include "util/Base64.hpp" -#include "SnowflakeS3Client.hpp" #include "StorageClientFactory.hpp" #include "crypto/CipherStreamBuf.hpp" #include "crypto/Cryptor.hpp" @@ -18,6 +19,7 @@ #include "util/ThreadPool.hpp" #include "EncryptionProvider.hpp" #include "logger/SFLogger.hpp" +#include "error.h" #include "snowflake/platform.h" #include "snowflake/SF_CRTFunctionSafe.h" #include @@ -961,3 +963,75 @@ std::string Snowflake::Client::FileTransferAgent::getLocalFilePathFromCommand( return localFilePath; } + +using namespace Snowflake::Client; +extern "C" { + SF_STATUS STDCALL _snowflake_execute_put_get_native( + SF_STMT* sfstmt, + struct SF_QUERY_RESULT_CAPTURE* result_capture) + { + if (!sfstmt) + { + return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; + } + SF_CONNECT* sfconn = sfstmt->connection; + if (!sfconn) + { + return SF_STATUS_ERROR_CONNECTION_NOT_EXIST; + } + StatementPutGet stmtPutGet(sfstmt); + TransferConfig transConfig; + transConfig.caBundleFile = NULL; // use the one from global settings + transConfig.compressLevel = sfconn->put_compress_level; + transConfig.getSizeThreshold = sfconn->get_threshold; + transConfig.proxy = NULL; // use the one from statement + transConfig.tempDir = sfconn->put_temp_dir; + transConfig.useS3regionalUrl = sfconn->use_s3_regional_url; + string command(sfstmt->sql_text); + + FileTransferAgent agent(&stmtPutGet, &transConfig); + agent.setPutFastFail(sfconn->put_fastfail); + agent.setPutMaxRetries(sfconn->put_maxretries); + agent.setGetFastFail(sfconn->get_fastfail); + agent.setGetMaxRetries(sfconn->get_maxretries); + agent.setRandomDeviceAsUrand(sfconn->put_use_urand_dev); + + ITransferResult* result; + try + { + result = agent.execute(&command); + } + catch (std::exception& e) + { + std::string errmsg("File transfer failed: "); + errmsg += e.what(); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, + errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_FILE_TRANSFER; + } + catch (...) + { + std::string errmsg("File transfer failed with unknown exception."); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, + errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_FILE_TRANSFER; + } + + ResultSetPutGet * resultset = new Snowflake::Client::ResultSetPutGet(result); + if (!resultset) + { + std::string errmsg("Failed to allocate put get result set."); + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_OUT_OF_MEMORY, + errmsg.c_str(), SF_SQLSTATE_MEMORY_ALLOCATION_ERROR); + return SF_STATUS_ERROR_OUT_OF_MEMORY; + } + + sfstmt->qrf = SF_PUTGET_FORMAT; + sfstmt->total_row_index = 0; + sfstmt->result_set = resultset; + sfstmt->chunk_rowcount = sfstmt->total_rowcount = result->getResultSize(); + sfstmt->total_fieldcount = resultset->setup_column_desc(&sfstmt->desc); + + return SF_STATUS_SUCCESS; + } +} diff --git a/cpp/StatementPutGet.cpp b/cpp/StatementPutGet.cpp index 883a129a98..e5077854a9 100755 --- a/cpp/StatementPutGet.cpp +++ b/cpp/StatementPutGet.cpp @@ -49,7 +49,7 @@ StatementPutGet::StatementPutGet(SF_STMT *stmt) : bool StatementPutGet::parsePutGetCommand(std::string *sql, PutGetParseResponse *putGetParseResponse) { - if (snowflake_query(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) + if (_snowflake_query_put_get_legacy(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) { return false; } diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index bd5ee8d0e4..5e583c7848 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -14,6 +14,14 @@ #include "snowflake/client.h" #include "result_set.h" +#define VERIFY_COLUMN_INDEX(index, total) \ + if (index < 1 || index > total) \ + { \ + setError(SF_STATUS_ERROR_OUT_OF_BOUNDS, \ + "Column index must be between 1 and snowflake_num_fields()"); \ + return SF_STATUS_ERROR_OUT_OF_BOUNDS; \ + } + namespace Snowflake { namespace Client diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 3ad409dd12..68aa43456f 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -64,7 +64,6 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, const std::string& tzString); - /** * Parameterized constructor. * @@ -78,20 +77,6 @@ class ResultSetArrow : public Snowflake::Client::ResultSet */ ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, const std::string& tzString); - - /** - * Parameterized constructor. - * - * This constructor will initialize m_records with the (partial) results - * contained in the initial chunk. It will also initialize m_metadata with - * the metadata in "metadata". - * - * @param jsonRowset64 A pointer to the rowset64 data in json result set. - * @param metadata An array of metadata objects for each column. - * @param tzString The time zone. - */ - ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, std::string& tzString); - /** * Destructor. */ diff --git a/cpp/lib/ResultSetPutGet.cpp b/cpp/lib/ResultSetPutGet.cpp new file mode 100644 index 0000000000..173ea18a70 --- /dev/null +++ b/cpp/lib/ResultSetPutGet.cpp @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +#include "ResultSetPutGet.hpp" +#include "../logger/SFLogger.hpp" +#include "memory.h" +#include "client_int.h" + +// helper functions +namespace +{ + void setup_string_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) + { + col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); + sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); + col_desc.byte_size = SF_DEFAULT_MAX_OBJECT_SIZE; + col_desc.c_type = SF_C_TYPE_STRING; + col_desc.internal_size = SF_DEFAULT_MAX_OBJECT_SIZE; + col_desc.null_ok = SF_BOOLEAN_TRUE; + col_desc.precision = 0; + col_desc.scale = 0; + col_desc.type = SF_DB_TYPE_TEXT; + col_desc.idx = idx; + } + + void setup_integer_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) + { + col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); + sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); + col_desc.byte_size = 8; + col_desc.c_type = SF_C_TYPE_UINT64; + col_desc.internal_size = 8; + col_desc.null_ok = SF_BOOLEAN_TRUE; + col_desc.precision = 38; + col_desc.scale = 0; + col_desc.type = SF_DB_TYPE_FIXED; + col_desc.idx = idx; + } + + struct putget_column + { + std::string name; + bool isInteger; + }; + + std::vector PUTGET_COLUMNS[2] = + { + // UPLOAD + { + {"source", false}, + {"target", false}, + {"source_size", true}, + {"target_size", true}, + {"source_compression", false}, + {"target_compression", false}, + {"status", false}, + {"encryption", false}, + {"message", false}, + }, + // DOWNLOAD + { + {"file", false}, + {"size", true}, + {"status", false}, + {"encryption", false}, + {"message", false}, + }, + }; +} + +namespace Snowflake +{ +namespace Client +{ +ResultSetPutGet::ResultSetPutGet(ITransferResult *result) : + ResultSet(SF_PUTGET_FORMAT), + m_cmdType(result->getCommandType()) +{ + m_values.reserve(result->getResultSize()); + while (result->next()) + { + std::vector row; + row.reserve(result->getColumnSize()); + for (unsigned int i = 0; i < result->getColumnSize(); i++) + { + row.emplace_back(); + result->getColumnAsString(i, row.back()); + } + m_values.push_back(row); + } +} + +ResultSetPutGet::~ResultSetPutGet() +{ + ; // Do nothing +} + +// Public methods ================================================================================== + +size_t ResultSetPutGet::setup_column_desc(SF_COLUMN_DESC** desc) +{ + if ((m_cmdType != CommandType::UPLOAD) && (m_cmdType != CommandType::DOWNLOAD)) + { + // impossible + return 0; + } + + SF_COLUMN_DESC * col_desc = NULL; + m_totalColumnCount = PUTGET_COLUMNS[m_cmdType].size(); + col_desc = (SF_COLUMN_DESC*)SF_CALLOC(m_totalColumnCount, sizeof(SF_COLUMN_DESC)); + for (size_t i = 0; i < m_totalColumnCount; i++) + { + if (PUTGET_COLUMNS[m_cmdType][i].isInteger) + { + setup_integer_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); + } + else + { + setup_string_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); + } + } + + *desc = col_desc; + return m_totalColumnCount; +} + +SF_STATUS STDCALL ResultSetPutGet::next() +{ + if (m_currRowIdx < m_values.size()) + { + m_currRowIdx++; + return SF_STATUS_SUCCESS; + } + + return SF_STATUS_ERROR_OUT_OF_BOUNDS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsBool(size_t idx, sf_bool * out_data) +{ + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Value cannot be converted to boolean."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt8(size_t idx, int8 * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + *out_data = static_cast(m_values[row_idx][idx - 1][0]); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt32(size_t idx, int32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_INT32_MAX) + { + CXX_LOG_ERROR("Value out of range for int32."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for int32."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsInt64(size_t idx, int64 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_INT64_MAX) + { + CXX_LOG_ERROR("Value out of range for int64."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for int64."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint8(size_t idx, uint8 * out_data) +{ + return getCellAsInt8(idx, (int8*)out_data); +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint32(size_t idx, uint32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + if (value > SF_UINT32_MAX) + { + CXX_LOG_ERROR("Value out of range for uint32."); + setError(SF_STATUS_ERROR_OUT_OF_RANGE, + "Value out of range for uint32."); + return SF_STATUS_ERROR_OUT_OF_RANGE; + } + + *out_data = static_cast(value); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsUint64(size_t idx, uint64 * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + *out_data = 0; + + if (!PUTGET_COLUMNS[m_cmdType][idx - 1].isInteger) + { + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Invalid conversion from string column to integer."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; + } + + uint64 value = 0; + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + try + { + value = std::stoull(m_values[row_idx][idx - 1], NULL, 10); + } + catch (...) + { + CXX_LOG_ERROR("Cannot convert value to uint64."); + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Cannot convert value to uint64."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; + } + + *out_data = value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat32(size_t idx, float32 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + *out_data = (float32)value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat64(size_t idx, float64 * out_data) +{ + *out_data = 0; + + uint64 value = 0; + SF_STATUS ret = getCellAsUint64(idx, &value); + + if (SF_STATUS_SUCCESS != ret) + { + return ret; + } + + *out_data = (float64)value; + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsConstString(size_t idx, const char ** out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + *out_data = m_values[row_idx][idx - 1].c_str(); + return SF_STATUS_SUCCESS; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data) +{ + setError(SF_STATUS_ERROR_CONVERSION_FAILURE, + "Value cannot be converted to timestamp."); + return SF_STATUS_ERROR_CONVERSION_FAILURE; +} + +SF_STATUS STDCALL ResultSetPutGet::getCellStrlen(size_t idx, size_t * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + + size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; + + *out_data = m_values[row_idx][idx - 1].length(); + return SF_STATUS_SUCCESS; +} + +size_t ResultSetPutGet::getRowCountInChunk() +{ + return m_values.size(); +} + +SF_STATUS STDCALL ResultSetPutGet::isCellNull(size_t idx, sf_bool * out_data) +{ + VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); + *out_data = SF_BOOLEAN_FALSE; + + return SF_STATUS_SUCCESS; +} + +} // namespace Client +} // namespace Snowflake diff --git a/cpp/lib/ResultSetPutGet.hpp b/cpp/lib/ResultSetPutGet.hpp new file mode 100644 index 0000000000..7e8e8c8681 --- /dev/null +++ b/cpp/lib/ResultSetPutGet.hpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. + */ + +#ifndef SNOWFLAKECLIENT_RESULTSETPUTGET_HPP +#define SNOWFLAKECLIENT_RESULTSETPUTGET_HPP + +#include "ResultSet.hpp" +#include "snowflake/ITransferResult.hpp" + +namespace Snowflake +{ +namespace Client +{ + +/** + * Represents a result set retrieved from PUT/GET execution. + */ +class ResultSetPutGet : public Snowflake::Client::ResultSet +{ +public: + + /** + * Parameterized constructor. + * + * @param result A pointer to the transfer result. + */ + ResultSetPutGet(ITransferResult *result); + + /** + * Destructor. + */ + ~ResultSetPutGet(); + + /** + * Setup column description of the transfer result. + * + * param desc The output parameter to return the pointer to the buffer allocated + * for the column description. + * Needs to be freed by the caller using SF_FREE. + * + * @return The number of columns. + */ + size_t setup_column_desc(SF_COLUMN_DESC** desc); + + /** + * Advances the internal iterator to the next row. If there are no more rows to consume, + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL next(); + + /** + * Writes the value of the given cell as a boolean to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsBool(size_t idx, sf_bool * out_data); + + /** + * Writes the value of the given cell as an int8 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt8(size_t idx, int8 * out_data); + + /** + * Writes the value of the given cell as an int32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt32(size_t idx, int32 * out_data); + + /** + * Writes the value of the given cell as an int64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsInt64(size_t idx, int64 * out_data); + + /** + * Writes the value of the given cell as a uint8 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint8(size_t idx, uint8 * out_data); + + /** + * Writes the value of the given cell as a uint32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint32(size_t idx, uint32 * out_data); + + /** + * Writes the value of the given cell as a uint64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsUint64(size_t idx, uint64 * out_data); + + /** + * Writes the value of the given cell as a float32 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsFloat32(size_t idx, float32 * out_data); + + /** + * Writes the value of the given cell as a float64 to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsFloat64(size_t idx, float64 * out_data); + + /** + * Writes the value of the given cell as a constant C-string to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsConstString(size_t idx, const char ** out_data); + + /** + * Writes the value of the given cell as a timestamp to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data); + + /** + * Writes the length of the given cell to the provided buffer. + * + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL getCellStrlen(size_t idx, size_t * out_data); + + /** + * Gets the total number of rows in the current chunk being processed. + * + * @return the number of rows in the current chunk. + */ + size_t getRowCountInChunk(); + + /** + * Indicates whether the given cell is null. + * + * @param idx The index of the column to check is null. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data); + +private: + + // Hidden default constructor. + ResultSetPutGet(); + + // the result values from transfer result + std::vector> m_values; + + // the command type of UPLOAD(PUT) or DOWNLOAD(GET) + CommandType m_cmdType; +}; + +} // namespace Client +} // namespace Snowflake + +#endif // SNOWFLAKECLIENT_RESULTSETPUTGET_HPP diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 1c5e83c9ee..3fca6fe496 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -6,6 +6,7 @@ #include "ResultSet.hpp" #include "ResultSetArrow.hpp" #include "ResultSetJson.hpp" +#include "ResultSetPutGet.hpp" #ifdef __cplusplus extern "C" { diff --git a/include/snowflake/client.h b/include/snowflake/client.h index 756a20a892..0a08109f92 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -164,7 +164,8 @@ typedef enum SF_STATUS { SF_STATUS_ERROR_NULL_POINTER = 240022, SF_STATUS_ERROR_BUFFER_TOO_SMALL = 240023, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT = 240024, - SF_STATUS_ERROR_OTHER = 240025 + SF_STATUS_ERROR_OTHER = 240025, + SF_STATUS_ERROR_FILE_TRANSFER = 240026 } SF_STATUS; /** @@ -268,6 +269,14 @@ typedef enum SF_ATTRIBUTE { SF_CON_MAX_BINARY_SIZE, SF_CON_MAX_VARIANT_SIZE, SF_CON_OCSP_FAIL_OPEN, + SF_CON_PUT_TEMPDIR, + SF_CON_PUT_COMPRESSLV, + SF_CON_PUT_USE_URANDOM_DEV, + SF_CON_PUT_FASTFAIL, + SF_CON_PUT_MAXRETRIES, + SF_CON_GET_FASTFAIL, + SF_CON_GET_MAXRETRIES, + SF_CON_GET_THRESHOLD, SF_DIR_QUERY_URL, SF_DIR_QUERY_URL_PARAM, SF_DIR_QUERY_TOKEN, @@ -403,6 +412,17 @@ typedef struct SF_CONNECT { uint64 max_varchar_size; uint64 max_binary_size; uint64 max_variant_size; + + // put get configurations + sf_bool use_s3_regional_url; + sf_bool put_use_urand_dev; + int8 put_compress_level; + char* put_temp_dir; + sf_bool put_fastfail; + int8 put_maxretries; + sf_bool get_fastfail; + int8 get_maxretries; + int64 get_threshold; } SF_CONNECT; /** @@ -459,7 +479,7 @@ typedef void* result_set_ptr; */ typedef enum QueryResultFormat_e { - SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_FORMAT_UNKNOWN + SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_PUTGET_FORMAT, SF_FORMAT_UNKNOWN } QueryResultFormat; /** diff --git a/lib/client.c b/lib/client.c index 9880113bab..729e2f4328 100644 --- a/lib/client.c +++ b/lib/client.c @@ -183,6 +183,9 @@ static SF_STATUS STDCALL _reset_connection_parameters( else if (strcmp(name->valuestring, "VARIANT_MAX_SIZE_IN_RESULT") == 0) { sf->max_variant_size = snowflake_cJSON_GetUint64Value(value); } + else if (strcmp(name->valuestring, "ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1") == 0) { + sf->use_s3_regional_url = snowflake_cJSON_IsTrue(value) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; + } } } SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -534,6 +537,15 @@ _snowflake_check_connection_parameters(SF_CONNECT *sf) { log_debug("retry_count: %d", sf->retry_count); log_debug("qcc_disable: %s", sf->qcc_disable ? "true" : "false"); log_debug("include_retry_reason: %s", sf->include_retry_reason ? "true" : "false"); + log_debug("use_s3_regional_url: %s", sf->use_s3_regional_url ? "true" : "false"); + log_debug("put_use_urand_dev: %s", sf->put_use_urand_dev ? "true" : "false"); + log_debug("put_compress_level: %d", sf->put_compress_level); + log_debug("put_temp_dir: %s", sf->put_temp_dir ? sf->put_temp_dir : ""); + log_debug("put_fastfail: %s", sf->put_fastfail ? "true" : "false"); + log_debug("put_maxretries: %d", sf->put_maxretries); + log_debug("get_fastfail: %s", sf->get_fastfail ? "true" : "false"); + log_debug("get_maxretries: %d", sf->get_maxretries); + log_debug("get_threshold: %d", sf->get_threshold); return SF_STATUS_SUCCESS; } @@ -726,6 +738,16 @@ SF_CONNECT *STDCALL snowflake_init() { sf->max_varchar_size = SF_DEFAULT_MAX_OBJECT_SIZE; sf->max_binary_size = SF_DEFAULT_MAX_OBJECT_SIZE / 2; sf->max_variant_size = SF_DEFAULT_MAX_OBJECT_SIZE; + + sf->use_s3_regional_url = SF_BOOLEAN_FALSE; + sf->put_use_urand_dev = SF_BOOLEAN_FALSE; + sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; + sf->put_temp_dir = NULL; + sf->put_fastfail = SF_BOOLEAN_FALSE; + sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; + sf->get_fastfail = SF_BOOLEAN_FALSE; + sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; + sf->get_threshold = SF_DEFAULT_GET_THRESHOLD; } return sf; @@ -1146,6 +1168,45 @@ SF_STATUS STDCALL snowflake_set_attribute( case SF_CON_INCLUDE_RETRY_REASON: sf->include_retry_reason = value ? *((sf_bool *)value) : SF_BOOLEAN_TRUE; break; + case SF_CON_PUT_TEMPDIR: + alloc_buffer_and_copy(&sf->put_temp_dir, value); + break; + case SF_CON_PUT_COMPRESSLV: + sf->put_compress_level = value ? *((int8 *)value) : SF_DEFAULT_PUT_COMPRESS_LEVEL; + if ((sf->put_compress_level > SF_MAX_PUT_COMPRESS_LEVEL) || + (sf->put_compress_level < 0)) + { + sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; + } + break; + case SF_CON_PUT_USE_URANDOM_DEV: + sf->put_use_urand_dev = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_PUT_FASTFAIL: + sf->put_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_PUT_MAXRETRIES: + sf->put_maxretries = value ? *((int8 *)value) : SF_DEFAULT_PUT_MAX_RETRIES; + if ((sf->put_maxretries > SF_MAX_PUT_MAX_RETRIES) || + (sf->put_maxretries < 0)) + { + sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; + } + break; + case SF_CON_GET_FASTFAIL: + sf->get_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; + break; + case SF_CON_GET_MAXRETRIES: + sf->get_maxretries = value ? *((int8 *)value) : SF_DEFAULT_GET_MAX_RETRIES; + if ((sf->get_maxretries > SF_MAX_GET_MAX_RETRIES) || + (sf->get_maxretries < 0)) + { + sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; + } + break; + case SF_CON_GET_THRESHOLD: + sf->get_threshold = value ? *((int64 *)value) : SF_DEFAULT_GET_THRESHOLD; + break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -1292,6 +1353,30 @@ SF_STATUS STDCALL snowflake_get_attribute( case SF_CON_MAX_VARIANT_SIZE: *value = &sf->max_variant_size; break; + case SF_CON_PUT_TEMPDIR: + *value = sf->put_temp_dir; + break; + case SF_CON_PUT_COMPRESSLV: + *value = &sf->put_compress_level; + break; + case SF_CON_PUT_USE_URANDOM_DEV: + *value = &sf->put_use_urand_dev; + break; + case SF_CON_PUT_FASTFAIL: + *value = &sf->put_fastfail; + break; + case SF_CON_PUT_MAXRETRIES: + *value = &sf->put_maxretries; + break; + case SF_CON_GET_FASTFAIL: + *value = &sf->get_fastfail; + break; + case SF_CON_GET_MAXRETRIES: + *value = &sf->get_maxretries; + break; + case SF_CON_GET_THRESHOLD: + *value = &sf->get_threshold; + break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -2039,6 +2124,29 @@ SF_STATUS STDCALL snowflake_query( return SF_STATUS_SUCCESS; } +SF_STATUS STDCALL _snowflake_query_put_get_legacy( + SF_STMT *sfstmt, const char *command, size_t command_size) { + if (!sfstmt) { + return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; + } + clear_snowflake_error(&sfstmt->error); + SF_STATUS ret = snowflake_prepare(sfstmt, command, command_size); + if (ret != SF_STATUS_SUCCESS) { + return ret; + } + if (!_is_put_get_command(sfstmt->sql_text)) + { + // this should never happen as this function should only be + // called internally for put/get command. + SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_GENERAL, + "Invalid query type, can be used for put get only", + SF_SQLSTATE_GENERAL_ERROR); + return SF_STATUS_ERROR_GENERAL; + } + + return _snowflake_execute_ex(sfstmt, SF_BOOLEAN_TRUE, SF_BOOLEAN_FALSE, NULL, SF_BOOLEAN_FALSE); +} + SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; @@ -2246,24 +2354,32 @@ snowflake_prepare(SF_STMT *sfstmt, const char *command, size_t command_size) { SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_TRUE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_FALSE, result_capture, SF_BOOLEAN_TRUE); } SF_STATUS STDCALL snowflake_execute(SF_STMT *sfstmt) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), NULL, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, NULL, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, result_capture, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, sf_bool is_put_get_command, + sf_bool is_native_put_get, SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; } + + if (is_put_get_command && is_native_put_get && !is_describe_only) + { + _snowflake_stmt_desc_reset(sfstmt); + return _snowflake_execute_put_get_native(sfstmt, result_capture); + } + clear_snowflake_error(&sfstmt->error); SF_STATUS ret = SF_STATUS_ERROR_GENERAL; SF_JSON_ERROR json_error; diff --git a/lib/client_int.h b/lib/client_int.h index 632856f031..47db68c595 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -25,6 +25,15 @@ #define SF_DEFAULT_MAX_OBJECT_SIZE 16777216 +// defaults for put get configurations +#define SF_DEFAULT_PUT_COMPRESS_LEVEL (-1) +#define SF_MAX_PUT_COMPRESS_LEVEL 9 +#define SF_DEFAULT_PUT_MAX_RETRIES 5 +#define SF_MAX_PUT_MAX_RETRIES 100 +#define SF_DEFAULT_GET_MAX_RETRIES 5 +#define SF_MAX_GET_MAX_RETRIES 100 +#define SF_DEFAULT_GET_THRESHOLD 5 + #define SESSION_URL "/session/v1/login-request" #define QUERY_URL "/queries/v1/query-request" #define RENEW_SESSION_URL "/session/token-request" @@ -141,7 +150,7 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); /** * Executes a statement. * @param sfstmt SNOWFLAKE_STMT context. - * @param sf_use_application_json_accept type true if this is a put/get command + * @param is_put_get_command type true if this is a put/get command * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, * @param is_describe_only should the statement be executed in describe only mode * if the query response is to be captured. @@ -149,7 +158,8 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); * @return 0 if success, otherwise an errno is returned. */ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, - sf_bool use_application_json_accept_type, + sf_bool is_put_get_command, + sf_bool is_native_put_get, struct SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only); @@ -163,4 +173,32 @@ sf_bool STDCALL _is_put_get_command(char* sql_text); */ PARAM_TYPE STDCALL _snowflake_get_param_style(const SF_BIND_INPUT *input); +#ifdef __cplusplus +extern "C" { +#endif +/** + * Legacy approach of Executing a query, not to execute put/get natively. + * Should only be called internally for put/get queries. + * + * @param sf SNOWFLAKE_STMT context. + * @param command a query or command that returns results. + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL +_snowflake_query_put_get_legacy(SF_STMT* sfstmt, const char* command, size_t command_size); + +/** + * Executes put get command natively. + * @param sfstmt SNOWFLAKE_STMT context. + * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, + * + * @return 0 if success, otherwise an errno is returned. + */ +SF_STATUS STDCALL _snowflake_execute_put_get_native( + SF_STMT *sfstmt, + struct SF_QUERY_RESULT_CAPTURE* result_capture); +#ifdef __cplusplus +} // extern "C" +#endif + #endif //SNOWFLAKE_CLIENT_INT_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2e1254ff77..6e55a0d063 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -195,7 +195,7 @@ if (LINUX) endif () message("valgrind suppression file is located at " ${VALGRIND_SUPPRESSION}) if (LINUX) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} -Wl,--whole-archive telemetry curl ssl crypto uuid + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid -Wl,--no-whole-archive pthread -Wl,--as-needed -static-libgcc -static-libstdc++) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++) @@ -209,12 +209,14 @@ endif() if (WIN32) if (WIN32_DEBUG) if(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ucrtd.lib) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} + Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() else() - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB}) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} + Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${CURL_LIB} ${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index c16a3f98c2..4096263221 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -118,7 +118,8 @@ void test_simple_put_core(const char * fileName, int compressLevel = -1, bool overwrite = false, SF_CONNECT * connection = nullptr, - bool testUnicode = false) + bool testUnicode = false, + bool native = false) { /* init */ SF_STATUS status; @@ -194,71 +195,140 @@ void test_simple_put_core(const char * fileName, putCommand += " overwrite=true"; } std::unique_ptr stmtPutGet; - if (testUnicode) - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else + if (!native) { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); + if (testUnicode) + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); + } } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (tmpDir) { + if (native) + { + snowflake_set_attribute(sf, SF_CON_PUT_TEMPDIR, tmpDir); + } + else + { transConfig.tempDir = tmpDir; transConfigPtr = &transConfig; + } } if(useS3regionalUrl) { - transConfig.useS3regionalUrl = true; - transConfigPtr = &transConfig; + if (native) + { + std::string cmd = "alter session set ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1=true"; + snowflake_query(sfstmt, cmd.c_str(), cmd.size()); + } + else + { + transConfig.useS3regionalUrl = true; + transConfigPtr = &transConfig; + } } if(compressLevel > 0) { - transConfig.compressLevel = compressLevel; - transConfigPtr = &transConfig; + if (native) + { + int8 lv = (int8)compressLevel; + snowflake_set_attribute(sf, SF_CON_PUT_COMPRESSLV, &lv); + } + else + { + transConfig.compressLevel = compressLevel; + transConfigPtr = &transConfig; + } } Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); if(useDevUrand){ - agent.setRandomDeviceAsUrand(true); + if (native) + { + sf_bool use_urand = SF_BOOLEAN_TRUE; + snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &use_urand); + } + else + { + agent.setRandomDeviceAsUrand(true); + } } - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + std::string expectedSrc = sf_filename_from_path(fileName); + std::string expectedTarget = (autoCompress && !strstr(expectedSrc.c_str(), ".gz")) ? + expectedSrc + ".gz" : expectedSrc; + std::string expectedSourceCompression = !strstr(fileName, ".gz") ? "none" : "gzip"; + std::string expectedTargetCompression = (!autoCompress && !strstr(fileName, ".gz")) ? "none" : "gzip"; - while(results->next()) + if (native) { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal( sf_filename_from_path(fileName), value.c_str()); - - std::string expectedTarget = (autoCompress && !strstr(fileName, ".gz")) ? - std::string(fileName) + ".gz" : - std::string(fileName); - results->getColumnAsString(1, value); // get target - assert_string_equal(sf_filename_from_path(expectedTarget.c_str()), value.c_str()); - - std::string expectedSourceCompression = !strstr(fileName, ".gz") ? - "none" : "gzip"; - results->getColumnAsString(4, value); // get source_compression - assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); - - std::string expectedTargetCompression = (!autoCompress && - !strstr(fileName, ".gz")) ? "none" : "gzip"; - results->getColumnAsString(5, value); // get target_compression - assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); - - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); - - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); + ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); + assert_int_equal(SF_STATUS_SUCCESS, ret); + + assert_int_equal(snowflake_num_rows(sfstmt), 1); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + + const char *out; + // source + snowflake_column_as_const_str(sfstmt, 1, &out); + assert_string_equal(expectedSrc.c_str(), out); + // target + snowflake_column_as_const_str(sfstmt, 2, &out); + assert_string_equal(expectedTarget.c_str(), out); + // source comparession + snowflake_column_as_const_str(sfstmt, 5, &out); + assert_string_equal(expectedSourceCompression.c_str(), out); + // target compression + snowflake_column_as_const_str(sfstmt, 6, &out); + assert_string_equal(expectedTargetCompression.c_str(), out); + snowflake_column_as_const_str(sfstmt, 7, &out); + // status + assert_string_equal("UPLOADED", out); + // encryption + snowflake_column_as_const_str(sfstmt, 8, &out); + assert_string_equal("ENCRYPTED", out); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_EOF, ret); + } + else + { + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); + + while(results->next()) + { + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal(expectedSrc.c_str(), value.c_str()); + + results->getColumnAsString(1, value); // get target + assert_string_equal(expectedTarget.c_str(), value.c_str()); + + results->getColumnAsString(4, value); // get source_compression + assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); + + results->getColumnAsString(5, value); // get target_compression + assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); + + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); + + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); + } } if (copyUploadFile) @@ -352,7 +422,8 @@ static int teardown(void **unused) } void test_simple_get_data(const char *getCommand, const char *size, - long getThreshold = 0, bool testUnicode = false) + long getThreshold = 0, bool testUnicode = false, + bool native = false) { /* init */ SF_STATUS status; @@ -367,39 +438,72 @@ void test_simple_get_data(const char *getCommand, const char *size, sfstmt = snowflake_stmt(sf); std::unique_ptr stmtPutGet; - if (testUnicode) + if (!native) { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); + if (testUnicode) + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); + } } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (getThreshold > 0) { + if (native) + { + int64 threshold = getThreshold; + status = snowflake_set_attribute(sf, SF_CON_GET_THRESHOLD, &threshold); + assert_int_equal(SF_STATUS_SUCCESS, status); + } + else + { transConfig.getSizeThreshold = getThreshold; transConfigPtr = &transConfig; + } } - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); - - // load first time should return uploaded - std::string get_status; - std::string getcmd(getCommand); - ITransferResult * results = agent.execute(&getcmd); - while(results && results->next()) + if (native) { - results->getColumnAsString(1, get_status); + status = snowflake_query(sfstmt, getCommand, strlen(getCommand)); + assert_int_equal(SF_STATUS_SUCCESS, status); + + while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) + { + const char *out; + // source + snowflake_column_as_const_str(sfstmt, 2, &out); //Compressed File sizes vary on Windows/Linux, So not verifying size. - results->getColumnAsString(2, get_status); - assert_string_equal("DOWNLOADED", get_status.c_str()); - results->getColumnAsString(3, get_status); - assert_string_equal("DECRYPTED", get_status.c_str()); + snowflake_column_as_const_str(sfstmt, 3, &out); + assert_string_equal("DOWNLOADED", out); + snowflake_column_as_const_str(sfstmt, 4, &out); + assert_string_equal("DECRYPTED", out); + } + assert_int_equal(SF_STATUS_EOF, status); + } + else + { + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); + + // load first time should return uploaded + std::string get_status; + std::string getcmd(getCommand); + ITransferResult * results = agent.execute(&getcmd); + while(results && results->next()) + { + results->getColumnAsString(1, get_status); + //Compressed File sizes vary on Windows/Linux, So not verifying size. + results->getColumnAsString(2, get_status); + assert_string_equal("DOWNLOADED", get_status.c_str()); + results->getColumnAsString(3, get_status); + assert_string_equal("DECRYPTED", get_status.c_str()); + } } snowflake_stmt_term(sfstmt); @@ -409,7 +513,7 @@ void test_simple_get_data(const char *getCommand, const char *size, } -void test_large_put_auto_compress(void **unused) +void test_large_put_auto_compress_core(bool native) { char *cenv = getenv("CLOUD_PROVIDER"); if ( cenv && !strncmp(cenv, "AWS", 4) ) { @@ -423,10 +527,32 @@ void test_large_put_auto_compress(void **unused) false, // auto compress true, // Load data into table false, // Run select * on loaded table (Not good for large data set) - true // copy data from Table to Staging. + true, // copy data from Table to Staging. + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); } +void test_large_put_auto_compress(void **unused) +{ + test_large_put_auto_compress_core(false); +} + +void test_large_put_auto_compress_native(void **unused) +{ + test_large_put_auto_compress_core(true); +} + void test_large_put_threshold(void **unused) { char *cenv = getenv("CLOUD_PROVIDER"); @@ -448,10 +574,10 @@ void test_large_put_threshold(void **unused) ); } -void test_large_reupload(void **unused) +void test_large_reupload_core(bool native) { - char *cenv = getenv("CLOUD_PROVIDER"); - if (cenv && !strncmp(cenv, "AWS", 4)) { + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "AWS", 4)) { errno = 0; return; } @@ -476,7 +602,7 @@ void test_large_reupload(void **unused) char tempFile[MAX_BUF_SIZE] ={0}; sf_get_tmp_dir(tempDir); #ifdef _WIN32 - getLongTempPath(tempDir); + getLongTempPath(tempDir); #endif for(const std::string s : fileList) { tempFile[0] = 0; @@ -487,10 +613,30 @@ void test_large_reupload(void **unused) true, // Load data into table false, // Run select * on loaded table (Not good for large data set) false, // copy data from Table to Staging. - true //Creates a dup table to compare uploaded data. + true, //Creates a dup table to compare uploaded data. + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); } +} +void test_large_reupload(void** unused) +{ + test_large_reupload_core(false); +} + +void test_large_reupload_native(void** unused) +{ + test_large_reupload_core(true); } /* @@ -538,7 +684,7 @@ void test_verify_upload(void **unused) } -void test_simple_put_use_dev_urandom(void **unused) +void test_simple_put_use_dev_urandom_core(bool native) { std::string dataDir = TestSetup::getDataDir(); std::string file = dataDir + "medium_file.csv"; @@ -567,32 +713,67 @@ void test_simple_put_use_dev_urandom(void **unused) std::string putCommand = "put file://" + file + " @%test_small_put auto_compress=true overwrite=true"; - std::unique_ptr stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); - - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); - agent.setRandomDeviceAsUrand(true); + if (native) + { + sf_bool useUrand = SF_BOOLEAN_TRUE; + ret = snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &useUrand); + assert_int_equal(SF_STATUS_SUCCESS, ret); + ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); + assert_int_equal(SF_STATUS_SUCCESS, ret); + assert_int_equal(snowflake_num_rows(sfstmt), 1); - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); - while (results->next()) + const char* out = NULL; + // source + snowflake_column_as_const_str(sfstmt, 1, &out); + assert_string_equal(sf_filename_from_path(file.c_str()), out); + // target + snowflake_column_as_const_str(sfstmt, 2, &out); + assert_string_equal("medium_file.csv.gz", out); + // source compression + snowflake_column_as_const_str(sfstmt, 5, &out); + assert_string_equal("none", out); + // status + snowflake_column_as_const_str(sfstmt, 7, &out); + assert_string_equal("UPLOADED", out); + // encryption + snowflake_column_as_const_str(sfstmt, 8, &out); + assert_string_equal("ENCRYPTED", out); + + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_EOF, ret); + } + else { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); + std::unique_ptr stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); - results->getColumnAsString(1, value); // get target - assert_string_equal("medium_file.csv.gz", value.c_str()); + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); + agent.setRandomDeviceAsUrand(true); + + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); + + while (results->next()) + { + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); - results->getColumnAsString(4, value); // get source_compression - assert_string_equal("none", value.c_str()); + results->getColumnAsString(1, value); // get target + assert_string_equal("medium_file.csv.gz", value.c_str()); - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); + results->getColumnAsString(4, value); // get source_compression + assert_string_equal("none", value.c_str()); - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); + + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); + } } std::string copyCommand = "copy into test_small_put from @%test_small_put"; @@ -610,30 +791,56 @@ void test_simple_put_use_dev_urandom(void **unused) for(int i = 0; i < 200000; ++i) { - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_SUCCESS, ret); - snowflake_column_as_const_str(sfstmt, 1, &out_c1); - snowflake_column_as_const_str(sfstmt, 2, &out_c2); - snowflake_column_as_const_str(sfstmt, 3, &out_c3); - std::string c1 = std::to_string(i); - std::string c2 = std::to_string(i + 1); - assert_string_equal(out_c1, c1.c_str()); - assert_string_equal(out_c2, c2.c_str()); - assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - out_c1 = NULL; - out_c2 = NULL; - out_c3 = NULL; + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + snowflake_column_as_const_str(sfstmt, 1, &out_c1); + snowflake_column_as_const_str(sfstmt, 2, &out_c2); + snowflake_column_as_const_str(sfstmt, 3, &out_c3); + std::string c1 = std::to_string(i); + std::string c2 = std::to_string(i + 1); + assert_string_equal(out_c1, c1.c_str()); + assert_string_equal(out_c2, c2.c_str()); + assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + out_c1 = NULL; + out_c2 = NULL; + out_c3 = NULL; } ret = snowflake_fetch(sfstmt); assert_int_equal(SF_STATUS_EOF, ret); } -void test_simple_put_auto_compress(void **unused) +void test_simple_put_use_dev_urandom(void **unused) +{ + test_simple_put_use_dev_urandom_core(false); +} + +void test_simple_put_use_dev_urandom_native(void **unused) +{ + test_simple_put_use_dev_urandom_core(true); +} + + +void test_simple_put_auto_compress_core(bool native) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true // auto compress + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); test_simple_put_core("small_file.csv", // filename @@ -650,7 +857,11 @@ void test_simple_put_auto_compress(void **unused) nullptr, false, 1, - true); + true, // overwrite + nullptr, // connection + false, // testUnicode + native + ); test_simple_put_core("small_file.csv", // filename "auto", //source compression @@ -666,10 +877,24 @@ void test_simple_put_auto_compress(void **unused) nullptr, false, 9, - true); + true, // overwrite + nullptr, // connection + false, // testUnicode + native + ); } -void test_simple_put_config_temp_dir(void **unused) +void test_simple_put_auto_compress(void **unused) +{ + test_simple_put_auto_compress_core(false); +} + +void test_simple_put_auto_compress_native(void **unused) +{ + test_simple_put_auto_compress_core(true); +} + +void test_simple_put_config_temp_dir_core(bool native) { char tmpDir[MAX_PATH] = {0}; char tmpDirInjection[MAX_PATH] = {0}; @@ -696,7 +921,13 @@ void test_simple_put_config_temp_dir(void **unused) 64*1024*1024, // customThreshold false, // useDevUrand false, // createSubfolder - tmpDir + tmpDir, + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); assert_true(sf_is_directory_exist(tmpDir)); @@ -721,31 +952,51 @@ void test_simple_put_config_temp_dir(void **unused) sf_delete_directory_if_exists(tmpDir); } - // try injection the command for folder deletion like - // rm -rf xxx ; mkdir /injection ; xxx - sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", - CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); - try + // native execution doesn't throw exception + if (!native) { - test_simple_put_core("small_file.csv", // filename - "auto", //source compression - true, // auto compress - true, // copyUploadFile - true, // verifyCopyUploadFile - false, // copyTableToStaging - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - tmpDirInjection - ); - } - catch (...) - { - //ignore exception as the failure is expected. - } - assert_false(sf_is_directory_exist(tmpDir)); + // try injection the command for folder deletion like + // rm -rf xxx ; mkdir /injection ; xxx + sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", + CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); + try + { + test_simple_put_core("small_file.csv", // filename + "auto", //source compression + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + tmpDirInjection, + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native + ); + } + catch (...) + { + //ignore exception as the failure is expected. + } + assert_false(sf_is_directory_exist(tmpDir)); + } +} + +void test_simple_put_config_temp_dir(void **unused) +{ + test_simple_put_config_temp_dir_core(false); +} + +void test_simple_put_config_temp_dir_native(void **unused) +{ + test_simple_put_config_temp_dir_core(true); } void test_simple_put_auto_detect_gzip(void ** unused) @@ -789,7 +1040,7 @@ void test_simple_put_create_subfolder(void **unused) test_simple_put_core("small_file.csv.gz", "gzip", false, false, false, false, false, false, 100*1024*1024, false, true); } -void test_simple_put_use_s3_regionalURL(void **unused) +void test_simple_put_use_s3_regionalURL_core(bool native) { test_simple_put_core("small_file.csv.gz", "gzip", false,false, false, @@ -800,30 +1051,76 @@ void test_simple_put_use_s3_regionalURL(void **unused) false, false, nullptr, - true); + true, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native + ); } -void test_simple_get(void **unused) +void test_simple_put_use_s3_regionalURL(void **unused) +{ + test_simple_put_use_s3_regionalURL_core(false); +} + +void test_simple_put_use_s3_regionalURL_native(void **unused) +{ + test_simple_put_use_s3_regionalURL_core(true); +} + +void test_simple_get_core(bool native) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true // auto compress + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, //compressLevel + false, // overwrite + nullptr, // connection + false, // testUnicode + native ); char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; sf_get_tmp_dir(tempDir); #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "48"); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "48", // size + 0, // getThreshold + false, // testUnicode + native + ); } -void test_large_get(void **unused) +void test_simple_get(void **unused) +{ + test_simple_get_core(false); +} + +void test_simple_get_native(void **unused) +{ + test_simple_get_core(true); +} + +void test_large_get_core(bool native) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -832,14 +1129,29 @@ void test_large_get(void **unused) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "5166848"); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "5166848", // size + 0, // getThreshold + false, // testUnicode + native + ); } -void test_large_get_threshold(void **unused) +void test_large_get(void **unused) +{ + test_large_get_core(false); +} + +void test_large_get_native(void **unused) +{ + test_large_get_core(true); +} + +void test_large_get_threshold_core(bool native) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -848,8 +1160,23 @@ void test_large_get_threshold(void **unused) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(tempPath, tempDir); - test_simple_get_data(tempPath, "5166848", 1000000); + strcat(command, tempDir); + test_simple_get_data(command, // getCommand + "5166848", // size + 1000000, // getThreshold + false, // testUnicode + native + ); +} + +void test_large_get_threshold(void **unused) +{ + test_large_get_threshold_core(false); +} + +void test_large_get_threshold_native(void **unused) +{ + test_large_get_threshold_core(true); } static int gr_setup(void **unused) @@ -1654,6 +1981,7 @@ int main(void) { #endif const struct CMUnitTest tests[] = { +/* cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), @@ -1685,6 +2013,17 @@ int main(void) { cmocka_unit_test_teardown(test_simple_put_with_noproxy_fromenv, teardown), cmocka_unit_test_teardown(test_upload_file_to_stage_using_stream, donothing), cmocka_unit_test_teardown(test_put_get_with_unicode, teardown), +*/ + cmocka_unit_test_teardown(test_simple_put_auto_compress_native, teardown), + cmocka_unit_test_teardown(test_simple_put_config_temp_dir_native, teardown), + cmocka_unit_test_teardown(test_simple_get_native, teardown), + cmocka_unit_test_teardown(test_large_put_auto_compress_native, donothing), + cmocka_unit_test_teardown(test_large_get_native, donothing), + cmocka_unit_test_teardown(test_large_get_threshold_native, donothing), + cmocka_unit_test_teardown(test_large_reupload_native, donothing), + cmocka_unit_test_teardown(test_verify_upload, teardown), + cmocka_unit_test_teardown(test_simple_put_use_dev_urandom_native, teardown), + cmocka_unit_test_teardown(test_simple_put_use_s3_regionalURL_native, teardown), }; int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown); return ret; From 96b9b87390dea69757fdf23b7b128ebb21450029 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 12:35:47 -0700 Subject: [PATCH 31/74] Implement easy logging --- CMakeLists.txt | 5 +- cpp/lib/ClientConfigParser.cpp | 289 +++++++++++++++++++++++ include/snowflake/ClientConfigParser.hpp | 93 ++++++++ include/snowflake/client.h | 5 +- lib/client.c | 68 +++++- lib/client_config_parser.h | 44 ++++ lib/logger.c | 4 + tests/test_unit_logger.c | 50 +++- 8 files changed, 549 insertions(+), 9 deletions(-) create mode 100644 cpp/lib/ClientConfigParser.cpp create mode 100644 include/snowflake/ClientConfigParser.hpp create mode 100644 lib/client_config_parser.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 82ccd84aa5..3515e21939 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ set(SOURCE_FILES_CPP_WRAPPER include/snowflake/SFURL.hpp include/snowflake/CurlDesc.hpp include/snowflake/CurlDescPool.hpp + include/snowflake/ClientConfigParser.hpp cpp/lib/Exceptions.cpp cpp/lib/Connection.cpp cpp/lib/Statement.cpp @@ -235,6 +236,7 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ResultSetJson.hpp cpp/lib/Authenticator.cpp cpp/lib/Authenticator.hpp + cpp/lib/ClientConfigParser.cpp cpp/jwt/jwtWrapper.cpp cpp/util/SnowflakeCommon.cpp cpp/util/SFURL.cpp @@ -243,7 +245,8 @@ set(SOURCE_FILES_CPP_WRAPPER lib/result_set.h lib/query_context_cache.h lib/curl_desc_pool.h - lib/authenticator.h) + lib/authenticator.h + lib/client_config_parser.h) if (UNIX) if (LINUX) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp new file mode 100644 index 0000000000..78a7964c0c --- /dev/null +++ b/cpp/lib/ClientConfigParser.cpp @@ -0,0 +1,289 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#include "snowflake/ClientConfigParser.hpp" +#include "snowflake/Exceptions.hpp" +#include "../logger/SFLogger.hpp" +#include "client_config_parser.h" +#include "memory.h" + +#include +#include +#include + +#undef snprintf +#include +#include + +using namespace Snowflake::Client; + +namespace +{ + // constants + const std::string SF_CLIENT_CONFIG_FILE_NAME("sf_client_config.json"); + const std::string SF_CLIENT_CONFIG_ENV_NAME("SF_CLIENT_CONFIG_FILE"); + std::set KnownCommonEntries = {"log_level", "log_path"}; +} + +//////////////////////////////////////////////////////////////////////////////// +void load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig) +{ + ClientConfigParser configParser; + ClientConfig clientConfig; + configParser.loadClientConfig(in_configFilePath, clientConfig); + if (!clientConfig.logLevel.empty()) + { + out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); + strcpy(out_clientConfig->logLevel, clientConfig.logLevel.data()); + } + if (!clientConfig.logPath.empty()) + { + out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); + strcpy(out_clientConfig->logPath, clientConfig.logPath.data()); + } +} + +// Public ====================================================================== +//////////////////////////////////////////////////////////////////////////////// +ClientConfigParser::ClientConfigParser() +{ + ; // Do nothing. +} + +//////////////////////////////////////////////////////////////////////////////// +ClientConfigParser::~ClientConfigParser() +{ + ; // Do nothing. +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::loadClientConfig( + const std::string& in_configFilePath, + ClientConfig& out_clientConfig) +{ + char envbuf[MAX_PATH + 1]; + std::string derivedConfigPath = ""; + if (!in_configFilePath.empty()) + { + // 1. Try config file if it was passed in + derivedConfigPath = in_configFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from a connection string: %s", in_configFilePath.c_str()); + } + else if (const char* clientConfigEnv = sf_getenv_s(SF_CLIENT_CONFIG_ENV_NAME.c_str(), envbuf, sizeof(envbuf))) + { + // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME + derivedConfigPath = clientConfigEnv; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from an environment variable: %s", clientConfigEnv); + } + else + { + // 3. Try DLL binary dir + std::string binaryDir = getBinaryPath(); + std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(binaryDirFilePath)) + { + derivedConfigPath = binaryDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); + } + else + { +#if defined(WIN32) || defined(_WIN64) + // 4. Try user home dir + std::string homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); + if (homeDir.empty()) + { + // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH + std::string homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); + std::string homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); + if (!homeDriveEnv.empty() && !homePathEnv.empty()) + { + homeDir = homeDriveEnv + homePathEnv; + } + } + std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(homeDirFilePath)) + { + derivedConfigPath = homeDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + } +#else + // 4. Try user home dir + if (std::string homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + { + std::string homeDirFilePath = homeDir + SF_PATH_SEPARATOR + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::exists(homeDirFilePath)) + { + derivedConfigPath = homeDirFilePath; + CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", + "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + } + } +#endif + } + } + if (!derivedConfigPath.empty()) + { + parseConfigFile(derivedConfigPath, out_clientConfig); + } +} + +// Private ===================================================================== +//////////////////////////////////////////////////////////////////////////////// +bool ClientConfigParser::checkFileExists(const std::string& in_filePath) +{ + FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); + if (file != nullptr) + { + fclose(file); + return true; + } + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::parseConfigFile( + const std::string& in_filePath, + ClientConfig& out_clientConfig) +{ + std::ifstream configFile; + cJSON* jsonConfig; + try + { + configFile.open(in_filePath); + if (!configFile) + { + CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", + "Could not open a file. The file may not exist: %s", + in_filePath.c_str()); + std::string errMsg = "Error finding client configuration file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } +#if !defined(WIN32) && !defined(_WIN64) + checkIfValidPermissions(in_filePath); +#endif + std::stringstream configJSON; + configJSON << configFile.rdbuf(); + jsonConfig = snowflake_cJSON_Parse(configJSON.str().c_str()); + const char* error_ptr = snowflake_cJSON_GetErrorPtr(); + if (error_ptr) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "parseConfigFile", + "Error in parsing JSON: %s, err: %s", in_filePath.c_str(), error_ptr); + std::string errMsg = "Error parsing client configuration file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } + } + catch (std::exception& ex) + { + configFile.close(); + throw; + } + + const cJSON* commonProps = snowflake_cJSON_GetObjectItem(jsonConfig, "common"); + checkUnknownEntries(snowflake_cJSON_Print(commonProps)); + const cJSON* logLevel = snowflake_cJSON_GetObjectItem(commonProps, "log_level"); + if (snowflake_cJSON_IsString(logLevel)) + { + out_clientConfig.logLevel = snowflake_cJSON_GetStringValue(logLevel); + } + const cJSON* logPath = snowflake_cJSON_GetObjectItem(commonProps, "log_path"); + if (snowflake_cJSON_IsString(logPath)) + { + out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); + } + configFile.close(); +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) +{ + boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); + boost::filesystem::perms permissions = fileStatus.permissions(); + if (permissions & boost::filesystem::group_write || + permissions & boost::filesystem::others_write) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "checkIfValidPermissions", + "Error due to other users having permission to modify the config file: %s", + in_filePath.c_str()); + std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; + throw std::exception(errMsg.c_str()); + } +} + +//////////////////////////////////////////////////////////////////////////////// +void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) +{ + cJSON* jsonConfig = snowflake_cJSON_Parse(in_jsonString.c_str()); + const char* error_ptr = snowflake_cJSON_GetErrorPtr(); + if (error_ptr) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "checkUnknownEntries", + "Error in parsing JSON: %s, err: %s", in_jsonString.c_str(), error_ptr); + std::string errMsg = "Error parsing json: " + in_jsonString; + throw std::exception(errMsg.c_str()); + } + + if (snowflake_cJSON_IsObject(jsonConfig)) + { + cJSON* entry = NULL; + snowflake_cJSON_ArrayForEach(entry, jsonConfig) + { + std::string key = entry->string; + boost::algorithm::to_lower(key); + if (KnownCommonEntries.find(key) == KnownCommonEntries.end()) + { + std::string warnMsg = + "Unknown configuration entry: " + key + " with value:" + entry->valuestring; + CXX_LOG_WARN( + "sf", + "ClientConfigParser", + "checkUnknownEntries", + warnMsg); + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +std::string ClientConfigParser::getBinaryPath() +{ + std::wstring binaryFullPath; + +#if defined(WIN32) || defined(_WIN64) + HMODULE hm = NULL; + wchar_t appName[256]; + GetModuleFileNameW(hm, appName, 256); +#else + Dl_info info; + int result = dladdr((void*)getBinaryPath, &info); + if (result) + { + binaryFullPath = info.dli_fname; + } +#endif + + binaryFullPath = appName; + size_t pos = binaryFullPath.find_last_of(PATH_SEP); + if (pos == std::wstring::npos) + { + return ""; + } + std::wstring binaryPath = binaryFullPath.substr(0, pos + 1); + return std::string(binaryPath.begin(), binaryPath.end()); +} diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp new file mode 100644 index 0000000000..9aebfe1ce0 --- /dev/null +++ b/include/snowflake/ClientConfigParser.hpp @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef PROJECT_CLIENTCONFIG_HPP +#define PROJECT_CLIENTCONFIG_HPP + +#include + +namespace Snowflake +{ +namespace Client +{ + struct ClientConfig + { + // The log level + std::string logLevel = ""; + + // The log path + std::string logPath = ""; + }; + + class ClientConfigParser + { + // Public ================================================================== + public: + /** + * Constructor for client config + */ + ClientConfigParser(); + + /// @brief Destructor. + ~ClientConfigParser(); + + /** + * Construct SFClientConfig from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The SFClientConfig object to be filled. + */ + void loadClientConfig( + const std::string& in_configFilePath, + ClientConfig& out_clientConfig); + + // Private ================================================================= + private: + /** + * @brief Check if the file exists. + * + * @param in_filePath The file path to check. + */ + bool checkFileExists(const std::string& in_filePath); + + /** + * @brief Parse JSON string. + * + * @param in_filePath The filePath of the config file to parse. + * @param out_clientConfig The SFClientConfig object to be filled. + */ + void parseConfigFile( + const std::string& in_filePath, + ClientConfig& out_clientConfig); + + /** + * @ brief Check if other have permission to modify file + * + * @param in_filePath The file path of the config file to check permissions. + */ + void checkIfValidPermissions(const std::string& in_filePath); + + /** + * @ brief Check if there are unknown entries in config file + * + * @param in_jsonString The json object to check in json config file. + */ + void checkUnknownEntries(const std::string& in_jsonString); + + /** + * @ brief Get the path to the binary file + */ + std::string getBinaryPath(); + }; + +} // namespace Client +} // namespace Snowflake + +#endif //PROJECT_CLIENTCONFIG_HPP diff --git a/include/snowflake/client.h b/include/snowflake/client.h index c045795f6a..f69f1353e1 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -298,7 +298,10 @@ typedef enum SF_GLOBAL_ATTRIBUTE { SF_GLOBAL_CA_BUNDLE_FILE, SF_GLOBAL_SSL_VERSION, SF_GLOBAL_DEBUG, - SF_GLOBAL_OCSP_CHECK + SF_GLOBAL_OCSP_CHECK, + SF_GLOBAL_CLIENT_CONFIG_FILE, + SF_GLOBAL_LOG_LEVEL, + SF_GLOBAL_LOG_PATH } SF_GLOBAL_ATTRIBUTE; /** diff --git a/lib/client.c b/lib/client.c index f6bdc93cdf..6d17e9ef90 100644 --- a/lib/client.c +++ b/lib/client.c @@ -19,6 +19,7 @@ #include "chunk_downloader.h" #include "authenticator.h" #include "query_context_cache.h" +#include "client_config_parser.h" #ifdef _WIN32 #include @@ -36,6 +37,8 @@ sf_bool DEBUG; sf_bool SF_OCSP_CHECK; char *SF_HEADER_USER_AGENT = NULL; +static char *CLIENT_CONFIG_FILE = NULL; +static char* LOG_LEVEL; static char *LOG_PATH = NULL; static FILE *LOG_FP = NULL; @@ -310,18 +313,34 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { SF_LOG_LEVEL sf_log_level = log_level; char strerror_buf[SF_ERROR_BUFSIZE]; + char client_config_file[MAX_PATH] = {0}; + snowflake_global_get_attribute( + SF_GLOBAL_CLIENT_CONFIG_FILE, client_config_file, sizeof(client_config_file)); + client_config clientConfig = { .logLevel = "", .logPath = "" }; + load_client_config(client_config_file, &clientConfig); + size_t log_path_size = 1; //Start with 1 to include null terminator log_path_size += strlen(time_str); - /* The environment variables takes precedence over the specified parameters */ - sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); - if (sf_log_path == NULL && log_path) { + /* The client config takes precedence over environmental variables */ + if (strlen(clientConfig.logPath) != 0) { + sf_log_path = clientConfig.logPath; + } else { + /* The environment variables takes precedence over the specified parameters */ + sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); + if (sf_log_path == NULL && log_path) { sf_log_path = log_path; + } } - sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); - if (sf_log_level_str != NULL) { + if (strlen(clientConfig.logLevel) != 0) { + sf_log_level = log_from_str_to_level(clientConfig.logLevel); + } else { + /* The client config takes precedence over environment variables */ + sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); + if (sf_log_level_str != NULL) { sf_log_level = log_from_str_to_level(sf_log_level_str); + } } // Set logging level @@ -354,6 +373,9 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { goto cleanup; } + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); + snowflake_global_set_attribute(SF_GLOBAL_LOG_LEVEL, log_from_level_to_str(sf_log_level)); + ret = SF_BOOLEAN_TRUE; cleanup: @@ -564,7 +586,6 @@ _snowflake_check_connection_parameters(SF_CONNECT *sf) { return SF_STATUS_SUCCESS; } - SF_STATUS STDCALL snowflake_global_init( const char *log_path, SF_LOG_LEVEL log_level, SF_USER_MEM_HOOKS *hooks) { SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -580,11 +601,13 @@ SF_STATUS STDCALL snowflake_global_init( _snowflake_memory_hooks_setup(hooks); sf_memory_init(); sf_error_init(); + if (!log_init(log_path, log_level)) { // no way to log error because log_init failed. sf_fprintf(stderr, "Error during log initialization"); goto cleanup; } + CURLcode curl_ret = curl_global_init(CURL_GLOBAL_DEFAULT); if (curl_ret != CURLE_OK) { log_fatal("curl_global_init() failed: %s", @@ -648,6 +671,15 @@ snowflake_global_set_attribute(SF_GLOBAL_ATTRIBUTE type, const void *value) { case SF_GLOBAL_OCSP_CHECK: SF_OCSP_CHECK = *(sf_bool *) value; break; + case SF_GLOBAL_CLIENT_CONFIG_FILE: + alloc_buffer_and_copy(&CLIENT_CONFIG_FILE, value); + break; + case SF_GLOBAL_LOG_LEVEL: + alloc_buffer_and_copy(&LOG_LEVEL, value); + break; + case SF_GLOBAL_LOG_PATH: + alloc_buffer_and_copy(&LOG_PATH, value); + break; default: break; } @@ -677,6 +709,30 @@ snowflake_global_get_attribute(SF_GLOBAL_ATTRIBUTE type, void *value, size_t siz case SF_GLOBAL_OCSP_CHECK: *((sf_bool *) value) = SF_OCSP_CHECK; break; + case SF_GLOBAL_CLIENT_CONFIG_FILE: + if (CLIENT_CONFIG_FILE) { + if (strlen(CLIENT_CONFIG_FILE) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, CLIENT_CONFIG_FILE, size); + } + break; + case SF_GLOBAL_LOG_LEVEL: + if (LOG_LEVEL) { + if (strlen(LOG_LEVEL) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, LOG_LEVEL, size); + } + break; + case SF_GLOBAL_LOG_PATH: + if (LOG_PATH) { + if (strlen(LOG_PATH) > size - 1) { + return SF_STATUS_ERROR_BUFFER_TOO_SMALL; + } + sf_strncpy(value, size, LOG_PATH, size); + } + break; default: break; } diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h new file mode 100644 index 0000000000..448ce1794b --- /dev/null +++ b/lib/client_config_parser.h @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef SNOWFLAKE_CLIENTCONFIG_H +#define SNOWFLAKE_CLIENTCONFIG_H + +#include "snowflake/client.h" +#include "cJSON.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct client_config + { + // The log level + char *logLevel; + + // The log path + char *logPath; + } client_config; + + /** + * Construct ClientConfig from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The ClientConfig object to be filled. + */ + void load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif //SNOWFLAKE_CLIENTCONFIG_H diff --git a/lib/logger.c b/lib/logger.c index 9d43929aaa..5bb5d6d72a 100644 --- a/lib/logger.c +++ b/lib/logger.c @@ -216,6 +216,10 @@ SF_LOG_LEVEL log_from_str_to_level(const char *level_in_str) { return SF_LOG_FATAL; } +const char* log_from_level_to_str(SF_LOG_LEVEL level) { + return level_names[level]; +} + void log_set_path(const char *path) { L.path = path; } diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 1e756b618d..78aae50fa8 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -4,7 +4,14 @@ #include "utils/test_setup.h" -#ifndef _WIN32 +#include +#include "memory.h" + +#ifdef _WIN32 +inline int access(const char *pathname, int mode) { + return _access(pathname, mode); +} +#else #include #endif @@ -24,6 +31,46 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } +/** + * Tests log settings from client config file + */ +void test_client_config_log(void **unused) { + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE *file; + file = fopen(configFilePath,"w"); + fprintf(file, clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + load_client_config(configFilePath, &clientConfig); + + // Set log name and level + char logname[] = "%s/dummy.log"; + size_t log_path_size = 1 + strlen(logname); + log_path_size += strlen(clientConfig.logPath); + char* LOG_PATH = (char*)SF_CALLOC(1, log_path_size); + sf_sprintf(LOG_PATH, log_path_size, logname, clientConfig.logPath); + log_set_level(log_from_str_to_level(clientConfig.logLevel)); + log_set_path(LOG_PATH); + + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, 0), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, 0), 0); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + #ifndef _WIN32 /** * Tests timing of log file creation @@ -140,6 +187,7 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), + cmocka_unit_test(test_client_config_log), #ifndef _WIN32 cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), From 8ce9da6fce2f719df1dbd799ee24cab26203d0be Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 13:08:40 -0700 Subject: [PATCH 32/74] Add missing function definition --- include/snowflake/logger.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/snowflake/logger.h b/include/snowflake/logger.h index b11ec1a14c..b860f8ecd8 100644 --- a/include/snowflake/logger.h +++ b/include/snowflake/logger.h @@ -94,6 +94,8 @@ void log_masked_va_list(FILE* fp, const char *fmt, va_list args); SF_LOG_LEVEL log_from_str_to_level(const char *level_in_str); +const char* log_from_level_to_str(SF_LOG_LEVEL level); + void log_set_path(const char* path); void log_close(); From 9fcc79046dc613f348509e833d23ba24ac3ce3e1 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 15:02:14 -0700 Subject: [PATCH 33/74] Fix linux build errors --- cpp/lib/ClientConfigParser.cpp | 19 +++++++++++-------- include/snowflake/ClientConfigParser.hpp | 11 +++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 78a7964c0c..0462c451e5 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -16,6 +16,10 @@ #include #include +#ifndef _WIN32 +#include +#endif + using namespace Snowflake::Client; namespace @@ -115,9 +119,9 @@ void ClientConfigParser::loadClientConfig( } #else // 4. Try user home dir - if (std::string homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { - std::string homeDirFilePath = homeDir + SF_PATH_SEPARATOR + SF_CLIENT_CONFIG_FILE_NAME; + std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (boost::filesystem::exists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; @@ -163,7 +167,7 @@ void ClientConfigParser::parseConfigFile( "Could not open a file. The file may not exist: %s", in_filePath.c_str()); std::string errMsg = "Error finding client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); @@ -180,7 +184,7 @@ void ClientConfigParser::parseConfigFile( "parseConfigFile", "Error in parsing JSON: %s, err: %s", in_filePath.c_str(), error_ptr); std::string errMsg = "Error parsing client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } catch (std::exception& ex) @@ -219,7 +223,7 @@ void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) "Error due to other users having permission to modify the config file: %s", in_filePath.c_str()); std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } @@ -236,7 +240,7 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) "checkUnknownEntries", "Error in parsing JSON: %s, err: %s", in_jsonString.c_str(), error_ptr); std::string errMsg = "Error parsing json: " + in_jsonString; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } if (snowflake_cJSON_IsObject(jsonConfig)) @@ -269,6 +273,7 @@ std::string ClientConfigParser::getBinaryPath() HMODULE hm = NULL; wchar_t appName[256]; GetModuleFileNameW(hm, appName, 256); + binaryFullPath = appName; #else Dl_info info; int result = dladdr((void*)getBinaryPath, &info); @@ -277,8 +282,6 @@ std::string ClientConfigParser::getBinaryPath() binaryFullPath = info.dli_fname; } #endif - - binaryFullPath = appName; size_t pos = binaryFullPath.find_last_of(PATH_SEP); if (pos == std::wstring::npos) { diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 9aebfe1ce0..3b12d970b5 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -20,6 +20,17 @@ namespace Client std::string logPath = ""; }; + struct ClientConfigException : public std::exception + { + ClientConfigException(const std::string& message) : message_(message) {} + const char* what() const noexcept + { + return message_.c_str(); + } + + std::string message_; + }; + class ClientConfigParser { // Public ================================================================== From 1f778c0a8d1a79a34f07c15b7a681b7acc632c19 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 15:58:02 -0700 Subject: [PATCH 34/74] Fix conversion issue on linux --- cpp/lib/ClientConfigParser.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 0462c451e5..5604838c0b 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -267,26 +267,27 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) //////////////////////////////////////////////////////////////////////////////// std::string ClientConfigParser::getBinaryPath() { - std::wstring binaryFullPath; - + std::string binaryFullPath; #if defined(WIN32) || defined(_WIN64) + std::wstring path; HMODULE hm = NULL; wchar_t appName[256]; GetModuleFileNameW(hm, appName, 256); - binaryFullPath = appName; + path = appName; + binaryFullPath = std::string(path.begin(), path.end()); #else Dl_info info; - int result = dladdr((void*)getBinaryPath, &info); + int result = dladdr((void*)load_client_config, &info); if (result) { - binaryFullPath = info.dli_fname; + binaryFullPath = std::string(info.dli_fname); } #endif size_t pos = binaryFullPath.find_last_of(PATH_SEP); - if (pos == std::wstring::npos) + if (pos == std::string::npos) { return ""; } - std::wstring binaryPath = binaryFullPath.substr(0, pos + 1); - return std::string(binaryPath.begin(), binaryPath.end()); + std::string binaryPath = binaryFullPath.substr(0, pos + 1); + return binaryPath; } From 17e1d10e480ec3cb275ca8e6aa1bc0e9bf102cff Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 2 Oct 2024 16:54:33 -0700 Subject: [PATCH 35/74] Fix mac build error and fix logging test failure --- cpp/lib/ClientConfigParser.cpp | 4 ++-- tests/test_unit_logger.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 5604838c0b..270f989587 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -121,7 +121,7 @@ void ClientConfigParser::loadClientConfig( // 4. Try user home dir if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { - std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (boost::filesystem::exists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; @@ -258,7 +258,7 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) "sf", "ClientConfigParser", "checkUnknownEntries", - warnMsg); + warnMsg.c_str()); } } } diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 78aae50fa8..aa49a870ac 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -65,6 +65,7 @@ void test_client_config_log(void **unused) { // Warning log will trigger the log file creation log_warn("dummy warning log"); assert_int_equal(access(LOG_PATH, 0), 0); + log_close(); // Cleanup remove(configFilePath); @@ -94,6 +95,7 @@ void test_log_creation(void **unused) { // warning log will trigger the log file creation log_warn("dummy warning log"); assert_int_equal(access(logname, F_OK), 0); + log_close(); remove(logname); } From 5c90290f57a55249933dd470e553dbf5ee91e26e Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 3 Oct 2024 10:05:05 -0700 Subject: [PATCH 36/74] Fix mac build error in test case --- tests/test_unit_logger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index aa49a870ac..a87183185f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -39,7 +39,7 @@ void test_client_config_log(void **unused) { char configFilePath[] = "sf_client_config.json"; FILE *file; file = fopen(configFilePath,"w"); - fprintf(file, clientConfigJSON); + fprintf(file, "%s", clientConfigJSON); fclose(file); // Parse client config for log details From ac79374b18561d582cb8c52e6f923df06ebba7f9 Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 3 Oct 2024 16:23:18 -0700 Subject: [PATCH 37/74] Fix setting of global log path --- lib/client.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/client.c b/lib/client.c index 6d17e9ef90..853c23f45b 100644 --- a/lib/client.c +++ b/lib/client.c @@ -359,10 +359,12 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "%s/snowflake_%s.txt", sf_log_path, (char *) time_str); + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); } else { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "logs/snowflake_%s.txt", (char *) time_str); + snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, "logs/"); } if (LOG_PATH != NULL) { // Set the log path only, the log file will be created when actual log output is needed. @@ -373,7 +375,6 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { goto cleanup; } - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); snowflake_global_set_attribute(SF_GLOBAL_LOG_LEVEL, log_from_level_to_str(sf_log_level)); ret = SF_BOOLEAN_TRUE; From 99daf14cd4a2d91f785cada00534e65163c67a11 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 4 Oct 2024 10:16:39 -0700 Subject: [PATCH 38/74] Use sf_memcpy instead --- cpp/lib/ClientConfigParser.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 270f989587..3979894030 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -41,12 +41,14 @@ void load_client_config( if (!clientConfig.logLevel.empty()) { out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); - strcpy(out_clientConfig->logLevel, clientConfig.logLevel.data()); + sf_memcpy(out_clientConfig->logLevel, sizeof(out_clientConfig->logLevel), + clientConfig.logLevel.data(), clientConfig.logLevel.size()); } if (!clientConfig.logPath.empty()) { out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); - strcpy(out_clientConfig->logPath, clientConfig.logPath.data()); + sf_memcpy(out_clientConfig->logPath, MAX_PATH, + clientConfig.logPath.data(), clientConfig.logPath.size()); } } From 16fd3bf18afb6bee56ae5e0b6d0140d042e18ffc Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 7 Oct 2024 11:38:34 -0700 Subject: [PATCH 39/74] Remove the use of boost algorithm string for compare --- cpp/lib/ClientConfigParser.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 3979894030..b3c5549d2d 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -14,7 +14,6 @@ #undef snprintf #include -#include #ifndef _WIN32 #include @@ -251,8 +250,15 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) snowflake_cJSON_ArrayForEach(entry, jsonConfig) { std::string key = entry->string; - boost::algorithm::to_lower(key); - if (KnownCommonEntries.find(key) == KnownCommonEntries.end()) + bool found = false; + for (std::string knownEntry : KnownCommonEntries) + { + if (sf_strncasecmp(key.c_str(), knownEntry.c_str(), knownEntry.length()) == 0) + { + found = true; + } + } + if (!found) { std::string warnMsg = "Unknown configuration entry: " + key + " with value:" + entry->valuestring; From 2a80a5ef3974842b520ef53a9aa2de5adb1ba391 Mon Sep 17 00:00:00 2001 From: norrislee Date: Tue, 8 Oct 2024 16:40:44 -0700 Subject: [PATCH 40/74] Fix test issue --- tests/test_unit_logger.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index a87183185f..b6085ad58f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -7,11 +7,7 @@ #include #include "memory.h" -#ifdef _WIN32 -inline int access(const char *pathname, int mode) { - return _access(pathname, mode); -} -#else +#ifndef _WIN32 #include #endif @@ -31,6 +27,7 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } +#ifndef _WIN32 /** * Tests log settings from client config file */ @@ -60,11 +57,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -72,7 +69,6 @@ void test_client_config_log(void **unused) { remove(LOG_PATH); } -#ifndef _WIN32 /** * Tests timing of log file creation */ From 47334d0770c67f30d7b28f5a0fa96491bff6dcd9 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 09:34:17 -0700 Subject: [PATCH 41/74] Fix build issue with test --- tests/test_unit_logger.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index b6085ad58f..b79f90b384 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -185,8 +185,8 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), - cmocka_unit_test(test_client_config_log), #ifndef _WIN32 + cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), #endif From 3e80253d6cc8d7d928e2ae677f21e230a7bcea3c Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 16:56:41 -0700 Subject: [PATCH 42/74] Change ifstream to FILE due to 32bit debug read issue --- cpp/lib/ClientConfigParser.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index b3c5549d2d..d71104c642 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -157,11 +157,11 @@ void ClientConfigParser::parseConfigFile( const std::string& in_filePath, ClientConfig& out_clientConfig) { - std::ifstream configFile; cJSON* jsonConfig; + FILE* configFile; try { - configFile.open(in_filePath); + configFile = fopen(in_filePath.c_str(), "r"); if (!configFile) { CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", @@ -173,9 +173,15 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - std::stringstream configJSON; - configJSON << configFile.rdbuf(); - jsonConfig = snowflake_cJSON_Parse(configJSON.str().c_str()); + fseek(configFile, 0, SEEK_END); + long length = ftell(configFile); + fseek(configFile, 0, SEEK_SET); + char* buffer = (char*)malloc(length); + if (buffer) + { + fread(buffer, 1, length, configFile); + } + jsonConfig = snowflake_cJSON_Parse(buffer); const char* error_ptr = snowflake_cJSON_GetErrorPtr(); if (error_ptr) { @@ -190,7 +196,7 @@ void ClientConfigParser::parseConfigFile( } catch (std::exception& ex) { - configFile.close(); + fclose(configFile); throw; } @@ -206,7 +212,7 @@ void ClientConfigParser::parseConfigFile( { out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); } - configFile.close(); + fclose(configFile); } //////////////////////////////////////////////////////////////////////////////// From 0d642b2f4edea9cc3b18dfd3d5c3554adb4d956b Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 18:09:09 -0700 Subject: [PATCH 43/74] Fix linux build issue --- cpp/lib/ClientConfigParser.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index d71104c642..2ccf1e170f 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -179,7 +179,15 @@ void ClientConfigParser::parseConfigFile( char* buffer = (char*)malloc(length); if (buffer) { - fread(buffer, 1, length, configFile); + size_t result = fread(buffer, 1, length, configFile); + if (result > 0) + { + CXX_LOG_ERROR( + "sf", + "ClientConfigParser", + "parseConfigFile", + "Error in reading file: %s", in_filePath.c_str()); + } } jsonConfig = snowflake_cJSON_Parse(buffer); const char* error_ptr = snowflake_cJSON_GetErrorPtr(); From c89424abaf53b29039677b04513a846f65cdb2f0 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 9 Oct 2024 20:55:58 -0700 Subject: [PATCH 44/74] Fix test issues --- cpp/lib/ClientConfigParser.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 2ccf1e170f..3b3e15e140 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -90,7 +90,7 @@ void ClientConfigParser::loadClientConfig( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(binaryDirFilePath)) + if (checkFileExists(binaryDirFilePath)) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -112,7 +112,7 @@ void ClientConfigParser::loadClientConfig( } } std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -123,7 +123,7 @@ void ClientConfigParser::loadClientConfig( if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::exists(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -161,7 +161,7 @@ void ClientConfigParser::parseConfigFile( FILE* configFile; try { - configFile = fopen(in_filePath.c_str(), "r"); + configFile = sf_fopen(&configFile, in_filePath.c_str(), "r"); if (!configFile) { CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", @@ -175,12 +175,12 @@ void ClientConfigParser::parseConfigFile( #endif fseek(configFile, 0, SEEK_END); long length = ftell(configFile); - fseek(configFile, 0, SEEK_SET); + rewind(configFile); char* buffer = (char*)malloc(length); if (buffer) { size_t result = fread(buffer, 1, length, configFile); - if (result > 0) + if (result != length) { CXX_LOG_ERROR( "sf", From a742893c066d5ff504df6bf526a1a0bc890ff703 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 11 Oct 2024 09:29:29 -0700 Subject: [PATCH 45/74] Rename ifndef for headers to be more consistent --- include/snowflake/ClientConfigParser.hpp | 6 +++--- lib/client_config_parser.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 3b12d970b5..f61ccaa8d9 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -2,8 +2,8 @@ * Copyright (c) 2024 Snowflake Computing */ -#ifndef PROJECT_CLIENTCONFIG_HPP -#define PROJECT_CLIENTCONFIG_HPP +#ifndef SNOWFLAKE_CONFIGPARSER_HPP +#define SNOWFLAKE_CONFIGPARSER_HPP #include @@ -101,4 +101,4 @@ namespace Client } // namespace Client } // namespace Snowflake -#endif //PROJECT_CLIENTCONFIG_HPP +#endif //SNOWFLAKE_CONFIGPARSER_HPP diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h index 448ce1794b..2d5b584802 100644 --- a/lib/client_config_parser.h +++ b/lib/client_config_parser.h @@ -2,8 +2,8 @@ * Copyright (c) 2024 Snowflake Computing */ -#ifndef SNOWFLAKE_CLIENTCONFIG_H -#define SNOWFLAKE_CLIENTCONFIG_H +#ifndef SNOWFLAKE_CONFIGPARSER_H +#define SNOWFLAKE_CONFIGPARSER_H #include "snowflake/client.h" #include "cJSON.h" @@ -41,4 +41,4 @@ extern "C" { } // extern "C" #endif -#endif //SNOWFLAKE_CLIENTCONFIG_H +#endif //SNOWFLAKE_CONFIGPARSER_H From e16a67cb0929125ad39bd23047f8ba84a80996c7 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 15:57:45 -0700 Subject: [PATCH 46/74] Replace boost with std::filesystem, fix possible NULL issues, add tests --- CMakeLists.txt | 4 +- cpp/lib/ClientConfigParser.cpp | 116 +++++++++++------------ include/snowflake/ClientConfigParser.hpp | 22 ++--- lib/client.c | 19 +--- lib/client_config_parser.h | 44 --------- tests/test_unit_logger.c | 93 ++++++++++++++++-- 6 files changed, 155 insertions(+), 143 deletions(-) delete mode 100644 lib/client_config_parser.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3515e21939..45eb141161 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,7 @@ set(SOURCE_FILES include/snowflake/logger.h include/snowflake/version.h include/snowflake/platform.h + include/snowflake/client_config_parser.h lib/client.c lib/constants.h lib/cJSON.h @@ -245,8 +246,7 @@ set(SOURCE_FILES_CPP_WRAPPER lib/result_set.h lib/query_context_cache.h lib/curl_desc_pool.h - lib/authenticator.h - lib/client_config_parser.h) + lib/authenticator.h) if (UNIX) if (LINUX) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 3b3e15e140..a999bf3c1f 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -4,22 +4,21 @@ #include "snowflake/ClientConfigParser.hpp" #include "snowflake/Exceptions.hpp" +#include "snowflake/client_config_parser.h" #include "../logger/SFLogger.hpp" -#include "client_config_parser.h" #include "memory.h" +#include #include #include #include -#undef snprintf -#include - #ifndef _WIN32 #include #endif using namespace Snowflake::Client; +using namespace std::filesystem; namespace { @@ -30,24 +29,16 @@ namespace } //////////////////////////////////////////////////////////////////////////////// -void load_client_config( +sf_bool load_client_config( const char* in_configFilePath, client_config* out_clientConfig) { - ClientConfigParser configParser; - ClientConfig clientConfig; - configParser.loadClientConfig(in_configFilePath, clientConfig); - if (!clientConfig.logLevel.empty()) - { - out_clientConfig->logLevel = (char*)SF_CALLOC(1, sizeof(clientConfig.logLevel)); - sf_memcpy(out_clientConfig->logLevel, sizeof(out_clientConfig->logLevel), - clientConfig.logLevel.data(), clientConfig.logLevel.size()); - } - if (!clientConfig.logPath.empty()) - { - out_clientConfig->logPath = (char*)SF_CALLOC(1, sizeof(clientConfig.logPath)); - sf_memcpy(out_clientConfig->logPath, MAX_PATH, - clientConfig.logPath.data(), clientConfig.logPath.size()); + try { + ClientConfigParser configParser; + configParser.loadClientConfig(in_configFilePath, *out_clientConfig); + } catch (std::exception e) { + sf_fprintf(stderr, e.what()); + return false; } } @@ -67,7 +58,19 @@ ClientConfigParser::~ClientConfigParser() //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::loadClientConfig( const std::string& in_configFilePath, - ClientConfig& out_clientConfig) + client_config& out_clientConfig) +{ + std::string derivedConfigPath = resolveClientConfigPath(in_configFilePath); + + if (!derivedConfigPath.empty()) + { + parseConfigFile(derivedConfigPath, out_clientConfig); + } +} + +//////////////////////////////////////////////////////////////////////////////// +std::string ClientConfigParser::resolveClientConfigPath( + const std::string& in_configFilePath) { char envbuf[MAX_PATH + 1]; std::string derivedConfigPath = ""; @@ -90,7 +93,7 @@ void ClientConfigParser::loadClientConfig( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(binaryDirFilePath)) + if (is_regular_file(binaryDirFilePath)) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -100,19 +103,22 @@ void ClientConfigParser::loadClientConfig( { #if defined(WIN32) || defined(_WIN64) // 4. Try user home dir - std::string homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); - if (homeDir.empty()) + std::string homeDirFilePath; + char* homeDir; + if ((homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf))) && strlen(homeDir) != 0) { + homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + } else { // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH - std::string homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); - std::string homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); - if (!homeDriveEnv.empty() && !homePathEnv.empty()) + char* homeDriveEnv; + char* homePathEnv; + if ((homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf))) && (strlen(homeDriveEnv) != 0) && + (homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf))) && (strlen(homePathEnv) != 0)) { - homeDir = homeDriveEnv + homePathEnv; + homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; } } - std::string homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(homeDirFilePath)) + if (is_regular_file(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -120,10 +126,11 @@ void ClientConfigParser::loadClientConfig( } #else // 4. Try user home dir - if (const char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) + char* homeDir; + if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(homeDirFilePath)) + if (is_regular_file(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -133,29 +140,14 @@ void ClientConfigParser::loadClientConfig( #endif } } - if (!derivedConfigPath.empty()) - { - parseConfigFile(derivedConfigPath, out_clientConfig); - } + return derivedConfigPath; } // Private ===================================================================== -//////////////////////////////////////////////////////////////////////////////// -bool ClientConfigParser::checkFileExists(const std::string& in_filePath) -{ - FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); - if (file != nullptr) - { - fclose(file); - return true; - } - return false; -} - //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::parseConfigFile( const std::string& in_filePath, - ClientConfig& out_clientConfig) + client_config& out_clientConfig) { cJSON* jsonConfig; FILE* configFile; @@ -173,14 +165,12 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - fseek(configFile, 0, SEEK_END); - long length = ftell(configFile); - rewind(configFile); - char* buffer = (char*)malloc(length); + const int fileSize = file_size(in_filePath); + char* buffer = (char*)malloc(fileSize); if (buffer) { - size_t result = fread(buffer, 1, length, configFile); - if (result != length) + size_t result = fread(buffer, 1, fileSize, configFile); + if (result != fileSize) { CXX_LOG_ERROR( "sf", @@ -204,7 +194,10 @@ void ClientConfigParser::parseConfigFile( } catch (std::exception& ex) { - fclose(configFile); + if (configFile) + { + fclose(configFile); + } throw; } @@ -220,16 +213,19 @@ void ClientConfigParser::parseConfigFile( { out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); } - fclose(configFile); + if (configFile) + { + fclose(configFile); + } } //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) { - boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); - boost::filesystem::perms permissions = fileStatus.permissions(); - if (permissions & boost::filesystem::group_write || - permissions & boost::filesystem::others_write) + file_status fileStatus = status(in_filePath); + perms permissions = fileStatus.permissions(); + if ((perms::group_write == (permissions & perms::group_write)) || + (perms::others_write == (permissions & perms::others_write))) { CXX_LOG_ERROR( "sf", diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index f61ccaa8d9..263ab0d45c 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -6,20 +6,12 @@ #define SNOWFLAKE_CONFIGPARSER_HPP #include +#include "client_config_parser.h" namespace Snowflake { namespace Client { - struct ClientConfig - { - // The log level - std::string logLevel = ""; - - // The log path - std::string logPath = ""; - }; - struct ClientConfigException : public std::exception { ClientConfigException(const std::string& message) : message_(message) {} @@ -57,16 +49,18 @@ namespace Client */ void loadClientConfig( const std::string& in_configFilePath, - ClientConfig& out_clientConfig); + client_config& out_clientConfig); // Private ================================================================= private: /** - * @brief Check if the file exists. + * @brief Resolve the client config path. * - * @param in_filePath The file path to check. + * @param in_configFilePath The config file path passed in by the user. + * + * @param The client config path */ - bool checkFileExists(const std::string& in_filePath); + std::string resolveClientConfigPath(const std::string& in_configFilePath); /** * @brief Parse JSON string. @@ -76,7 +70,7 @@ namespace Client */ void parseConfigFile( const std::string& in_filePath, - ClientConfig& out_clientConfig); + client_config& out_clientConfig); /** * @ brief Check if other have permission to modify file diff --git a/lib/client.c b/lib/client.c index 853c23f45b..b0a87c024d 100644 --- a/lib/client.c +++ b/lib/client.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "constants.h" #include "client_int.h" #include "connection.h" @@ -19,7 +20,6 @@ #include "chunk_downloader.h" #include "authenticator.h" #include "query_context_cache.h" -#include "client_config_parser.h" #ifdef _WIN32 #include @@ -38,7 +38,6 @@ sf_bool SF_OCSP_CHECK; char *SF_HEADER_USER_AGENT = NULL; static char *CLIENT_CONFIG_FILE = NULL; -static char* LOG_LEVEL; static char *LOG_PATH = NULL; static FILE *LOG_FP = NULL; @@ -359,12 +358,10 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "%s/snowflake_%s.txt", sf_log_path, (char *) time_str); - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, sf_log_path); } else { LOG_PATH = (char *) SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, "logs/snowflake_%s.txt", (char *) time_str); - snowflake_global_set_attribute(SF_GLOBAL_LOG_PATH, "logs/"); } if (LOG_PATH != NULL) { // Set the log path only, the log file will be created when actual log output is needed. @@ -675,12 +672,6 @@ snowflake_global_set_attribute(SF_GLOBAL_ATTRIBUTE type, const void *value) { case SF_GLOBAL_CLIENT_CONFIG_FILE: alloc_buffer_and_copy(&CLIENT_CONFIG_FILE, value); break; - case SF_GLOBAL_LOG_LEVEL: - alloc_buffer_and_copy(&LOG_LEVEL, value); - break; - case SF_GLOBAL_LOG_PATH: - alloc_buffer_and_copy(&LOG_PATH, value); - break; default: break; } @@ -719,13 +710,13 @@ snowflake_global_get_attribute(SF_GLOBAL_ATTRIBUTE type, void *value, size_t siz } break; case SF_GLOBAL_LOG_LEVEL: - if (LOG_LEVEL) { - if (strlen(LOG_LEVEL) > size - 1) { + { + if (strlen(log_from_level_to_str(log_get_level())) > size - 1) { return SF_STATUS_ERROR_BUFFER_TOO_SMALL; } - sf_strncpy(value, size, LOG_LEVEL, size); + sf_strncpy(value, size, log_from_level_to_str(log_get_level()), size); + break; } - break; case SF_GLOBAL_LOG_PATH: if (LOG_PATH) { if (strlen(LOG_PATH) > size - 1) { diff --git a/lib/client_config_parser.h b/lib/client_config_parser.h deleted file mode 100644 index 2d5b584802..0000000000 --- a/lib/client_config_parser.h +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) 2024 Snowflake Computing - */ - -#ifndef SNOWFLAKE_CONFIGPARSER_H -#define SNOWFLAKE_CONFIGPARSER_H - -#include "snowflake/client.h" -#include "cJSON.h" - -#ifdef __cplusplus -extern "C" { -#endif - - typedef struct client_config - { - // The log level - char *logLevel; - - // The log path - char *logPath; - } client_config; - - /** - * Construct ClientConfig from client config file passed by user. This method searches the - * config file in following order: 1. configFilePath param which is read from connection URL or - * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to - * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the - * driver directory from where the driver gets loaded. 4. Searches for default config file - * name(sf_client_config.json) under user home directory 5. Searches for default config file - * name(sf_client_config.json) under tmp directory - * - * @param in_configFilePath The config file path passed in by the user. - * @param out_clientConfig The ClientConfig object to be filled. - */ - void load_client_config( - const char* in_configFilePath, - client_config* out_clientConfig); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif //SNOWFLAKE_CONFIGPARSER_H diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index b79f90b384..a4fe7823d2 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -4,13 +4,18 @@ #include "utils/test_setup.h" -#include +#include #include "memory.h" -#ifndef _WIN32 +#ifdef _WIN32 +inline int access(const char* pathname, int mode) { + return _access(pathname, mode); +} +#else #include #endif + /** * Tests converting a string representation of log level to the log level enum */ @@ -27,7 +32,38 @@ void test_log_str_to_level(void **unused) { assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); } -#ifndef _WIN32 +/** + * Tests log settings with invalid client config filepath + */ +void test_invalid_client_config_path(void** unused) { + char configFilePath[] = "fakePath.json"; + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); +} + +/** + * Tests log settings from client config file with invalid json + */ +void test_client_config_log_invalid_json(void** unused) { + char clientConfigJSON[] = "{{{\"invalid json\"}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); + + // Cleanup + remove(configFilePath); +} + /** * Tests log settings from client config file */ @@ -57,11 +93,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, F_OK), 0); + assert_int_not_equal(access(LOG_PATH, 0), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, F_OK), 0); + assert_int_equal(access(LOG_PATH, 0), 0); log_close(); // Cleanup @@ -69,6 +105,41 @@ void test_client_config_log(void **unused) { remove(LOG_PATH); } +/** + * Tests log settings from client config file via global init + */ +void test_client_config_log_init(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); + snowflake_global_init("./logs", SF_LOG_TRACE, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, 0), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, 0), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + + /** * Tests timing of log file creation */ @@ -77,7 +148,7 @@ void test_log_creation(void **unused) { // ensure the log file doesn't exist at the beginning remove(logname); - assert_int_not_equal(access(logname, F_OK), 0); + assert_int_not_equal(access(logname, 0), 0); log_set_lock(NULL); log_set_level(SF_LOG_WARN); @@ -86,16 +157,17 @@ void test_log_creation(void **unused) { // info log won't trigger the log file creation since log level is set to warning log_info("dummy info log"); - assert_int_not_equal(access(logname, F_OK), 0); + assert_int_not_equal(access(logname, 0), 0); // warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(logname, F_OK), 0); + assert_int_equal(access(logname, 0), 0); log_close(); remove(logname); } +#ifndef _WIN32 /** * Tests masking secret information in log */ @@ -185,8 +257,11 @@ void test_mask_secret_log(void **unused) { int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), -#ifndef _WIN32 + cmocka_unit_test(test_invalid_client_config_path), + cmocka_unit_test(test_client_config_log_invalid_json), cmocka_unit_test(test_client_config_log), + cmocka_unit_test(test_client_config_log_init), +#ifndef _WIN32 cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), #endif From 05331fb28be1546ec073e84ac638af3e31691237 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 15:58:23 -0700 Subject: [PATCH 47/74] Add client_config_parser to include dir --- include/snowflake/client_config_parser.h | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 include/snowflake/client_config_parser.h diff --git a/include/snowflake/client_config_parser.h b/include/snowflake/client_config_parser.h new file mode 100644 index 0000000000..86ae0683b6 --- /dev/null +++ b/include/snowflake/client_config_parser.h @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2024 Snowflake Computing + */ + +#ifndef SNOWFLAKE_CONFIGPARSER_H +#define SNOWFLAKE_CONFIGPARSER_H + +#include "snowflake/client.h" +#include "cJSON.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct client_config + { + // The log level + char *logLevel; + + // The log path + char *logPath; + } client_config; + + /** + * Construct client_config from client config file passed by user. This method searches the + * config file in following order: 1. configFilePath param which is read from connection URL or + * connection property. 2. Environment variable: SF_CLIENT_CONFIG_FILE containing full path to + * sf_client_config file. 3. Searches for default config file name(sf_client_config.json under the + * driver directory from where the driver gets loaded. 4. Searches for default config file + * name(sf_client_config.json) under user home directory 5. Searches for default config file + * name(sf_client_config.json) under tmp directory + * + * @param in_configFilePath The config file path passed in by the user. + * @param out_clientConfig The client_config object to be filled. + * + * @return true if successful + */ + sf_bool load_client_config( + const char* in_configFilePath, + client_config* out_clientConfig); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif //SNOWFLAKE_CONFIGPARSER_H From f370f9b26d0d95ad4226f4361babdb9f5d682d5b Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 16:55:25 -0700 Subject: [PATCH 48/74] Fix compilation error --- cpp/lib/ClientConfigParser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index a999bf3c1f..d41b450a8a 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -127,7 +127,7 @@ std::string ClientConfigParser::resolveClientConfigPath( #else // 4. Try user home dir char* homeDir; - if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0) + if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; if (is_regular_file(homeDirFilePath)) From d0cbc0cf82ae39c5abcb171f57e570f126d18571 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 17:24:31 -0700 Subject: [PATCH 49/74] Fix return value error --- cpp/lib/ClientConfigParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index d41b450a8a..13c50a7c66 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -40,6 +40,7 @@ sf_bool load_client_config( sf_fprintf(stderr, e.what()); return false; } + return true; } // Public ====================================================================== From e0204291e328db6a601ef45ba0b8bf76d67c7f6b Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 18:14:34 -0700 Subject: [PATCH 50/74] Use experimental filesystem for non-windows --- cpp/lib/ClientConfigParser.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 13c50a7c66..7a1e5083b0 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -8,17 +8,21 @@ #include "../logger/SFLogger.hpp" #include "memory.h" -#include #include #include #include #ifndef _WIN32 #include +#include +using namespace std::experimental::filesystem; +#else +#include +using namespace std::filesystem; #endif using namespace Snowflake::Client; -using namespace std::filesystem; + namespace { From d35186638bd07e75797a7c1f51f6f0a52d882c74 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 18:49:51 -0700 Subject: [PATCH 51/74] Add stdc++fs to link libraries for unix builds --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45eb141161..c3ad92c87b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -515,11 +515,11 @@ endif() if (LINUX) # Linux - target_link_libraries(snowflakeclient rt dl z) + target_link_libraries(snowflakeclient rt dl z stdc++fs) endif () if (APPLE) # OSX. no librt is required. - target_link_libraries(snowflakeclient dl z) + target_link_libraries(snowflakeclient dl z stdc++fs) endif () if (WIN32) # Windows From 5cc39aee3e3604a276735fe36e69548a5c2fb71b Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 16 Oct 2024 19:41:56 -0700 Subject: [PATCH 52/74] Fix macos filesystem error --- cpp/lib/ClientConfigParser.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 7a1e5083b0..882d8f0580 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -14,11 +14,14 @@ #ifndef _WIN32 #include -#include -using namespace std::experimental::filesystem; -#else +#endif + +#ifdef _WIN32 #include using namespace std::filesystem; +#elif defined(__linux__) +#include +using namespace std::experimental::filesystem; #endif using namespace Snowflake::Client; From 75eb36602b528e73fc10d7dfb00ebac90d2e3bcf Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 17 Oct 2024 13:35:35 -0700 Subject: [PATCH 53/74] Revert from using std::filesystem, add back check file exists due to boost 32-bit windows debug issues --- CMakeLists.txt | 4 +-- cpp/lib/ClientConfigParser.cpp | 40 ++++++++++++++---------- include/snowflake/ClientConfigParser.hpp | 7 +++++ 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3ad92c87b..45eb141161 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -515,11 +515,11 @@ endif() if (LINUX) # Linux - target_link_libraries(snowflakeclient rt dl z stdc++fs) + target_link_libraries(snowflakeclient rt dl z) endif () if (APPLE) # OSX. no librt is required. - target_link_libraries(snowflakeclient dl z stdc++fs) + target_link_libraries(snowflakeclient dl z) endif () if (WIN32) # Windows diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 882d8f0580..4c5c569c9b 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -12,18 +12,12 @@ #include #include +#include + #ifndef _WIN32 #include #endif -#ifdef _WIN32 -#include -using namespace std::filesystem; -#elif defined(__linux__) -#include -using namespace std::experimental::filesystem; -#endif - using namespace Snowflake::Client; @@ -101,7 +95,7 @@ std::string ClientConfigParser::resolveClientConfigPath( // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (is_regular_file(binaryDirFilePath)) + if (checkFileExists(binaryDirFilePath.c_str())) { derivedConfigPath = binaryDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -126,7 +120,7 @@ std::string ClientConfigParser::resolveClientConfigPath( homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; } } - if (is_regular_file(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -138,7 +132,7 @@ std::string ClientConfigParser::resolveClientConfigPath( if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) { std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (is_regular_file(homeDirFilePath)) + if (checkFileExists(homeDirFilePath)) { derivedConfigPath = homeDirFilePath; CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", @@ -152,6 +146,18 @@ std::string ClientConfigParser::resolveClientConfigPath( } // Private ===================================================================== +//////////////////////////////////////////////////////////////////////////////// +bool ClientConfigParser::checkFileExists(const std::string& in_filePath) +{ + FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); + if (file != nullptr) + { + fclose(file); + return true; + } + return false; +} + //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::parseConfigFile( const std::string& in_filePath, @@ -173,7 +179,9 @@ void ClientConfigParser::parseConfigFile( #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - const int fileSize = file_size(in_filePath); + fseek(configFile, 0, SEEK_END); + long fileSize = ftell(configFile); + fseek(configFile, 0, SEEK_SET); char* buffer = (char*)malloc(fileSize); if (buffer) { @@ -230,10 +238,10 @@ void ClientConfigParser::parseConfigFile( //////////////////////////////////////////////////////////////////////////////// void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) { - file_status fileStatus = status(in_filePath); - perms permissions = fileStatus.permissions(); - if ((perms::group_write == (permissions & perms::group_write)) || - (perms::others_write == (permissions & perms::others_write))) + boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); + boost::filesystem::perms permissions = fileStatus.permissions(); + if (permissions & boost::filesystem::group_write || + permissions & boost::filesystem::others_write) { CXX_LOG_ERROR( "sf", diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 263ab0d45c..87a2cd2e41 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -53,6 +53,13 @@ namespace Client // Private ================================================================= private: + /** + * @brief Check if the file exists. + * + * @param in_filePath The file path to check. + */ + bool checkFileExists(const std::string& in_filePath); + /** * @brief Resolve the client config path. * From 86517887a6cb8f631b76e44fa8d28cb7b669d17c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Szczerbi=C5=84ski?= Date: Mon, 28 Oct 2024 11:44:45 +0100 Subject: [PATCH 54/74] SNOW-1744756 add picojson to libsnowflakeclient (#755) --- ci/build/build.sh | 1 + ci/build_win.bat | 3 + deps/picojson/.gitignore | 4 + deps/picojson/Changes | 25 + deps/picojson/LICENSE | 25 + deps/picojson/Makefile | 26 + deps/picojson/README.mkdn | 195 ++++++ deps/picojson/picojson.h | 1010 +++++++++++++++++++++++++++++ deps/picojson/picotest/picotest.c | 99 +++ deps/picojson/picotest/picotest.h | 39 ++ deps/picojson/test.cc | 312 +++++++++ scripts/build_picojson.bat | 46 ++ scripts/build_picojson.sh | 18 + 13 files changed, 1803 insertions(+) create mode 100644 deps/picojson/.gitignore create mode 100644 deps/picojson/Changes create mode 100644 deps/picojson/LICENSE create mode 100644 deps/picojson/Makefile create mode 100644 deps/picojson/README.mkdn create mode 100644 deps/picojson/picojson.h create mode 100644 deps/picojson/picotest/picotest.c create mode 100644 deps/picojson/picotest/picotest.h create mode 100644 deps/picojson/test.cc create mode 100644 scripts/build_picojson.bat create mode 100755 scripts/build_picojson.sh diff --git a/ci/build/build.sh b/ci/build/build.sh index 7fb31c75a2..256be4a3e9 100755 --- a/ci/build/build.sh +++ b/ci/build/build.sh @@ -82,6 +82,7 @@ download_build_component aws "$SCRIPTS_DIR/build_awssdk.sh" "$target" download_build_component azure "$SCRIPTS_DIR/build_azuresdk.sh" "$target" download_build_component cmocka "$SCRIPTS_DIR/build_cmocka.sh" "$target" download_build_component arrow "$SCRIPTS_DIR/build_arrow.sh" "$target" +download_build_component picojson "$SCRIPTS_DIR/build_picojson.sh" "$target" # very tight diskspace limit on github runners, clear deps folder with all .o files if [[ -n "$GITHUB_ACTIONS" ]]; then diff --git a/ci/build_win.bat b/ci/build_win.bat index 0576e257d8..58fb19422e 100644 --- a/ci/build_win.bat +++ b/ci/build_win.bat @@ -26,6 +26,7 @@ set aws_build_script="%scriptdir%..\scripts\build_awssdk.bat" set azure_build_script="%scriptdir%..\scripts\build_azuresdk.bat" set cmocka_build_script="%scriptdir%..\scripts\build_cmocka.bat" set arrow_build_script="%scriptdir%..\scripts\build_arrow.bat" +set picojson_build_script="%scriptdir%..\scripts\build_picojson.bat" set libsnowflakeclient_build_script="%scriptdir%..\scripts\build_libsnowflakeclient.bat" set upload_artifact_script="%scriptdir%container\upload_artifact.bat" @@ -59,6 +60,8 @@ goto :EOF if %ERRORLEVEL% NEQ 0 goto :error call :download_build_component arrow "%arrow_build_script%" "%dynamic_runtime%" if %ERRORLEVEL% NEQ 0 goto :error + call :download_build_component picojson "%picojson_build_script%" "%dynamic_runtime%" + if %ERRORLEVEL% NEQ 0 goto :error if defined GITHUB_ACTIONS ( rd /S /Q %scriptdir%\..\deps ) diff --git a/deps/picojson/.gitignore b/deps/picojson/.gitignore new file mode 100644 index 0000000000..815b59fad3 --- /dev/null +++ b/deps/picojson/.gitignore @@ -0,0 +1,4 @@ +*~ +a.out +test-core +test-core-int64 diff --git a/deps/picojson/Changes b/deps/picojson/Changes new file mode 100644 index 0000000000..d6017ddf9a --- /dev/null +++ b/deps/picojson/Changes @@ -0,0 +1,25 @@ +Revision history for picojson + +1.3.0 2015-02-25 13:05:00+0900 + - `make check` is now synonym of `make test` (#62) + - operator= is now safe when part of LHS is being assigned, as well as exception-safe (#66) + +1.2.1 2014-12-16 15:33:00+0900 + - bundle the contents of `picotest/` (#61) + +1.2.0 2014-12-15 16:20:00+0900 + - `make install` to install picojson.h (#58) + - two-argument `picojson::parse()` for ease-of-use (#57) + +1.1.1 2014-06-25 10:35:00+0900 + - tweaks to suppress compiler errors / warning (#38 #39) + - clarify the licenses of the files in exmaple/ (#42) + +1.1 2014-06-16 12:57:00+0900 + - added experimental support for int64 type (#34) + - by default, throw std::runtime_error instead of using assert for runtime errors (#33) + - refine compatibility regarding the use of isinf/isnan (#29, #36) + - remove `.get()` (#35) + +1.0 2014-06-05 12:54:00+0900 + - initial release with a version number diff --git a/deps/picojson/LICENSE b/deps/picojson/LICENSE new file mode 100644 index 0000000000..72f3553911 --- /dev/null +++ b/deps/picojson/LICENSE @@ -0,0 +1,25 @@ +Copyright 2009-2010 Cybozu Labs, Inc. +Copyright 2011-2014 Kazuho Oku +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/picojson/Makefile b/deps/picojson/Makefile new file mode 100644 index 0000000000..40f6450b89 --- /dev/null +++ b/deps/picojson/Makefile @@ -0,0 +1,26 @@ +prefix=/usr/local +includedir=$(prefix)/include + +check: test + +test: test-core test-core-int64 + ./test-core + ./test-core-int64 + +test-core: test.cc picotest/picotest.c picotest/picotest.h + $(CXX) -Wall test.cc picotest/picotest.c -o $@ + +test-core-int64: test.cc picotest/picotest.c picotest/picotest.h + $(CXX) -Wall -DPICOJSON_USE_INT64 test.cc picotest/picotest.c -o $@ + +clean: + rm -f test-core test-core-int64 + +install: + install -d $(DESTDIR)$(includedir) + install -p -m 0644 picojson.h $(DESTDIR)$(includedir) + +uninstall: + rm -f $(DESTDIR)$(includedir)/picojson.h + +.PHONY: test check clean install uninstall diff --git a/deps/picojson/README.mkdn b/deps/picojson/README.mkdn new file mode 100644 index 0000000000..7db7fd6a8f --- /dev/null +++ b/deps/picojson/README.mkdn @@ -0,0 +1,195 @@ +# PicoJSON - a C++ JSON parser / serializer + +Copyright © 2009-2010 Cybozu Labs, Inc. +Copyright © 2011-2015 Kazuho Oku + +Licensed under [2-clause BSD license](http://opensource.org/licenses/BSD-2-Clause) + +## Version + +1.3.0 [![Build Status](https://travis-ci.org/kazuho/picojson.svg?branch=rel/1.3.0)](https://travis-ci.org/kazuho/picojson) + +## Introduction + +PicoJSON is a tiny JSON parser / serializer for C++ with following properties: + +- header-file only +- no external dependencies (only uses standard C++ libraries) +- STL-frendly (arrays are represented by using std::vector, objects are std::map) +- provides both pull interface and streaming (event-based) interface + +## Reading JSON using the pull interface + +There are several ways to use the pull (DOM-like) interface of picojson. + +The easiest way is to use the two-argument `parse` function. + +``` +std::string json = "[ \"hello JSON\" ]"; +picojson::value v; +std::string err = picojson::parse(v, json); +if (! err.empty()) { + std:cerr << err << std::endl; +} +``` + +Four-argument `parse` function accepts a pair of iterators, and returns the end position of the input. + +``` +const char* json = "{\"a\":1}"; +picojson::value v; +std::string err; +const char* json_end = picojson::parse(v, json, json + strlen(json), &err); +if (! err.empty()) { + std::cerr << err << std::endl; +} +``` + +``` +std::istream_iterator input(std::cin); +picojson::value v; +std::string err; +input = picojson::parse(v, input, std::istream_iterator(), &err); +if (! err.empty()) { + std::cerr << err << std::endl; +} +``` + +It is also possible to use the `>>` operator to parse the input, however this interface is not thread-safe. + +``` +picosjon::value v; +std::cin >> v; +std::string err = picojson::get_last_error(); +``` + +## Accessing the values + +Values of a JSON object is represented as instances of picojson::value class. + +
+namespace picojson {
+
+  class value {
+    ...
+
+  public:
+
+    typedef std::vector<value> array;
+    typedef std::map<std::string, value> object;
+
+    value();                               // create a null object
+    explicit value(bool b);                // create a boolean object
+    explicit value(double n);              // create a number object
+    explicit value(const std::string& s);  // create a string object
+    explicit value(const array& a);        // create an array object
+    explicit value(const object& o);       // create an "object"
+
+    bool is<picojson::null>() const;       // check if the object is "null"
+
+    bool is<bool>() const;                 // check if the object is a boolean
+    const bool& get<bool>() const;         // const accessor (usable only if the object is a boolean)
+    bool& get<bool>();                     // non-const accessor (usable only if the object is a boolean)
+
+    bool is<double>() const;               // check if the object is a number
+    const double& get<double>() const;     // const accessor (usable only if the object is a number)
+    double& get<double>();                 // non-const accessor (usable only if the object is a number)
+
+    bool is<std::string>() const;          // check if the object is a string
+    const std::string& get<std::string>() const;
+                                           // const accessor (usable only if the object is a string)
+    std::string& get<std::string>();       // non-const accessor (usable only if the object is a string)
+
+    bool is<array>() const;                // check if the object is an array
+    const array& get<array>() const;       // const accessor (usable only if the object is an array)
+    array& get<array>();                   // non-const accessor (usable only if the object is an array)
+
+    bool is<object>() const;               // check if the object is an "object"
+    const object& get<object>() const;     // const accessor (usable only if the object is an object)
+    object& get<object>();                 // non-const accessor (usable only if the object is an array)
+
+    bool evaluate_as_boolean() const;      // evaluates the object as a boolean
+
+    std::string serialize() const;         // returns the object in JSON representation
+    template void serialize(Iter os) const;
+                                           // serializes the object in JSON representation through an output iterator
+
+    std::string to_str() const;            // returns the object in string (for casual use)
+
+  };
+
+}
+
+ +The code below parses a JSON string and prints the contents of the object. + +
+picojson::value v;
+
+// parse the input
+std::cin >> v;
+std::string err = picojson::get_last_error();
+if (! err.empty()) {
+  std::cerr << err << std::endl;
+  exit(1);
+}
+
+// check if the type of the value is "object"
+if (! v.is<picojson::object>()) {
+  std::cerr << "JSON is not an object" << std::endl;
+  exit(2);
+}
+
+// obtain a const reference to the map, and print the contents
+const picojson::value::object& obj = v.get<picojson::object>();
+for (picojson::value::object::const_iterator i = obj.begin();
+     i != obj.end();
+     ++i) {
+  std::cout << i->first << ': ' << i->second.to_str() << std::endl;
+}
+
+ +Please note that the type check is mandatory; do not forget to check the type of the object by calling is<type>() before accessing the value by calling get<type>(). + +## Reading JSON using the streaming (event-driven) interface + +Please refer to the implementation of picojson::default_parse_context and picojson::null_parse_context. There is also an example (examples/streaming.cc) . + +## Serializing to JSON + +Instances of the picojson::value class can be serialized in three ways, to ostream, to std::string, or to an output iterator. + +
+picojson::value v;
+...
+std::cout << v;
+
+ +
+picojson::value v;
+...
+std::string json = v.serialize();
+
+ +
+picojson::value v;
+...
+v.serialize(std::ostream_iterator(std::cout));
+
+ +## Experimental support for int64_t + +Experimental suport for int64_t becomes available if the code is compiled with preprocessor macro `PICOJSON_USE_INT64`. + +Turning on the feature will cause following changes to picojson: +- new constructor `picojson::value(int64_t)` is defined +- `is()` and `get()` become available +- numerics in JSON within the bounds of int64_t and not using `.` nor `e`/`E` are considered as int64 type + - the values are also avaliable as `double`s as well (i.e. all values which are `.is() == true` are also `.is() == true`) +- int64 values are converted to double once `get()` is called + +Enabling the feature should not cause compatibility problem with code that do not use the feature. + +## Further reading + +Examples can be found in the examples directory, and on the [Wiki](https://github.com/kazuho/picojson/wiki). Please add your favorite examples to the Wiki. diff --git a/deps/picojson/picojson.h b/deps/picojson/picojson.h new file mode 100644 index 0000000000..48bb64e672 --- /dev/null +++ b/deps/picojson/picojson.h @@ -0,0 +1,1010 @@ +/* + * Copyright 2009-2010 Cybozu Labs, Inc. + * Copyright 2011-2014 Kazuho Oku + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef picojson_h +#define picojson_h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// for isnan/isinf +#if __cplusplus>=201103L +# include +#else +extern "C" { +# ifdef _MSC_VER +# include +# elif defined(__INTEL_COMPILER) +# include +# else +# include +# endif +} +#endif + +// experimental support for int64_t (see README.mkdn for detail) +#ifdef PICOJSON_USE_INT64 +# define __STDC_FORMAT_MACROS +# include +# include +#endif + +// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 +#ifndef PICOJSON_USE_LOCALE +# define PICOJSON_USE_LOCALE 1 +#endif +#if PICOJSON_USE_LOCALE +extern "C" { +# include +} +#endif + +#ifndef PICOJSON_ASSERT +# define PICOJSON_ASSERT(e) do { if (! (e)) throw std::runtime_error(#e); } while (0) +#endif + +#ifdef _MSC_VER + #define SNPRINTF _snprintf_s + #pragma warning(push) + #pragma warning(disable : 4244) // conversion from int to char + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4702) // unreachable code +#else + #define SNPRINTF snprintf +#endif + +namespace picojson { + + enum { + null_type, + boolean_type, + number_type, + string_type, + array_type, + object_type +#ifdef PICOJSON_USE_INT64 + , int64_type +#endif + }; + + enum { + INDENT_WIDTH = 2 + }; + + struct null {}; + + class value { + public: + typedef std::vector array; + typedef std::map object; + union _storage { + bool boolean_; + double number_; +#ifdef PICOJSON_USE_INT64 + int64_t int64_; +#endif + std::string* string_; + array* array_; + object* object_; + }; + protected: + int type_; + _storage u_; + public: + value(); + value(int type, bool); + explicit value(bool b); +#ifdef PICOJSON_USE_INT64 + explicit value(int64_t i); +#endif + explicit value(double n); + explicit value(const std::string& s); + explicit value(const array& a); + explicit value(const object& o); + explicit value(const char* s); + value(const char* s, size_t len); + ~value(); + value(const value& x); + value& operator=(const value& x); + void swap(value& x); + template bool is() const; + template const T& get() const; + template T& get(); + bool evaluate_as_boolean() const; + const value& get(size_t idx) const; + const value& get(const std::string& key) const; + value& get(size_t idx); + value& get(const std::string& key); + + bool contains(size_t idx) const; + bool contains(const std::string& key) const; + std::string to_str() const; + template void serialize(Iter os, bool prettify = false) const; + std::string serialize(bool prettify = false) const; + private: + template value(const T*); // intentionally defined to block implicit conversion of pointer to bool + template static void _indent(Iter os, int indent); + template void _serialize(Iter os, int indent) const; + std::string _serialize(int indent) const; + }; + + typedef value::array array; + typedef value::object object; + + inline value::value() : type_(null_type) {} + + inline value::value(int type, bool) : type_(type) { + switch (type) { +#define INIT(p, v) case p##type: u_.p = v; break + INIT(boolean_, false); + INIT(number_, 0.0); +#ifdef PICOJSON_USE_INT64 + INIT(int64_, 0); +#endif + INIT(string_, new std::string()); + INIT(array_, new array()); + INIT(object_, new object()); +#undef INIT + default: break; + } + } + + inline value::value(bool b) : type_(boolean_type) { + u_.boolean_ = b; + } + +#ifdef PICOJSON_USE_INT64 + inline value::value(int64_t i) : type_(int64_type) { + u_.int64_ = i; + } +#endif + + inline value::value(double n) : type_(number_type) { + if ( +#ifdef _MSC_VER + ! _finite(n) +#elif __cplusplus>=201103L || !(defined(isnan) && defined(isinf)) + std::isnan(n) || std::isinf(n) +#else + isnan(n) || isinf(n) +#endif + ) { + throw std::overflow_error(""); + } + u_.number_ = n; + } + + inline value::value(const std::string& s) : type_(string_type) { + u_.string_ = new std::string(s); + } + + inline value::value(const array& a) : type_(array_type) { + u_.array_ = new array(a); + } + + inline value::value(const object& o) : type_(object_type) { + u_.object_ = new object(o); + } + + inline value::value(const char* s) : type_(string_type) { + u_.string_ = new std::string(s); + } + + inline value::value(const char* s, size_t len) : type_(string_type) { + u_.string_ = new std::string(s, len); + } + + inline value::~value() { + switch (type_) { +#define DEINIT(p) case p##type: delete u_.p; break + DEINIT(string_); + DEINIT(array_); + DEINIT(object_); +#undef DEINIT + default: break; + } + } + + inline value::value(const value& x) : type_(x.type_) { + switch (type_) { +#define INIT(p, v) case p##type: u_.p = v; break + INIT(string_, new std::string(*x.u_.string_)); + INIT(array_, new array(*x.u_.array_)); + INIT(object_, new object(*x.u_.object_)); +#undef INIT + default: + u_ = x.u_; + break; + } + } + + inline value& value::operator=(const value& x) { + if (this != &x) { + value t(x); + swap(t); + } + return *this; + } + + inline void value::swap(value& x) { + std::swap(type_, x.type_); + std::swap(u_, x.u_); + } + +#define IS(ctype, jtype) \ + template <> inline bool value::is() const { \ + return type_ == jtype##_type; \ + } + IS(null, null) + IS(bool, boolean) +#ifdef PICOJSON_USE_INT64 + IS(int64_t, int64) +#endif + IS(std::string, string) + IS(array, array) + IS(object, object) +#undef IS + template <> inline bool value::is() const { + return type_ == number_type +#ifdef PICOJSON_USE_INT64 + || type_ == int64_type +#endif + ; + } + +#define GET(ctype, var) \ + template <> inline const ctype& value::get() const { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" \ + && is()); \ + return var; \ + } \ + template <> inline ctype& value::get() { \ + PICOJSON_ASSERT("type mismatch! call is() before get()" \ + && is()); \ + return var; \ + } + GET(bool, u_.boolean_) + GET(std::string, *u_.string_) + GET(array, *u_.array_) + GET(object, *u_.object_) +#ifdef PICOJSON_USE_INT64 + GET(double, (type_ == int64_type && (const_cast(this)->type_ = number_type, const_cast(this)->u_.number_ = u_.int64_), u_.number_)) + GET(int64_t, u_.int64_) +#else + GET(double, u_.number_) +#endif +#undef GET + + inline bool value::evaluate_as_boolean() const { + switch (type_) { + case null_type: + return false; + case boolean_type: + return u_.boolean_; + case number_type: + return u_.number_ != 0; + case string_type: + return ! u_.string_->empty(); + default: + return true; + } + } + + inline const value& value::get(size_t idx) const { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; + } + + inline value& value::get(size_t idx) { + static value s_null; + PICOJSON_ASSERT(is()); + return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; + } + + inline const value& value::get(const std::string& key) const { + static value s_null; + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; + } + + inline value& value::get(const std::string& key) { + static value s_null; + PICOJSON_ASSERT(is()); + object::iterator i = u_.object_->find(key); + return i != u_.object_->end() ? i->second : s_null; + } + + inline bool value::contains(size_t idx) const { + PICOJSON_ASSERT(is()); + return idx < u_.array_->size(); + } + + inline bool value::contains(const std::string& key) const { + PICOJSON_ASSERT(is()); + object::const_iterator i = u_.object_->find(key); + return i != u_.object_->end(); + } + + inline std::string value::to_str() const { + switch (type_) { + case null_type: return "null"; + case boolean_type: return u_.boolean_ ? "true" : "false"; +#ifdef PICOJSON_USE_INT64 + case int64_type: { + char buf[sizeof("-9223372036854775808")]; + SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); + return buf; + } +#endif + case number_type: { + char buf[256]; + double tmp; + SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); +#if PICOJSON_USE_LOCALE + char *decimal_point = localeconv()->decimal_point; + if (strcmp(decimal_point, ".") != 0) { + size_t decimal_point_len = strlen(decimal_point); + for (char *p = buf; *p != '\0'; ++p) { + if (strncmp(p, decimal_point, decimal_point_len) == 0) { + return std::string(buf, p) + "." + (p + decimal_point_len); + } + } + } +#endif + return buf; + } + case string_type: return *u_.string_; + case array_type: return "array"; + case object_type: return "object"; + default: PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + } + return std::string(); + } + + template void copy(const std::string& s, Iter oi) { + std::copy(s.begin(), s.end(), oi); + } + + template void serialize_str(const std::string& s, Iter oi) { + *oi++ = '"'; + for (std::string::const_iterator i = s.begin(); i != s.end(); ++i) { + switch (*i) { +#define MAP(val, sym) case val: copy(sym, oi); break + MAP('"', "\\\""); + MAP('\\', "\\\\"); + MAP('/', "\\/"); + MAP('\b', "\\b"); + MAP('\f', "\\f"); + MAP('\n', "\\n"); + MAP('\r', "\\r"); + MAP('\t', "\\t"); +#undef MAP + default: + if (static_cast(*i) < 0x20 || *i == 0x7f) { + char buf[7]; + SNPRINTF(buf, sizeof(buf), "\\u%04x", *i & 0xff); + copy(buf, buf + 6, oi); + } else { + *oi++ = *i; + } + break; + } + } + *oi++ = '"'; + } + + template void value::serialize(Iter oi, bool prettify) const { + return _serialize(oi, prettify ? 0 : -1); + } + + inline std::string value::serialize(bool prettify) const { + return _serialize(prettify ? 0 : -1); + } + + template void value::_indent(Iter oi, int indent) { + *oi++ = '\n'; + for (int i = 0; i < indent * INDENT_WIDTH; ++i) { + *oi++ = ' '; + } + } + + template void value::_serialize(Iter oi, int indent) const { + switch (type_) { + case string_type: + serialize_str(*u_.string_, oi); + break; + case array_type: { + *oi++ = '['; + if (indent != -1) { + ++indent; + } + for (array::const_iterator i = u_.array_->begin(); + i != u_.array_->end(); + ++i) { + if (i != u_.array_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + i->_serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (! u_.array_->empty()) { + _indent(oi, indent); + } + } + *oi++ = ']'; + break; + } + case object_type: { + *oi++ = '{'; + if (indent != -1) { + ++indent; + } + for (object::const_iterator i = u_.object_->begin(); + i != u_.object_->end(); + ++i) { + if (i != u_.object_->begin()) { + *oi++ = ','; + } + if (indent != -1) { + _indent(oi, indent); + } + serialize_str(i->first, oi); + *oi++ = ':'; + if (indent != -1) { + *oi++ = ' '; + } + i->second._serialize(oi, indent); + } + if (indent != -1) { + --indent; + if (! u_.object_->empty()) { + _indent(oi, indent); + } + } + *oi++ = '}'; + break; + } + default: + copy(to_str(), oi); + break; + } + if (indent == 0) { + *oi++ = '\n'; + } + } + + inline std::string value::_serialize(int indent) const { + std::string s; + _serialize(std::back_inserter(s), indent); + return s; + } + + template class input { + protected: + Iter cur_, end_; + int last_ch_; + bool ungot_; + int line_; + public: + input(const Iter& first, const Iter& last) : cur_(first), end_(last), last_ch_(-1), ungot_(false), line_(1) {} + int getc() { + if (ungot_) { + ungot_ = false; + return last_ch_; + } + if (cur_ == end_) { + last_ch_ = -1; + return -1; + } + if (last_ch_ == '\n') { + line_++; + } + last_ch_ = *cur_ & 0xff; + ++cur_; + return last_ch_; + } + void ungetc() { + if (last_ch_ != -1) { + PICOJSON_ASSERT(! ungot_); + ungot_ = true; + } + } + Iter cur() const { return cur_; } + int line() const { return line_; } + void skip_ws() { + while (1) { + int ch = getc(); + if (! (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { + ungetc(); + break; + } + } + } + bool expect(int expect) { + skip_ws(); + if (getc() != expect) { + ungetc(); + return false; + } + return true; + } + bool match(const std::string& pattern) { + for (std::string::const_iterator pi(pattern.begin()); + pi != pattern.end(); + ++pi) { + if (getc() != *pi) { + ungetc(); + return false; + } + } + return true; + } + }; + + template inline int _parse_quadhex(input &in) { + int uni_ch = 0, hex; + for (int i = 0; i < 4; i++) { + if ((hex = in.getc()) == -1) { + return -1; + } + if ('0' <= hex && hex <= '9') { + hex -= '0'; + } else if ('A' <= hex && hex <= 'F') { + hex -= 'A' - 0xa; + } else if ('a' <= hex && hex <= 'f') { + hex -= 'a' - 0xa; + } else { + in.ungetc(); + return -1; + } + uni_ch = uni_ch * 16 + hex; + } + return uni_ch; + } + + template inline bool _parse_codepoint(String& out, input& in) { + int uni_ch; + if ((uni_ch = _parse_quadhex(in)) == -1) { + return false; + } + if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { + if (0xdc00 <= uni_ch) { + // a second 16-bit of a surrogate pair appeared + return false; + } + // first 16-bit of surrogate pair, get the next one + if (in.getc() != '\\' || in.getc() != 'u') { + in.ungetc(); + return false; + } + int second = _parse_quadhex(in); + if (! (0xdc00 <= second && second <= 0xdfff)) { + return false; + } + uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); + uni_ch += 0x10000; + } + if (uni_ch < 0x80) { + out.push_back(uni_ch); + } else { + if (uni_ch < 0x800) { + out.push_back(0xc0 | (uni_ch >> 6)); + } else { + if (uni_ch < 0x10000) { + out.push_back(0xe0 | (uni_ch >> 12)); + } else { + out.push_back(0xf0 | (uni_ch >> 18)); + out.push_back(0x80 | ((uni_ch >> 12) & 0x3f)); + } + out.push_back(0x80 | ((uni_ch >> 6) & 0x3f)); + } + out.push_back(0x80 | (uni_ch & 0x3f)); + } + return true; + } + + template inline bool _parse_string(String& out, input& in) { + while (1) { + int ch = in.getc(); + if (ch < ' ') { + in.ungetc(); + return false; + } else if (ch == '"') { + return true; + } else if (ch == '\\') { + if ((ch = in.getc()) == -1) { + return false; + } + switch (ch) { +#define MAP(sym, val) case sym: out.push_back(val); break + MAP('"', '\"'); + MAP('\\', '\\'); + MAP('/', '/'); + MAP('b', '\b'); + MAP('f', '\f'); + MAP('n', '\n'); + MAP('r', '\r'); + MAP('t', '\t'); +#undef MAP + case 'u': + if (! _parse_codepoint(out, in)) { + return false; + } + break; + default: + return false; + } + } else { + out.push_back(ch); + } + } + return false; + } + + template inline bool _parse_array(Context& ctx, input& in) { + if (! ctx.parse_array_start()) { + return false; + } + size_t idx = 0; + if (in.expect(']')) { + return ctx.parse_array_stop(idx); + } + do { + if (! ctx.parse_array_item(in, idx)) { + return false; + } + idx++; + } while (in.expect(',')); + return in.expect(']') && ctx.parse_array_stop(idx); + } + + template inline bool _parse_object(Context& ctx, input& in) { + if (! ctx.parse_object_start()) { + return false; + } + if (in.expect('}')) { + return true; + } + do { + std::string key; + if (! in.expect('"') + || ! _parse_string(key, in) + || ! in.expect(':')) { + return false; + } + if (! ctx.parse_object_item(in, key)) { + return false; + } + } while (in.expect(',')); + return in.expect('}'); + } + + template inline std::string _parse_number(input& in) { + std::string num_str; + while (1) { + int ch = in.getc(); + if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' + || ch == 'e' || ch == 'E') { + num_str.push_back(ch); + } else if (ch == '.') { +#if PICOJSON_USE_LOCALE + num_str += localeconv()->decimal_point; +#else + num_str.push_back('.'); +#endif + } else { + in.ungetc(); + break; + } + } + return num_str; + } + + template inline bool _parse(Context& ctx, input& in) { + in.skip_ws(); + int ch = in.getc(); + switch (ch) { +#define IS(ch, text, op) case ch: \ + if (in.match(text) && op) { \ + return true; \ + } else { \ + return false; \ + } + IS('n', "ull", ctx.set_null()); + IS('f', "alse", ctx.set_bool(false)); + IS('t', "rue", ctx.set_bool(true)); +#undef IS + case '"': + return ctx.parse_string(in); + case '[': + return _parse_array(ctx, in); + case '{': + return _parse_object(ctx, in); + default: + if (('0' <= ch && ch <= '9') || ch == '-') { + double f; + char *endp; + in.ungetc(); + std::string num_str = _parse_number(in); + if (num_str.empty()) { + return false; + } +#ifdef PICOJSON_USE_INT64 + { + errno = 0; + intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); + if (errno == 0 + && std::numeric_limits::min() <= ival + && ival <= std::numeric_limits::max() + && endp == num_str.c_str() + num_str.size()) { + ctx.set_int64(ival); + return true; + } + } +#endif + f = strtod(num_str.c_str(), &endp); + if (endp == num_str.c_str() + num_str.size()) { + ctx.set_number(f); + return true; + } + return false; + } + break; + } + in.ungetc(); + return false; + } + + class deny_parse_context { + public: + bool set_null() { return false; } + bool set_bool(bool) { return false; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return false; } +#endif + bool set_number(double) { return false; } + template bool parse_string(input&) { return false; } + bool parse_array_start() { return false; } + template bool parse_array_item(input&, size_t) { + return false; + } + bool parse_array_stop(size_t) { return false; } + bool parse_object_start() { return false; } + template bool parse_object_item(input&, const std::string&) { + return false; + } + }; + + class default_parse_context { + protected: + value* out_; + public: + default_parse_context(value* out) : out_(out) {} + bool set_null() { + *out_ = value(); + return true; + } + bool set_bool(bool b) { + *out_ = value(b); + return true; + } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t i) { + *out_ = value(i); + return true; + } +#endif + bool set_number(double f) { + *out_ = value(f); + return true; + } + template bool parse_string(input& in) { + *out_ = value(string_type, false); + return _parse_string(out_->get(), in); + } + bool parse_array_start() { + *out_ = value(array_type, false); + return true; + } + template bool parse_array_item(input& in, size_t) { + array& a = out_->get(); + a.push_back(value()); + default_parse_context ctx(&a.back()); + return _parse(ctx, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { + *out_ = value(object_type, false); + return true; + } + template bool parse_object_item(input& in, const std::string& key) { + object& o = out_->get(); + default_parse_context ctx(&o[key]); + return _parse(ctx, in); + } + private: + default_parse_context(const default_parse_context&); + default_parse_context& operator=(const default_parse_context&); + }; + + class null_parse_context { + public: + struct dummy_str { + void push_back(int) {} + }; + public: + null_parse_context() {} + bool set_null() { return true; } + bool set_bool(bool) { return true; } +#ifdef PICOJSON_USE_INT64 + bool set_int64(int64_t) { return true; } +#endif + bool set_number(double) { return true; } + template bool parse_string(input& in) { + dummy_str s; + return _parse_string(s, in); + } + bool parse_array_start() { return true; } + template bool parse_array_item(input& in, size_t) { + return _parse(*this, in); + } + bool parse_array_stop(size_t) { return true; } + bool parse_object_start() { return true; } + template bool parse_object_item(input& in, const std::string&) { + return _parse(*this, in); + } + private: + null_parse_context(const null_parse_context&); + null_parse_context& operator=(const null_parse_context&); + }; + + // obsolete, use the version below + template inline std::string parse(value& out, Iter& pos, const Iter& last) { + std::string err; + pos = parse(out, pos, last, &err); + return err; + } + + template inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { + input in(first, last); + if (! _parse(ctx, in) && err != NULL) { + char buf[64]; + SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); + *err = buf; + while (1) { + int ch = in.getc(); + if (ch == -1 || ch == '\n') { + break; + } else if (ch >= ' ') { + err->push_back(ch); + } + } + } + return in.cur(); + } + + template inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { + default_parse_context ctx(&out); + return _parse(ctx, first, last, err); + } + + inline std::string parse(value& out, const std::string& s) { + std::string err; + parse(out, s.begin(), s.end(), &err); + return err; + } + + inline std::string parse(value& out, std::istream& is) { + std::string err; + parse(out, std::istreambuf_iterator(is.rdbuf()), + std::istreambuf_iterator(), &err); + return err; + } + + template struct last_error_t { + static std::string s; + }; + template std::string last_error_t::s; + + inline void set_last_error(const std::string& s) { + last_error_t::s = s; + } + + inline const std::string& get_last_error() { + return last_error_t::s; + } + + inline bool operator==(const value& x, const value& y) { + if (x.is()) + return y.is(); +#define PICOJSON_CMP(type) \ + if (x.is()) \ + return y.is() && x.get() == y.get() + PICOJSON_CMP(bool); + PICOJSON_CMP(double); + PICOJSON_CMP(std::string); + PICOJSON_CMP(array); + PICOJSON_CMP(object); +#undef PICOJSON_CMP + PICOJSON_ASSERT(0); +#ifdef _MSC_VER + __assume(0); +#endif + return false; + } + + inline bool operator!=(const value& x, const value& y) { + return ! (x == y); + } +} + +namespace std { + template<> inline void swap(picojson::value& x, picojson::value& y) + { + x.swap(y); + } +} + +inline std::istream& operator>>(std::istream& is, picojson::value& x) +{ + picojson::set_last_error(std::string()); + std::string err = picojson::parse(x, is); + if (! err.empty()) { + picojson::set_last_error(err); + is.setstate(std::ios::failbit); + } + return is; +} + +inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) +{ + x.serialize(std::ostream_iterator(os)); + return os; +} +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif diff --git a/deps/picojson/picotest/picotest.c b/deps/picojson/picotest/picotest.c new file mode 100644 index 0000000000..d1fe699d5a --- /dev/null +++ b/deps/picojson/picotest/picotest.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2014 DeNA Co., Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include +#include +#include +#include "picotest.h" + +struct test_t { + int num_tests; + int failed; +}; +struct test_t main_tests, *cur_tests = &main_tests; +static int test_level = 0; + +static void indent(void) +{ + int i; + for (i = 0; i != test_level; ++i) + printf(" "); +} + +__attribute__((format (printf, 1, 2))) +void note(const char *fmt, ...) +{ + va_list arg; + + indent(); + printf("# "); + + va_start(arg, fmt); + vprintf(fmt, arg); + va_end(arg); + + printf("\n"); +} + +__attribute__((format (printf, 2, 3))) +void _ok(int cond, const char *fmt, ...) +{ + va_list arg; + + if (! cond) + cur_tests->failed = 1; + indent(); + + printf("%s %d - ", cond ? "ok" : "not ok", ++cur_tests->num_tests); + va_start(arg, fmt); + vprintf(fmt, arg); + va_end(arg); + + printf("\n"); +} + +int done_testing(void) +{ + indent(); + printf("1..%d\n", cur_tests->num_tests); + return cur_tests->failed; +} + +void subtest(const char *name, void (*cb)(void)) +{ + struct test_t test = {}, *parent_tests; + + parent_tests = cur_tests; + cur_tests = &test; + ++test_level; + + note("Subtest: %s", name); + + cb(); + + done_testing(); + + --test_level; + cur_tests = parent_tests; + if (test.failed) + cur_tests->failed = 1; + _ok(! test.failed, "%s", name); +} diff --git a/deps/picojson/picotest/picotest.h b/deps/picojson/picotest/picotest.h new file mode 100644 index 0000000000..22ab02fb2d --- /dev/null +++ b/deps/picojson/picotest/picotest.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014 DeNA Co., Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef picotest_h +#define picotest_h + +#ifdef __cplusplus +extern "C" { +#endif + +void note(const char *fmt, ...) __attribute__((format (printf, 1, 2))); +void _ok(int cond, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +#define ok(cond) _ok(cond, "%s %d", __FILE__, __LINE__) +int done_testing(void); +void subtest(const char *name, void (*cb)(void)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/picojson/test.cc b/deps/picojson/test.cc new file mode 100644 index 0000000000..ed9656d2cb --- /dev/null +++ b/deps/picojson/test.cc @@ -0,0 +1,312 @@ +/* + * Copyright 2009-2010 Cybozu Labs, Inc. + * Copyright 2011-2014 Kazuho Oku, Yasuhiro Matsumoto, Shigeo Mitsunari + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "picojson.h" +#include "picotest/picotest.h" + +#ifdef _MSC_VER + #pragma warning(disable : 4127) // conditional expression is constant +#endif + +using namespace std; + +#define is(x, y, name) _ok((x) == (y), name) + +#include +#include +#include +#include + +int main(void) +{ +#if PICOJSON_USE_LOCALE + setlocale(LC_ALL, ""); +#endif + + // constructors +#define TEST(expr, expected) \ + is(picojson::value expr .serialize(), string(expected), "picojson::value" #expr) + + TEST( (true), "true"); + TEST( (false), "false"); + TEST( (42.0), "42"); + TEST( (string("hello")), "\"hello\""); + TEST( ("hello"), "\"hello\""); + TEST( ("hello", 4), "\"hell\""); + + { + double a = 1; + for (int i = 0; i < 1024; i++) { + picojson::value vi(a); + std::stringstream ss; + ss << vi; + picojson::value vo; + ss >> vo; + double b = vo.get(); + if ((i < 53 && a != b) || fabs(a - b) / b > 1e-8) { + printf("ng i=%d a=%.18e b=%.18e\n", i, a, b); + } + a *= 2; + } + } + +#undef TEST + +#define TEST(in, type, cmp, serialize_test) { \ + picojson::value v; \ + const char* s = in; \ + string err = picojson::parse(v, s, s + strlen(s)); \ + _ok(err.empty(), in " no error"); \ + _ok(v.is(), in " check type"); \ + is(v.get(), static_cast(cmp), in " correct output"); \ + is(*s, '\0', in " read to eof"); \ + if (serialize_test) { \ + is(v.serialize(), string(in), in " serialize"); \ + } \ + } + TEST("false", bool, false, true); + TEST("true", bool, true, true); + TEST("90.5", double, 90.5, false); + TEST("1.7976931348623157e+308", double, DBL_MAX, false); + TEST("\"hello\"", string, string("hello"), true); + TEST("\"\\\"\\\\\\/\\b\\f\\n\\r\\t\"", string, string("\"\\/\b\f\n\r\t"), + true); + TEST("\"\\u0061\\u30af\\u30ea\\u30b9\"", string, + string("a\xe3\x82\xaf\xe3\x83\xaa\xe3\x82\xb9"), false); + TEST("\"\\ud840\\udc0b\"", string, string("\xf0\xa0\x80\x8b"), false); +#ifdef PICOJSON_USE_INT64 + TEST("0", int64_t, 0, true); + TEST("-9223372036854775808", int64_t, std::numeric_limits::min(), true); + TEST("9223372036854775807", int64_t, std::numeric_limits::max(), true); +#endif +#undef TEST + +#define TEST(type, expr) { \ + picojson::value v; \ + const char *s = expr; \ + string err = picojson::parse(v, s, s + strlen(s)); \ + _ok(err.empty(), "empty " #type " no error"); \ + _ok(v.is(), "empty " #type " check type"); \ + _ok(v.get().empty(), "check " #type " array size"); \ + } + TEST(array, "[]"); + TEST(object, "{}"); +#undef TEST + + { + picojson::value v; + const char *s = "[1,true,\"hello\"]"; + string err = picojson::parse(v, s, s + strlen(s)); + _ok(err.empty(), "array no error"); + _ok(v.is(), "array check type"); + is(v.get().size(), size_t(3), "check array size"); + _ok(v.contains(0), "check contains array[0]"); + _ok(v.get(0).is(), "check array[0] type"); + is(v.get(0).get(), 1.0, "check array[0] value"); + _ok(v.contains(1), "check contains array[1]"); + _ok(v.get(1).is(), "check array[1] type"); + _ok(v.get(1).get(), "check array[1] value"); + _ok(v.contains(2), "check contains array[2]"); + _ok(v.get(2).is(), "check array[2] type"); + is(v.get(2).get(), string("hello"), "check array[2] value"); + _ok(!v.contains(3), "check not contains array[3]"); + } + + { + picojson::value v; + const char *s = "{ \"a\": true }"; + string err = picojson::parse(v, s, s + strlen(s)); + _ok(err.empty(), "object no error"); + _ok(v.is(), "object check type"); + is(v.get().size(), size_t(1), "check object size"); + _ok(v.contains("a"), "check contains property"); + _ok(v.get("a").is(), "check bool property exists"); + is(v.get("a").get(), true, "check bool property value"); + is(v.serialize(), string("{\"a\":true}"), "serialize object"); + _ok(!v.contains("z"), "check not contains property"); + } + +#define TEST(json, msg) do { \ + picojson::value v; \ + const char *s = json; \ + string err = picojson::parse(v, s, s + strlen(s)); \ + is(err, string("syntax error at line " msg), msg); \ + } while (0) + TEST("falsoa", "1 near: oa"); + TEST("{]", "1 near: ]"); + TEST("\n\bbell", "2 near: bell"); + TEST("\"abc\nd\"", "1 near: "); +#undef TEST + + { + picojson::value v1, v2; + const char *s; + string err; + s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; + err = picojson::parse(v1, s, s + strlen(s)); + s = "{ \"d\": 2.0, \"b\": true, \"a\": [1,2,\"three\"] }"; + err = picojson::parse(v2, s, s + strlen(s)); + _ok((v1 == v2), "check == operator in deep comparison"); + } + + { + picojson::value v1, v2; + const char *s; + string err; + s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; + err = picojson::parse(v1, s, s + strlen(s)); + s = "{ \"d\": 2.0, \"a\": [1,\"three\"], \"b\": true }"; + err = picojson::parse(v2, s, s + strlen(s)); + _ok((v1 != v2), "check != operator for array in deep comparison"); + } + + { + picojson::value v1, v2; + const char *s; + string err; + s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; + err = picojson::parse(v1, s, s + strlen(s)); + s = "{ \"d\": 2.0, \"a\": [1,2,\"three\"], \"b\": false }"; + err = picojson::parse(v2, s, s + strlen(s)); + _ok((v1 != v2), "check != operator for object in deep comparison"); + } + + { + picojson::value v1, v2; + const char *s; + string err; + s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; + err = picojson::parse(v1, s, s + strlen(s)); + picojson::object& o = v1.get(); + o.erase("b"); + picojson::array& a = o["a"].get(); + picojson::array::iterator i; + i = std::remove(a.begin(), a.end(), picojson::value(std::string("three"))); + a.erase(i, a.end()); + s = "{ \"a\": [1,2], \"d\": 2 }"; + err = picojson::parse(v2, s, s + strlen(s)); + _ok((v1 == v2), "check erase()"); + } + + _ok(picojson::value(3.0).serialize() == "3", + "integral number should be serialized as a integer"); + + { + const char* s = "{ \"a\": [1,2], \"d\": 2 }"; + picojson::null_parse_context ctx; + string err; + picojson::_parse(ctx, s, s + strlen(s), &err); + _ok(err.empty(), "null_parse_context"); + } + + { + picojson::value v1, v2; + v1 = picojson::value(true); + swap(v1, v2); + _ok(v1.is(), "swap (null)"); + _ok(v2.get() == true, "swap (bool)"); + + v1 = picojson::value("a"); + v2 = picojson::value(1.0); + swap(v1, v2); + _ok(v1.get() == 1.0, "swap (dobule)"); + _ok(v2.get() == "a", "swap (string)"); + + v1 = picojson::value(picojson::object()); + v2 = picojson::value(picojson::array()); + swap(v1, v2); + _ok(v1.is(), "swap (array)"); + _ok(v2.is(), "swap (object)"); + } + + { + picojson::value v; + const char *s = "{ \"a\": 1, \"b\": [ 2, { \"b1\": \"abc\" } ], \"c\": {}, \"d\": [] }"; + string err; + err = picojson::parse(v, s, s + strlen(s)); + _ok(err.empty(), "parse test data for prettifying output"); + _ok(v.serialize() == "{\"a\":1,\"b\":[2,{\"b1\":\"abc\"}],\"c\":{},\"d\":[]}", "non-prettifying output"); + _ok(v.serialize(true) == "{\n \"a\": 1,\n \"b\": [\n 2,\n {\n \"b1\": \"abc\"\n }\n ],\n \"c\": {},\n \"d\": []\n}\n", "prettifying output"); + } + + try { + picojson::value v(std::numeric_limits::quiet_NaN()); + _ok(false, "should not accept NaN"); + } catch (std::overflow_error e) { + _ok(true, "should not accept NaN"); + } + + try { + picojson::value v(std::numeric_limits::infinity()); + _ok(false, "should not accept infinity"); + } catch (std::overflow_error e) { + _ok(true, "should not accept infinity"); + } + + try { + picojson::value v(123.); + _ok(! v.is(), "is() should return false"); + v.get(); + _ok(false, "get() should raise an error"); + } catch (std::runtime_error e) { + _ok(true, "get() should raise an error"); + } + +#ifdef PICOJSON_USE_INT64 + { + picojson::value v1((int64_t)123); + _ok(v1.is(), "is int64_t"); + _ok(v1.is(), "is double as well"); + _ok(v1.serialize() == "123", "serialize the value"); + _ok(v1.get() == 123, "value is correct as int64_t"); + _ok(v1.get(), "value is correct as double"); + + _ok(! v1.is(), "is no more int64_type once get() is called"); + _ok(v1.is(), "and is still a double"); + + const char *s = "-9223372036854775809"; + _ok(picojson::parse(v1, s, s + strlen(s)).empty(), "parse underflowing int64_t"); + _ok(! v1.is(), "underflowing int is not int64_t"); + _ok(v1.is(), "underflowing int is double"); + _ok(v1.get() + 9.22337203685478e+18 < 65536, "double value is somewhat correct"); + } +#endif + + { + picojson::value v; + std::string err = picojson::parse(v, "[ 1, \"abc\" ]"); + _ok(err.empty(), "simple API no error"); + _ok(v.is(), "simple API return type is array"); + is(v.get().size(), 2, "simple API array size"); + _ok(v.get()[0].is(), "simple API type #0"); + is(v.get()[0].get(), 1, "simple API value #0"); + _ok(v.get()[1].is(), "simple API type #1"); + is(v.get()[1].get(), "abc", "simple API value #1"); + } + + return done_testing(); +} diff --git a/scripts/build_picojson.bat b/scripts/build_picojson.bat new file mode 100644 index 0000000000..38f4ffae98 --- /dev/null +++ b/scripts/build_picojson.bat @@ -0,0 +1,46 @@ + +set picojson_version=1.3.0 +call %* + +:get-version + set version=%picojson_version% + goto :EOF + +:build +setlocal +set platform=%1 +set build_type=%2 +set vs_version=%3 +set dynamic_runtime=%4 + +set scriptdir=%~dp0 +call "%scriptdir%_init.bat" %platform% %build_type% %vs_version% +if %ERRORLEVEL% NEQ 0 goto :error +call "%scriptdir%utils.bat" :setup_visual_studio %vs_version% +if %ERRORLEVEL% NEQ 0 goto :error + +set DEPS_DIR=%scriptdir%..\deps +set PICOJSON_SOURCE_DIR=%DEPS_DIR%\picojson +set PICOJSON_INSTALL_DIR=%scriptdir%..\deps-build\%build_dir%\picojson + +cd "%PICOJSON_SOURCE_DIR%" + +rd /S /Q %PICOJSON_INSTALL_DIR% +md %PICOJSON_INSTALL_DIR% +md %PICOJSON_INSTALL_DIR%\include +copy %PICOJSON_SOURCE_DIR%\picojson.h %PICOJSON_INSTALL_DIR%\include + +cd "%curdir%" + +echo === archiving the library +call "%scriptdir%utils.bat" :zip_file picojson %picojson_version% +if %ERRORLEVEL% NEQ 0 goto :error + +goto :success + +:success +exit /b 0 + +:error +cd "%curdir%" +exit /b 1 diff --git a/scripts/build_picojson.sh b/scripts/build_picojson.sh new file mode 100755 index 0000000000..5ded316b86 --- /dev/null +++ b/scripts/build_picojson.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$DIR/_init.sh" "$@" +source "$DIR/utils.sh" + +PICOJSON_SRC_VERSION=1.3.0 +PICOJSON_BUILD_VERSION=1 +PICOJSON_VERSION=$PICOJSON_SRC_VERSION.$PICOJSON_BUILD_VERSION + +SOURCE_DIR="$DIR/../deps/picojson" +INSTALL_DIR="$DEPENDENCY_DIR/picojson" + +mkdir -p "${INSTALL_DIR}/include" +cp "$SOURCE_DIR/picojson.h" "${INSTALL_DIR}/include" + +echo === zip_file "picojson" "$PICOJSON_VERSION" "$target" +zip_file "picojson" "$PICOJSON_VERSION" "$target" From 48d562f9e803cd6c18c9ccd727f72cb9ec6e8fda Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:49:44 -0700 Subject: [PATCH 55/74] Fix boost error --- cpp/lib/ClientConfigParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 4c5c569c9b..335de78594 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -12,6 +12,7 @@ #include #include +#undef snprintf #include #ifndef _WIN32 From 79128b484bcf8cfe1566523c41e0f519c28cf46d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:02:04 -0700 Subject: [PATCH 56/74] Fix test cases --- tests/test_unit_logger.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index a4fe7823d2..2980b10943 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -139,7 +139,6 @@ void test_client_config_log_init(void** unused) { remove(LOG_PATH); } - /** * Tests timing of log file creation */ @@ -261,8 +260,8 @@ int main(void) { cmocka_unit_test(test_client_config_log_invalid_json), cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), -#ifndef _WIN32 cmocka_unit_test(test_log_creation), +#ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), #endif }; From 98c4924232c8b8eec9e86ca15aac593ec5ab0e56 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-nl <143542970+sfc-gh-ext-simba-nl@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:05:19 -0700 Subject: [PATCH 57/74] Fix logger unit tests --- tests/test_unit_logger.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 2980b10943..893213420f 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -7,11 +7,7 @@ #include #include "memory.h" -#ifdef _WIN32 -inline int access(const char* pathname, int mode) { - return _access(pathname, mode); -} -#else +#ifndef _WIN32 #include #endif @@ -64,6 +60,7 @@ void test_client_config_log_invalid_json(void** unused) { remove(configFilePath); } +#ifndef _WIN32 /** * Tests log settings from client config file */ @@ -93,11 +90,11 @@ void test_client_config_log(void **unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -127,11 +124,11 @@ void test_client_config_log_init(void** unused) { // Info log won't trigger the log file creation since log level is set to warn in config log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, 0), 0); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); // Warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, 0), 0); + assert_int_equal(access(LOG_PATH, F_OK), 0); log_close(); // Cleanup @@ -147,7 +144,7 @@ void test_log_creation(void **unused) { // ensure the log file doesn't exist at the beginning remove(logname); - assert_int_not_equal(access(logname, 0), 0); + assert_int_not_equal(access(logname, F_OK), 0); log_set_lock(NULL); log_set_level(SF_LOG_WARN); @@ -156,17 +153,16 @@ void test_log_creation(void **unused) { // info log won't trigger the log file creation since log level is set to warning log_info("dummy info log"); - assert_int_not_equal(access(logname, 0), 0); + assert_int_not_equal(access(logname, F_OK), 0); // warning log will trigger the log file creation log_warn("dummy warning log"); - assert_int_equal(access(logname, 0), 0); + assert_int_equal(access(logname, F_OK), 0); log_close(); remove(logname); } -#ifndef _WIN32 /** * Tests masking secret information in log */ @@ -258,10 +254,10 @@ int main(void) { cmocka_unit_test(test_log_str_to_level), cmocka_unit_test(test_invalid_client_config_path), cmocka_unit_test(test_client_config_log_invalid_json), +#ifndef _WIN32 cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), cmocka_unit_test(test_log_creation), -#ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), #endif }; From a5549aa021089b4c996c3211d7c639a6b7ab679b Mon Sep 17 00:00:00 2001 From: norrislee Date: Tue, 29 Oct 2024 09:54:47 -0700 Subject: [PATCH 58/74] Update log level and path precedence logic, add SF_LOG_DEFAULT --- include/snowflake/logger.h | 3 ++- lib/client.c | 30 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/include/snowflake/logger.h b/include/snowflake/logger.h index b860f8ecd8..3304ed9fdc 100644 --- a/include/snowflake/logger.h +++ b/include/snowflake/logger.h @@ -47,7 +47,8 @@ typedef enum SF_LOG_LEVEL { SF_LOG_INFO, SF_LOG_WARN, SF_LOG_ERROR, - SF_LOG_FATAL + SF_LOG_FATAL, + SF_LOG_DEFAULT } SF_LOG_LEVEL; #define CXX_LOG_NS "C++" diff --git a/lib/client.c b/lib/client.c index b0a87c024d..bfed48f073 100644 --- a/lib/client.c +++ b/lib/client.c @@ -321,24 +321,26 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { size_t log_path_size = 1; //Start with 1 to include null terminator log_path_size += strlen(time_str); - /* The client config takes precedence over environmental variables */ - if (strlen(clientConfig.logPath) != 0) { - sf_log_path = clientConfig.logPath; - } else { - /* The environment variables takes precedence over the specified parameters */ - sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); - if (sf_log_path == NULL && log_path) { + /* The environment variables takes precedence over the specified parameters. + Specified parameters takes precedence over client config */ + sf_log_path = sf_getenv_s("SNOWFLAKE_LOG_PATH", log_path_buf, sizeof(log_path_buf)); + if (sf_log_path == NULL) { + if (log_path && strlen(log_path) != 0) { sf_log_path = log_path; + } else if (strlen(clientConfig.logPath) != 0) { + sf_log_path = clientConfig.logPath; } } - if (strlen(clientConfig.logLevel) != 0) { - sf_log_level = log_from_str_to_level(clientConfig.logLevel); - } else { - /* The client config takes precedence over environment variables */ - sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); - if (sf_log_level_str != NULL) { - sf_log_level = log_from_str_to_level(sf_log_level_str); + sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); + if (sf_log_level_str != NULL) { + sf_log_level = log_from_str_to_level(sf_log_level_str); + } else if (sf_log_level == SF_LOG_DEFAULT) { + if (strlen(clientConfig.logLevel) != 0) { + sf_log_level = log_from_str_to_level(clientConfig.logLevel); + } + else { + sf_log_level = SF_LOG_FATAL; } } From 169063c00267af97a1ddb67a70a855f444601c56 Mon Sep 17 00:00:00 2001 From: norrislee Date: Tue, 29 Oct 2024 10:50:54 -0700 Subject: [PATCH 59/74] Remove accidentally added files --- ci/build/build.sh | 1 - ci/build_win.bat | 3 - deps/picojson/.gitignore | 4 - deps/picojson/Changes | 25 - deps/picojson/LICENSE | 25 - deps/picojson/Makefile | 26 - deps/picojson/README.mkdn | 195 ------ deps/picojson/picojson.h | 1010 ----------------------------- deps/picojson/picotest/picotest.c | 99 --- deps/picojson/picotest/picotest.h | 39 -- deps/picojson/test.cc | 312 --------- scripts/build_picojson.bat | 46 -- scripts/build_picojson.sh | 18 - 13 files changed, 1803 deletions(-) delete mode 100644 deps/picojson/.gitignore delete mode 100644 deps/picojson/Changes delete mode 100644 deps/picojson/LICENSE delete mode 100644 deps/picojson/Makefile delete mode 100644 deps/picojson/README.mkdn delete mode 100644 deps/picojson/picojson.h delete mode 100644 deps/picojson/picotest/picotest.c delete mode 100644 deps/picojson/picotest/picotest.h delete mode 100644 deps/picojson/test.cc delete mode 100644 scripts/build_picojson.bat delete mode 100755 scripts/build_picojson.sh diff --git a/ci/build/build.sh b/ci/build/build.sh index 256be4a3e9..7fb31c75a2 100755 --- a/ci/build/build.sh +++ b/ci/build/build.sh @@ -82,7 +82,6 @@ download_build_component aws "$SCRIPTS_DIR/build_awssdk.sh" "$target" download_build_component azure "$SCRIPTS_DIR/build_azuresdk.sh" "$target" download_build_component cmocka "$SCRIPTS_DIR/build_cmocka.sh" "$target" download_build_component arrow "$SCRIPTS_DIR/build_arrow.sh" "$target" -download_build_component picojson "$SCRIPTS_DIR/build_picojson.sh" "$target" # very tight diskspace limit on github runners, clear deps folder with all .o files if [[ -n "$GITHUB_ACTIONS" ]]; then diff --git a/ci/build_win.bat b/ci/build_win.bat index 58fb19422e..0576e257d8 100644 --- a/ci/build_win.bat +++ b/ci/build_win.bat @@ -26,7 +26,6 @@ set aws_build_script="%scriptdir%..\scripts\build_awssdk.bat" set azure_build_script="%scriptdir%..\scripts\build_azuresdk.bat" set cmocka_build_script="%scriptdir%..\scripts\build_cmocka.bat" set arrow_build_script="%scriptdir%..\scripts\build_arrow.bat" -set picojson_build_script="%scriptdir%..\scripts\build_picojson.bat" set libsnowflakeclient_build_script="%scriptdir%..\scripts\build_libsnowflakeclient.bat" set upload_artifact_script="%scriptdir%container\upload_artifact.bat" @@ -60,8 +59,6 @@ goto :EOF if %ERRORLEVEL% NEQ 0 goto :error call :download_build_component arrow "%arrow_build_script%" "%dynamic_runtime%" if %ERRORLEVEL% NEQ 0 goto :error - call :download_build_component picojson "%picojson_build_script%" "%dynamic_runtime%" - if %ERRORLEVEL% NEQ 0 goto :error if defined GITHUB_ACTIONS ( rd /S /Q %scriptdir%\..\deps ) diff --git a/deps/picojson/.gitignore b/deps/picojson/.gitignore deleted file mode 100644 index 815b59fad3..0000000000 --- a/deps/picojson/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*~ -a.out -test-core -test-core-int64 diff --git a/deps/picojson/Changes b/deps/picojson/Changes deleted file mode 100644 index d6017ddf9a..0000000000 --- a/deps/picojson/Changes +++ /dev/null @@ -1,25 +0,0 @@ -Revision history for picojson - -1.3.0 2015-02-25 13:05:00+0900 - - `make check` is now synonym of `make test` (#62) - - operator= is now safe when part of LHS is being assigned, as well as exception-safe (#66) - -1.2.1 2014-12-16 15:33:00+0900 - - bundle the contents of `picotest/` (#61) - -1.2.0 2014-12-15 16:20:00+0900 - - `make install` to install picojson.h (#58) - - two-argument `picojson::parse()` for ease-of-use (#57) - -1.1.1 2014-06-25 10:35:00+0900 - - tweaks to suppress compiler errors / warning (#38 #39) - - clarify the licenses of the files in exmaple/ (#42) - -1.1 2014-06-16 12:57:00+0900 - - added experimental support for int64 type (#34) - - by default, throw std::runtime_error instead of using assert for runtime errors (#33) - - refine compatibility regarding the use of isinf/isnan (#29, #36) - - remove `.get()` (#35) - -1.0 2014-06-05 12:54:00+0900 - - initial release with a version number diff --git a/deps/picojson/LICENSE b/deps/picojson/LICENSE deleted file mode 100644 index 72f3553911..0000000000 --- a/deps/picojson/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright 2009-2010 Cybozu Labs, Inc. -Copyright 2011-2014 Kazuho Oku -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/picojson/Makefile b/deps/picojson/Makefile deleted file mode 100644 index 40f6450b89..0000000000 --- a/deps/picojson/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -prefix=/usr/local -includedir=$(prefix)/include - -check: test - -test: test-core test-core-int64 - ./test-core - ./test-core-int64 - -test-core: test.cc picotest/picotest.c picotest/picotest.h - $(CXX) -Wall test.cc picotest/picotest.c -o $@ - -test-core-int64: test.cc picotest/picotest.c picotest/picotest.h - $(CXX) -Wall -DPICOJSON_USE_INT64 test.cc picotest/picotest.c -o $@ - -clean: - rm -f test-core test-core-int64 - -install: - install -d $(DESTDIR)$(includedir) - install -p -m 0644 picojson.h $(DESTDIR)$(includedir) - -uninstall: - rm -f $(DESTDIR)$(includedir)/picojson.h - -.PHONY: test check clean install uninstall diff --git a/deps/picojson/README.mkdn b/deps/picojson/README.mkdn deleted file mode 100644 index 7db7fd6a8f..0000000000 --- a/deps/picojson/README.mkdn +++ /dev/null @@ -1,195 +0,0 @@ -# PicoJSON - a C++ JSON parser / serializer - -Copyright © 2009-2010 Cybozu Labs, Inc. -Copyright © 2011-2015 Kazuho Oku - -Licensed under [2-clause BSD license](http://opensource.org/licenses/BSD-2-Clause) - -## Version - -1.3.0 [![Build Status](https://travis-ci.org/kazuho/picojson.svg?branch=rel/1.3.0)](https://travis-ci.org/kazuho/picojson) - -## Introduction - -PicoJSON is a tiny JSON parser / serializer for C++ with following properties: - -- header-file only -- no external dependencies (only uses standard C++ libraries) -- STL-frendly (arrays are represented by using std::vector, objects are std::map) -- provides both pull interface and streaming (event-based) interface - -## Reading JSON using the pull interface - -There are several ways to use the pull (DOM-like) interface of picojson. - -The easiest way is to use the two-argument `parse` function. - -``` -std::string json = "[ \"hello JSON\" ]"; -picojson::value v; -std::string err = picojson::parse(v, json); -if (! err.empty()) { - std:cerr << err << std::endl; -} -``` - -Four-argument `parse` function accepts a pair of iterators, and returns the end position of the input. - -``` -const char* json = "{\"a\":1}"; -picojson::value v; -std::string err; -const char* json_end = picojson::parse(v, json, json + strlen(json), &err); -if (! err.empty()) { - std::cerr << err << std::endl; -} -``` - -``` -std::istream_iterator input(std::cin); -picojson::value v; -std::string err; -input = picojson::parse(v, input, std::istream_iterator(), &err); -if (! err.empty()) { - std::cerr << err << std::endl; -} -``` - -It is also possible to use the `>>` operator to parse the input, however this interface is not thread-safe. - -``` -picosjon::value v; -std::cin >> v; -std::string err = picojson::get_last_error(); -``` - -## Accessing the values - -Values of a JSON object is represented as instances of picojson::value class. - -
-namespace picojson {
-
-  class value {
-    ...
-
-  public:
-
-    typedef std::vector<value> array;
-    typedef std::map<std::string, value> object;
-
-    value();                               // create a null object
-    explicit value(bool b);                // create a boolean object
-    explicit value(double n);              // create a number object
-    explicit value(const std::string& s);  // create a string object
-    explicit value(const array& a);        // create an array object
-    explicit value(const object& o);       // create an "object"
-
-    bool is<picojson::null>() const;       // check if the object is "null"
-
-    bool is<bool>() const;                 // check if the object is a boolean
-    const bool& get<bool>() const;         // const accessor (usable only if the object is a boolean)
-    bool& get<bool>();                     // non-const accessor (usable only if the object is a boolean)
-
-    bool is<double>() const;               // check if the object is a number
-    const double& get<double>() const;     // const accessor (usable only if the object is a number)
-    double& get<double>();                 // non-const accessor (usable only if the object is a number)
-
-    bool is<std::string>() const;          // check if the object is a string
-    const std::string& get<std::string>() const;
-                                           // const accessor (usable only if the object is a string)
-    std::string& get<std::string>();       // non-const accessor (usable only if the object is a string)
-
-    bool is<array>() const;                // check if the object is an array
-    const array& get<array>() const;       // const accessor (usable only if the object is an array)
-    array& get<array>();                   // non-const accessor (usable only if the object is an array)
-
-    bool is<object>() const;               // check if the object is an "object"
-    const object& get<object>() const;     // const accessor (usable only if the object is an object)
-    object& get<object>();                 // non-const accessor (usable only if the object is an array)
-
-    bool evaluate_as_boolean() const;      // evaluates the object as a boolean
-
-    std::string serialize() const;         // returns the object in JSON representation
-    template void serialize(Iter os) const;
-                                           // serializes the object in JSON representation through an output iterator
-
-    std::string to_str() const;            // returns the object in string (for casual use)
-
-  };
-
-}
-
- -The code below parses a JSON string and prints the contents of the object. - -
-picojson::value v;
-
-// parse the input
-std::cin >> v;
-std::string err = picojson::get_last_error();
-if (! err.empty()) {
-  std::cerr << err << std::endl;
-  exit(1);
-}
-
-// check if the type of the value is "object"
-if (! v.is<picojson::object>()) {
-  std::cerr << "JSON is not an object" << std::endl;
-  exit(2);
-}
-
-// obtain a const reference to the map, and print the contents
-const picojson::value::object& obj = v.get<picojson::object>();
-for (picojson::value::object::const_iterator i = obj.begin();
-     i != obj.end();
-     ++i) {
-  std::cout << i->first << ': ' << i->second.to_str() << std::endl;
-}
-
- -Please note that the type check is mandatory; do not forget to check the type of the object by calling is<type>() before accessing the value by calling get<type>(). - -## Reading JSON using the streaming (event-driven) interface - -Please refer to the implementation of picojson::default_parse_context and picojson::null_parse_context. There is also an example (examples/streaming.cc) . - -## Serializing to JSON - -Instances of the picojson::value class can be serialized in three ways, to ostream, to std::string, or to an output iterator. - -
-picojson::value v;
-...
-std::cout << v;
-
- -
-picojson::value v;
-...
-std::string json = v.serialize();
-
- -
-picojson::value v;
-...
-v.serialize(std::ostream_iterator(std::cout));
-
- -## Experimental support for int64_t - -Experimental suport for int64_t becomes available if the code is compiled with preprocessor macro `PICOJSON_USE_INT64`. - -Turning on the feature will cause following changes to picojson: -- new constructor `picojson::value(int64_t)` is defined -- `is()` and `get()` become available -- numerics in JSON within the bounds of int64_t and not using `.` nor `e`/`E` are considered as int64 type - - the values are also avaliable as `double`s as well (i.e. all values which are `.is() == true` are also `.is() == true`) -- int64 values are converted to double once `get()` is called - -Enabling the feature should not cause compatibility problem with code that do not use the feature. - -## Further reading - -Examples can be found in the examples directory, and on the [Wiki](https://github.com/kazuho/picojson/wiki). Please add your favorite examples to the Wiki. diff --git a/deps/picojson/picojson.h b/deps/picojson/picojson.h deleted file mode 100644 index 48bb64e672..0000000000 --- a/deps/picojson/picojson.h +++ /dev/null @@ -1,1010 +0,0 @@ -/* - * Copyright 2009-2010 Cybozu Labs, Inc. - * Copyright 2011-2014 Kazuho Oku - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef picojson_h -#define picojson_h - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// for isnan/isinf -#if __cplusplus>=201103L -# include -#else -extern "C" { -# ifdef _MSC_VER -# include -# elif defined(__INTEL_COMPILER) -# include -# else -# include -# endif -} -#endif - -// experimental support for int64_t (see README.mkdn for detail) -#ifdef PICOJSON_USE_INT64 -# define __STDC_FORMAT_MACROS -# include -# include -#endif - -// to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 -#ifndef PICOJSON_USE_LOCALE -# define PICOJSON_USE_LOCALE 1 -#endif -#if PICOJSON_USE_LOCALE -extern "C" { -# include -} -#endif - -#ifndef PICOJSON_ASSERT -# define PICOJSON_ASSERT(e) do { if (! (e)) throw std::runtime_error(#e); } while (0) -#endif - -#ifdef _MSC_VER - #define SNPRINTF _snprintf_s - #pragma warning(push) - #pragma warning(disable : 4244) // conversion from int to char - #pragma warning(disable : 4127) // conditional expression is constant - #pragma warning(disable : 4702) // unreachable code -#else - #define SNPRINTF snprintf -#endif - -namespace picojson { - - enum { - null_type, - boolean_type, - number_type, - string_type, - array_type, - object_type -#ifdef PICOJSON_USE_INT64 - , int64_type -#endif - }; - - enum { - INDENT_WIDTH = 2 - }; - - struct null {}; - - class value { - public: - typedef std::vector array; - typedef std::map object; - union _storage { - bool boolean_; - double number_; -#ifdef PICOJSON_USE_INT64 - int64_t int64_; -#endif - std::string* string_; - array* array_; - object* object_; - }; - protected: - int type_; - _storage u_; - public: - value(); - value(int type, bool); - explicit value(bool b); -#ifdef PICOJSON_USE_INT64 - explicit value(int64_t i); -#endif - explicit value(double n); - explicit value(const std::string& s); - explicit value(const array& a); - explicit value(const object& o); - explicit value(const char* s); - value(const char* s, size_t len); - ~value(); - value(const value& x); - value& operator=(const value& x); - void swap(value& x); - template bool is() const; - template const T& get() const; - template T& get(); - bool evaluate_as_boolean() const; - const value& get(size_t idx) const; - const value& get(const std::string& key) const; - value& get(size_t idx); - value& get(const std::string& key); - - bool contains(size_t idx) const; - bool contains(const std::string& key) const; - std::string to_str() const; - template void serialize(Iter os, bool prettify = false) const; - std::string serialize(bool prettify = false) const; - private: - template value(const T*); // intentionally defined to block implicit conversion of pointer to bool - template static void _indent(Iter os, int indent); - template void _serialize(Iter os, int indent) const; - std::string _serialize(int indent) const; - }; - - typedef value::array array; - typedef value::object object; - - inline value::value() : type_(null_type) {} - - inline value::value(int type, bool) : type_(type) { - switch (type) { -#define INIT(p, v) case p##type: u_.p = v; break - INIT(boolean_, false); - INIT(number_, 0.0); -#ifdef PICOJSON_USE_INT64 - INIT(int64_, 0); -#endif - INIT(string_, new std::string()); - INIT(array_, new array()); - INIT(object_, new object()); -#undef INIT - default: break; - } - } - - inline value::value(bool b) : type_(boolean_type) { - u_.boolean_ = b; - } - -#ifdef PICOJSON_USE_INT64 - inline value::value(int64_t i) : type_(int64_type) { - u_.int64_ = i; - } -#endif - - inline value::value(double n) : type_(number_type) { - if ( -#ifdef _MSC_VER - ! _finite(n) -#elif __cplusplus>=201103L || !(defined(isnan) && defined(isinf)) - std::isnan(n) || std::isinf(n) -#else - isnan(n) || isinf(n) -#endif - ) { - throw std::overflow_error(""); - } - u_.number_ = n; - } - - inline value::value(const std::string& s) : type_(string_type) { - u_.string_ = new std::string(s); - } - - inline value::value(const array& a) : type_(array_type) { - u_.array_ = new array(a); - } - - inline value::value(const object& o) : type_(object_type) { - u_.object_ = new object(o); - } - - inline value::value(const char* s) : type_(string_type) { - u_.string_ = new std::string(s); - } - - inline value::value(const char* s, size_t len) : type_(string_type) { - u_.string_ = new std::string(s, len); - } - - inline value::~value() { - switch (type_) { -#define DEINIT(p) case p##type: delete u_.p; break - DEINIT(string_); - DEINIT(array_); - DEINIT(object_); -#undef DEINIT - default: break; - } - } - - inline value::value(const value& x) : type_(x.type_) { - switch (type_) { -#define INIT(p, v) case p##type: u_.p = v; break - INIT(string_, new std::string(*x.u_.string_)); - INIT(array_, new array(*x.u_.array_)); - INIT(object_, new object(*x.u_.object_)); -#undef INIT - default: - u_ = x.u_; - break; - } - } - - inline value& value::operator=(const value& x) { - if (this != &x) { - value t(x); - swap(t); - } - return *this; - } - - inline void value::swap(value& x) { - std::swap(type_, x.type_); - std::swap(u_, x.u_); - } - -#define IS(ctype, jtype) \ - template <> inline bool value::is() const { \ - return type_ == jtype##_type; \ - } - IS(null, null) - IS(bool, boolean) -#ifdef PICOJSON_USE_INT64 - IS(int64_t, int64) -#endif - IS(std::string, string) - IS(array, array) - IS(object, object) -#undef IS - template <> inline bool value::is() const { - return type_ == number_type -#ifdef PICOJSON_USE_INT64 - || type_ == int64_type -#endif - ; - } - -#define GET(ctype, var) \ - template <> inline const ctype& value::get() const { \ - PICOJSON_ASSERT("type mismatch! call is() before get()" \ - && is()); \ - return var; \ - } \ - template <> inline ctype& value::get() { \ - PICOJSON_ASSERT("type mismatch! call is() before get()" \ - && is()); \ - return var; \ - } - GET(bool, u_.boolean_) - GET(std::string, *u_.string_) - GET(array, *u_.array_) - GET(object, *u_.object_) -#ifdef PICOJSON_USE_INT64 - GET(double, (type_ == int64_type && (const_cast(this)->type_ = number_type, const_cast(this)->u_.number_ = u_.int64_), u_.number_)) - GET(int64_t, u_.int64_) -#else - GET(double, u_.number_) -#endif -#undef GET - - inline bool value::evaluate_as_boolean() const { - switch (type_) { - case null_type: - return false; - case boolean_type: - return u_.boolean_; - case number_type: - return u_.number_ != 0; - case string_type: - return ! u_.string_->empty(); - default: - return true; - } - } - - inline const value& value::get(size_t idx) const { - static value s_null; - PICOJSON_ASSERT(is()); - return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; - } - - inline value& value::get(size_t idx) { - static value s_null; - PICOJSON_ASSERT(is()); - return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; - } - - inline const value& value::get(const std::string& key) const { - static value s_null; - PICOJSON_ASSERT(is()); - object::const_iterator i = u_.object_->find(key); - return i != u_.object_->end() ? i->second : s_null; - } - - inline value& value::get(const std::string& key) { - static value s_null; - PICOJSON_ASSERT(is()); - object::iterator i = u_.object_->find(key); - return i != u_.object_->end() ? i->second : s_null; - } - - inline bool value::contains(size_t idx) const { - PICOJSON_ASSERT(is()); - return idx < u_.array_->size(); - } - - inline bool value::contains(const std::string& key) const { - PICOJSON_ASSERT(is()); - object::const_iterator i = u_.object_->find(key); - return i != u_.object_->end(); - } - - inline std::string value::to_str() const { - switch (type_) { - case null_type: return "null"; - case boolean_type: return u_.boolean_ ? "true" : "false"; -#ifdef PICOJSON_USE_INT64 - case int64_type: { - char buf[sizeof("-9223372036854775808")]; - SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); - return buf; - } -#endif - case number_type: { - char buf[256]; - double tmp; - SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); -#if PICOJSON_USE_LOCALE - char *decimal_point = localeconv()->decimal_point; - if (strcmp(decimal_point, ".") != 0) { - size_t decimal_point_len = strlen(decimal_point); - for (char *p = buf; *p != '\0'; ++p) { - if (strncmp(p, decimal_point, decimal_point_len) == 0) { - return std::string(buf, p) + "." + (p + decimal_point_len); - } - } - } -#endif - return buf; - } - case string_type: return *u_.string_; - case array_type: return "array"; - case object_type: return "object"; - default: PICOJSON_ASSERT(0); -#ifdef _MSC_VER - __assume(0); -#endif - } - return std::string(); - } - - template void copy(const std::string& s, Iter oi) { - std::copy(s.begin(), s.end(), oi); - } - - template void serialize_str(const std::string& s, Iter oi) { - *oi++ = '"'; - for (std::string::const_iterator i = s.begin(); i != s.end(); ++i) { - switch (*i) { -#define MAP(val, sym) case val: copy(sym, oi); break - MAP('"', "\\\""); - MAP('\\', "\\\\"); - MAP('/', "\\/"); - MAP('\b', "\\b"); - MAP('\f', "\\f"); - MAP('\n', "\\n"); - MAP('\r', "\\r"); - MAP('\t', "\\t"); -#undef MAP - default: - if (static_cast(*i) < 0x20 || *i == 0x7f) { - char buf[7]; - SNPRINTF(buf, sizeof(buf), "\\u%04x", *i & 0xff); - copy(buf, buf + 6, oi); - } else { - *oi++ = *i; - } - break; - } - } - *oi++ = '"'; - } - - template void value::serialize(Iter oi, bool prettify) const { - return _serialize(oi, prettify ? 0 : -1); - } - - inline std::string value::serialize(bool prettify) const { - return _serialize(prettify ? 0 : -1); - } - - template void value::_indent(Iter oi, int indent) { - *oi++ = '\n'; - for (int i = 0; i < indent * INDENT_WIDTH; ++i) { - *oi++ = ' '; - } - } - - template void value::_serialize(Iter oi, int indent) const { - switch (type_) { - case string_type: - serialize_str(*u_.string_, oi); - break; - case array_type: { - *oi++ = '['; - if (indent != -1) { - ++indent; - } - for (array::const_iterator i = u_.array_->begin(); - i != u_.array_->end(); - ++i) { - if (i != u_.array_->begin()) { - *oi++ = ','; - } - if (indent != -1) { - _indent(oi, indent); - } - i->_serialize(oi, indent); - } - if (indent != -1) { - --indent; - if (! u_.array_->empty()) { - _indent(oi, indent); - } - } - *oi++ = ']'; - break; - } - case object_type: { - *oi++ = '{'; - if (indent != -1) { - ++indent; - } - for (object::const_iterator i = u_.object_->begin(); - i != u_.object_->end(); - ++i) { - if (i != u_.object_->begin()) { - *oi++ = ','; - } - if (indent != -1) { - _indent(oi, indent); - } - serialize_str(i->first, oi); - *oi++ = ':'; - if (indent != -1) { - *oi++ = ' '; - } - i->second._serialize(oi, indent); - } - if (indent != -1) { - --indent; - if (! u_.object_->empty()) { - _indent(oi, indent); - } - } - *oi++ = '}'; - break; - } - default: - copy(to_str(), oi); - break; - } - if (indent == 0) { - *oi++ = '\n'; - } - } - - inline std::string value::_serialize(int indent) const { - std::string s; - _serialize(std::back_inserter(s), indent); - return s; - } - - template class input { - protected: - Iter cur_, end_; - int last_ch_; - bool ungot_; - int line_; - public: - input(const Iter& first, const Iter& last) : cur_(first), end_(last), last_ch_(-1), ungot_(false), line_(1) {} - int getc() { - if (ungot_) { - ungot_ = false; - return last_ch_; - } - if (cur_ == end_) { - last_ch_ = -1; - return -1; - } - if (last_ch_ == '\n') { - line_++; - } - last_ch_ = *cur_ & 0xff; - ++cur_; - return last_ch_; - } - void ungetc() { - if (last_ch_ != -1) { - PICOJSON_ASSERT(! ungot_); - ungot_ = true; - } - } - Iter cur() const { return cur_; } - int line() const { return line_; } - void skip_ws() { - while (1) { - int ch = getc(); - if (! (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { - ungetc(); - break; - } - } - } - bool expect(int expect) { - skip_ws(); - if (getc() != expect) { - ungetc(); - return false; - } - return true; - } - bool match(const std::string& pattern) { - for (std::string::const_iterator pi(pattern.begin()); - pi != pattern.end(); - ++pi) { - if (getc() != *pi) { - ungetc(); - return false; - } - } - return true; - } - }; - - template inline int _parse_quadhex(input &in) { - int uni_ch = 0, hex; - for (int i = 0; i < 4; i++) { - if ((hex = in.getc()) == -1) { - return -1; - } - if ('0' <= hex && hex <= '9') { - hex -= '0'; - } else if ('A' <= hex && hex <= 'F') { - hex -= 'A' - 0xa; - } else if ('a' <= hex && hex <= 'f') { - hex -= 'a' - 0xa; - } else { - in.ungetc(); - return -1; - } - uni_ch = uni_ch * 16 + hex; - } - return uni_ch; - } - - template inline bool _parse_codepoint(String& out, input& in) { - int uni_ch; - if ((uni_ch = _parse_quadhex(in)) == -1) { - return false; - } - if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { - if (0xdc00 <= uni_ch) { - // a second 16-bit of a surrogate pair appeared - return false; - } - // first 16-bit of surrogate pair, get the next one - if (in.getc() != '\\' || in.getc() != 'u') { - in.ungetc(); - return false; - } - int second = _parse_quadhex(in); - if (! (0xdc00 <= second && second <= 0xdfff)) { - return false; - } - uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); - uni_ch += 0x10000; - } - if (uni_ch < 0x80) { - out.push_back(uni_ch); - } else { - if (uni_ch < 0x800) { - out.push_back(0xc0 | (uni_ch >> 6)); - } else { - if (uni_ch < 0x10000) { - out.push_back(0xe0 | (uni_ch >> 12)); - } else { - out.push_back(0xf0 | (uni_ch >> 18)); - out.push_back(0x80 | ((uni_ch >> 12) & 0x3f)); - } - out.push_back(0x80 | ((uni_ch >> 6) & 0x3f)); - } - out.push_back(0x80 | (uni_ch & 0x3f)); - } - return true; - } - - template inline bool _parse_string(String& out, input& in) { - while (1) { - int ch = in.getc(); - if (ch < ' ') { - in.ungetc(); - return false; - } else if (ch == '"') { - return true; - } else if (ch == '\\') { - if ((ch = in.getc()) == -1) { - return false; - } - switch (ch) { -#define MAP(sym, val) case sym: out.push_back(val); break - MAP('"', '\"'); - MAP('\\', '\\'); - MAP('/', '/'); - MAP('b', '\b'); - MAP('f', '\f'); - MAP('n', '\n'); - MAP('r', '\r'); - MAP('t', '\t'); -#undef MAP - case 'u': - if (! _parse_codepoint(out, in)) { - return false; - } - break; - default: - return false; - } - } else { - out.push_back(ch); - } - } - return false; - } - - template inline bool _parse_array(Context& ctx, input& in) { - if (! ctx.parse_array_start()) { - return false; - } - size_t idx = 0; - if (in.expect(']')) { - return ctx.parse_array_stop(idx); - } - do { - if (! ctx.parse_array_item(in, idx)) { - return false; - } - idx++; - } while (in.expect(',')); - return in.expect(']') && ctx.parse_array_stop(idx); - } - - template inline bool _parse_object(Context& ctx, input& in) { - if (! ctx.parse_object_start()) { - return false; - } - if (in.expect('}')) { - return true; - } - do { - std::string key; - if (! in.expect('"') - || ! _parse_string(key, in) - || ! in.expect(':')) { - return false; - } - if (! ctx.parse_object_item(in, key)) { - return false; - } - } while (in.expect(',')); - return in.expect('}'); - } - - template inline std::string _parse_number(input& in) { - std::string num_str; - while (1) { - int ch = in.getc(); - if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' - || ch == 'e' || ch == 'E') { - num_str.push_back(ch); - } else if (ch == '.') { -#if PICOJSON_USE_LOCALE - num_str += localeconv()->decimal_point; -#else - num_str.push_back('.'); -#endif - } else { - in.ungetc(); - break; - } - } - return num_str; - } - - template inline bool _parse(Context& ctx, input& in) { - in.skip_ws(); - int ch = in.getc(); - switch (ch) { -#define IS(ch, text, op) case ch: \ - if (in.match(text) && op) { \ - return true; \ - } else { \ - return false; \ - } - IS('n', "ull", ctx.set_null()); - IS('f', "alse", ctx.set_bool(false)); - IS('t', "rue", ctx.set_bool(true)); -#undef IS - case '"': - return ctx.parse_string(in); - case '[': - return _parse_array(ctx, in); - case '{': - return _parse_object(ctx, in); - default: - if (('0' <= ch && ch <= '9') || ch == '-') { - double f; - char *endp; - in.ungetc(); - std::string num_str = _parse_number(in); - if (num_str.empty()) { - return false; - } -#ifdef PICOJSON_USE_INT64 - { - errno = 0; - intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); - if (errno == 0 - && std::numeric_limits::min() <= ival - && ival <= std::numeric_limits::max() - && endp == num_str.c_str() + num_str.size()) { - ctx.set_int64(ival); - return true; - } - } -#endif - f = strtod(num_str.c_str(), &endp); - if (endp == num_str.c_str() + num_str.size()) { - ctx.set_number(f); - return true; - } - return false; - } - break; - } - in.ungetc(); - return false; - } - - class deny_parse_context { - public: - bool set_null() { return false; } - bool set_bool(bool) { return false; } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t) { return false; } -#endif - bool set_number(double) { return false; } - template bool parse_string(input&) { return false; } - bool parse_array_start() { return false; } - template bool parse_array_item(input&, size_t) { - return false; - } - bool parse_array_stop(size_t) { return false; } - bool parse_object_start() { return false; } - template bool parse_object_item(input&, const std::string&) { - return false; - } - }; - - class default_parse_context { - protected: - value* out_; - public: - default_parse_context(value* out) : out_(out) {} - bool set_null() { - *out_ = value(); - return true; - } - bool set_bool(bool b) { - *out_ = value(b); - return true; - } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t i) { - *out_ = value(i); - return true; - } -#endif - bool set_number(double f) { - *out_ = value(f); - return true; - } - template bool parse_string(input& in) { - *out_ = value(string_type, false); - return _parse_string(out_->get(), in); - } - bool parse_array_start() { - *out_ = value(array_type, false); - return true; - } - template bool parse_array_item(input& in, size_t) { - array& a = out_->get(); - a.push_back(value()); - default_parse_context ctx(&a.back()); - return _parse(ctx, in); - } - bool parse_array_stop(size_t) { return true; } - bool parse_object_start() { - *out_ = value(object_type, false); - return true; - } - template bool parse_object_item(input& in, const std::string& key) { - object& o = out_->get(); - default_parse_context ctx(&o[key]); - return _parse(ctx, in); - } - private: - default_parse_context(const default_parse_context&); - default_parse_context& operator=(const default_parse_context&); - }; - - class null_parse_context { - public: - struct dummy_str { - void push_back(int) {} - }; - public: - null_parse_context() {} - bool set_null() { return true; } - bool set_bool(bool) { return true; } -#ifdef PICOJSON_USE_INT64 - bool set_int64(int64_t) { return true; } -#endif - bool set_number(double) { return true; } - template bool parse_string(input& in) { - dummy_str s; - return _parse_string(s, in); - } - bool parse_array_start() { return true; } - template bool parse_array_item(input& in, size_t) { - return _parse(*this, in); - } - bool parse_array_stop(size_t) { return true; } - bool parse_object_start() { return true; } - template bool parse_object_item(input& in, const std::string&) { - return _parse(*this, in); - } - private: - null_parse_context(const null_parse_context&); - null_parse_context& operator=(const null_parse_context&); - }; - - // obsolete, use the version below - template inline std::string parse(value& out, Iter& pos, const Iter& last) { - std::string err; - pos = parse(out, pos, last, &err); - return err; - } - - template inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { - input in(first, last); - if (! _parse(ctx, in) && err != NULL) { - char buf[64]; - SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); - *err = buf; - while (1) { - int ch = in.getc(); - if (ch == -1 || ch == '\n') { - break; - } else if (ch >= ' ') { - err->push_back(ch); - } - } - } - return in.cur(); - } - - template inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { - default_parse_context ctx(&out); - return _parse(ctx, first, last, err); - } - - inline std::string parse(value& out, const std::string& s) { - std::string err; - parse(out, s.begin(), s.end(), &err); - return err; - } - - inline std::string parse(value& out, std::istream& is) { - std::string err; - parse(out, std::istreambuf_iterator(is.rdbuf()), - std::istreambuf_iterator(), &err); - return err; - } - - template struct last_error_t { - static std::string s; - }; - template std::string last_error_t::s; - - inline void set_last_error(const std::string& s) { - last_error_t::s = s; - } - - inline const std::string& get_last_error() { - return last_error_t::s; - } - - inline bool operator==(const value& x, const value& y) { - if (x.is()) - return y.is(); -#define PICOJSON_CMP(type) \ - if (x.is()) \ - return y.is() && x.get() == y.get() - PICOJSON_CMP(bool); - PICOJSON_CMP(double); - PICOJSON_CMP(std::string); - PICOJSON_CMP(array); - PICOJSON_CMP(object); -#undef PICOJSON_CMP - PICOJSON_ASSERT(0); -#ifdef _MSC_VER - __assume(0); -#endif - return false; - } - - inline bool operator!=(const value& x, const value& y) { - return ! (x == y); - } -} - -namespace std { - template<> inline void swap(picojson::value& x, picojson::value& y) - { - x.swap(y); - } -} - -inline std::istream& operator>>(std::istream& is, picojson::value& x) -{ - picojson::set_last_error(std::string()); - std::string err = picojson::parse(x, is); - if (! err.empty()) { - picojson::set_last_error(err); - is.setstate(std::ios::failbit); - } - return is; -} - -inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) -{ - x.serialize(std::ostream_iterator(os)); - return os; -} -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -#endif diff --git a/deps/picojson/picotest/picotest.c b/deps/picojson/picotest/picotest.c deleted file mode 100644 index d1fe699d5a..0000000000 --- a/deps/picojson/picotest/picotest.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2014 DeNA Co., Ltd. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#include -#include -#include -#include "picotest.h" - -struct test_t { - int num_tests; - int failed; -}; -struct test_t main_tests, *cur_tests = &main_tests; -static int test_level = 0; - -static void indent(void) -{ - int i; - for (i = 0; i != test_level; ++i) - printf(" "); -} - -__attribute__((format (printf, 1, 2))) -void note(const char *fmt, ...) -{ - va_list arg; - - indent(); - printf("# "); - - va_start(arg, fmt); - vprintf(fmt, arg); - va_end(arg); - - printf("\n"); -} - -__attribute__((format (printf, 2, 3))) -void _ok(int cond, const char *fmt, ...) -{ - va_list arg; - - if (! cond) - cur_tests->failed = 1; - indent(); - - printf("%s %d - ", cond ? "ok" : "not ok", ++cur_tests->num_tests); - va_start(arg, fmt); - vprintf(fmt, arg); - va_end(arg); - - printf("\n"); -} - -int done_testing(void) -{ - indent(); - printf("1..%d\n", cur_tests->num_tests); - return cur_tests->failed; -} - -void subtest(const char *name, void (*cb)(void)) -{ - struct test_t test = {}, *parent_tests; - - parent_tests = cur_tests; - cur_tests = &test; - ++test_level; - - note("Subtest: %s", name); - - cb(); - - done_testing(); - - --test_level; - cur_tests = parent_tests; - if (test.failed) - cur_tests->failed = 1; - _ok(! test.failed, "%s", name); -} diff --git a/deps/picojson/picotest/picotest.h b/deps/picojson/picotest/picotest.h deleted file mode 100644 index 22ab02fb2d..0000000000 --- a/deps/picojson/picotest/picotest.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2014 DeNA Co., Ltd. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ -#ifndef picotest_h -#define picotest_h - -#ifdef __cplusplus -extern "C" { -#endif - -void note(const char *fmt, ...) __attribute__((format (printf, 1, 2))); -void _ok(int cond, const char *fmt, ...) __attribute__((format (printf, 2, 3))); -#define ok(cond) _ok(cond, "%s %d", __FILE__, __LINE__) -int done_testing(void); -void subtest(const char *name, void (*cb)(void)); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/deps/picojson/test.cc b/deps/picojson/test.cc deleted file mode 100644 index ed9656d2cb..0000000000 --- a/deps/picojson/test.cc +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright 2009-2010 Cybozu Labs, Inc. - * Copyright 2011-2014 Kazuho Oku, Yasuhiro Matsumoto, Shigeo Mitsunari - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -#include "picojson.h" -#include "picotest/picotest.h" - -#ifdef _MSC_VER - #pragma warning(disable : 4127) // conditional expression is constant -#endif - -using namespace std; - -#define is(x, y, name) _ok((x) == (y), name) - -#include -#include -#include -#include - -int main(void) -{ -#if PICOJSON_USE_LOCALE - setlocale(LC_ALL, ""); -#endif - - // constructors -#define TEST(expr, expected) \ - is(picojson::value expr .serialize(), string(expected), "picojson::value" #expr) - - TEST( (true), "true"); - TEST( (false), "false"); - TEST( (42.0), "42"); - TEST( (string("hello")), "\"hello\""); - TEST( ("hello"), "\"hello\""); - TEST( ("hello", 4), "\"hell\""); - - { - double a = 1; - for (int i = 0; i < 1024; i++) { - picojson::value vi(a); - std::stringstream ss; - ss << vi; - picojson::value vo; - ss >> vo; - double b = vo.get(); - if ((i < 53 && a != b) || fabs(a - b) / b > 1e-8) { - printf("ng i=%d a=%.18e b=%.18e\n", i, a, b); - } - a *= 2; - } - } - -#undef TEST - -#define TEST(in, type, cmp, serialize_test) { \ - picojson::value v; \ - const char* s = in; \ - string err = picojson::parse(v, s, s + strlen(s)); \ - _ok(err.empty(), in " no error"); \ - _ok(v.is(), in " check type"); \ - is(v.get(), static_cast(cmp), in " correct output"); \ - is(*s, '\0', in " read to eof"); \ - if (serialize_test) { \ - is(v.serialize(), string(in), in " serialize"); \ - } \ - } - TEST("false", bool, false, true); - TEST("true", bool, true, true); - TEST("90.5", double, 90.5, false); - TEST("1.7976931348623157e+308", double, DBL_MAX, false); - TEST("\"hello\"", string, string("hello"), true); - TEST("\"\\\"\\\\\\/\\b\\f\\n\\r\\t\"", string, string("\"\\/\b\f\n\r\t"), - true); - TEST("\"\\u0061\\u30af\\u30ea\\u30b9\"", string, - string("a\xe3\x82\xaf\xe3\x83\xaa\xe3\x82\xb9"), false); - TEST("\"\\ud840\\udc0b\"", string, string("\xf0\xa0\x80\x8b"), false); -#ifdef PICOJSON_USE_INT64 - TEST("0", int64_t, 0, true); - TEST("-9223372036854775808", int64_t, std::numeric_limits::min(), true); - TEST("9223372036854775807", int64_t, std::numeric_limits::max(), true); -#endif -#undef TEST - -#define TEST(type, expr) { \ - picojson::value v; \ - const char *s = expr; \ - string err = picojson::parse(v, s, s + strlen(s)); \ - _ok(err.empty(), "empty " #type " no error"); \ - _ok(v.is(), "empty " #type " check type"); \ - _ok(v.get().empty(), "check " #type " array size"); \ - } - TEST(array, "[]"); - TEST(object, "{}"); -#undef TEST - - { - picojson::value v; - const char *s = "[1,true,\"hello\"]"; - string err = picojson::parse(v, s, s + strlen(s)); - _ok(err.empty(), "array no error"); - _ok(v.is(), "array check type"); - is(v.get().size(), size_t(3), "check array size"); - _ok(v.contains(0), "check contains array[0]"); - _ok(v.get(0).is(), "check array[0] type"); - is(v.get(0).get(), 1.0, "check array[0] value"); - _ok(v.contains(1), "check contains array[1]"); - _ok(v.get(1).is(), "check array[1] type"); - _ok(v.get(1).get(), "check array[1] value"); - _ok(v.contains(2), "check contains array[2]"); - _ok(v.get(2).is(), "check array[2] type"); - is(v.get(2).get(), string("hello"), "check array[2] value"); - _ok(!v.contains(3), "check not contains array[3]"); - } - - { - picojson::value v; - const char *s = "{ \"a\": true }"; - string err = picojson::parse(v, s, s + strlen(s)); - _ok(err.empty(), "object no error"); - _ok(v.is(), "object check type"); - is(v.get().size(), size_t(1), "check object size"); - _ok(v.contains("a"), "check contains property"); - _ok(v.get("a").is(), "check bool property exists"); - is(v.get("a").get(), true, "check bool property value"); - is(v.serialize(), string("{\"a\":true}"), "serialize object"); - _ok(!v.contains("z"), "check not contains property"); - } - -#define TEST(json, msg) do { \ - picojson::value v; \ - const char *s = json; \ - string err = picojson::parse(v, s, s + strlen(s)); \ - is(err, string("syntax error at line " msg), msg); \ - } while (0) - TEST("falsoa", "1 near: oa"); - TEST("{]", "1 near: ]"); - TEST("\n\bbell", "2 near: bell"); - TEST("\"abc\nd\"", "1 near: "); -#undef TEST - - { - picojson::value v1, v2; - const char *s; - string err; - s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; - err = picojson::parse(v1, s, s + strlen(s)); - s = "{ \"d\": 2.0, \"b\": true, \"a\": [1,2,\"three\"] }"; - err = picojson::parse(v2, s, s + strlen(s)); - _ok((v1 == v2), "check == operator in deep comparison"); - } - - { - picojson::value v1, v2; - const char *s; - string err; - s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; - err = picojson::parse(v1, s, s + strlen(s)); - s = "{ \"d\": 2.0, \"a\": [1,\"three\"], \"b\": true }"; - err = picojson::parse(v2, s, s + strlen(s)); - _ok((v1 != v2), "check != operator for array in deep comparison"); - } - - { - picojson::value v1, v2; - const char *s; - string err; - s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; - err = picojson::parse(v1, s, s + strlen(s)); - s = "{ \"d\": 2.0, \"a\": [1,2,\"three\"], \"b\": false }"; - err = picojson::parse(v2, s, s + strlen(s)); - _ok((v1 != v2), "check != operator for object in deep comparison"); - } - - { - picojson::value v1, v2; - const char *s; - string err; - s = "{ \"b\": true, \"a\": [1,2,\"three\"], \"d\": 2 }"; - err = picojson::parse(v1, s, s + strlen(s)); - picojson::object& o = v1.get(); - o.erase("b"); - picojson::array& a = o["a"].get(); - picojson::array::iterator i; - i = std::remove(a.begin(), a.end(), picojson::value(std::string("three"))); - a.erase(i, a.end()); - s = "{ \"a\": [1,2], \"d\": 2 }"; - err = picojson::parse(v2, s, s + strlen(s)); - _ok((v1 == v2), "check erase()"); - } - - _ok(picojson::value(3.0).serialize() == "3", - "integral number should be serialized as a integer"); - - { - const char* s = "{ \"a\": [1,2], \"d\": 2 }"; - picojson::null_parse_context ctx; - string err; - picojson::_parse(ctx, s, s + strlen(s), &err); - _ok(err.empty(), "null_parse_context"); - } - - { - picojson::value v1, v2; - v1 = picojson::value(true); - swap(v1, v2); - _ok(v1.is(), "swap (null)"); - _ok(v2.get() == true, "swap (bool)"); - - v1 = picojson::value("a"); - v2 = picojson::value(1.0); - swap(v1, v2); - _ok(v1.get() == 1.0, "swap (dobule)"); - _ok(v2.get() == "a", "swap (string)"); - - v1 = picojson::value(picojson::object()); - v2 = picojson::value(picojson::array()); - swap(v1, v2); - _ok(v1.is(), "swap (array)"); - _ok(v2.is(), "swap (object)"); - } - - { - picojson::value v; - const char *s = "{ \"a\": 1, \"b\": [ 2, { \"b1\": \"abc\" } ], \"c\": {}, \"d\": [] }"; - string err; - err = picojson::parse(v, s, s + strlen(s)); - _ok(err.empty(), "parse test data for prettifying output"); - _ok(v.serialize() == "{\"a\":1,\"b\":[2,{\"b1\":\"abc\"}],\"c\":{},\"d\":[]}", "non-prettifying output"); - _ok(v.serialize(true) == "{\n \"a\": 1,\n \"b\": [\n 2,\n {\n \"b1\": \"abc\"\n }\n ],\n \"c\": {},\n \"d\": []\n}\n", "prettifying output"); - } - - try { - picojson::value v(std::numeric_limits::quiet_NaN()); - _ok(false, "should not accept NaN"); - } catch (std::overflow_error e) { - _ok(true, "should not accept NaN"); - } - - try { - picojson::value v(std::numeric_limits::infinity()); - _ok(false, "should not accept infinity"); - } catch (std::overflow_error e) { - _ok(true, "should not accept infinity"); - } - - try { - picojson::value v(123.); - _ok(! v.is(), "is() should return false"); - v.get(); - _ok(false, "get() should raise an error"); - } catch (std::runtime_error e) { - _ok(true, "get() should raise an error"); - } - -#ifdef PICOJSON_USE_INT64 - { - picojson::value v1((int64_t)123); - _ok(v1.is(), "is int64_t"); - _ok(v1.is(), "is double as well"); - _ok(v1.serialize() == "123", "serialize the value"); - _ok(v1.get() == 123, "value is correct as int64_t"); - _ok(v1.get(), "value is correct as double"); - - _ok(! v1.is(), "is no more int64_type once get() is called"); - _ok(v1.is(), "and is still a double"); - - const char *s = "-9223372036854775809"; - _ok(picojson::parse(v1, s, s + strlen(s)).empty(), "parse underflowing int64_t"); - _ok(! v1.is(), "underflowing int is not int64_t"); - _ok(v1.is(), "underflowing int is double"); - _ok(v1.get() + 9.22337203685478e+18 < 65536, "double value is somewhat correct"); - } -#endif - - { - picojson::value v; - std::string err = picojson::parse(v, "[ 1, \"abc\" ]"); - _ok(err.empty(), "simple API no error"); - _ok(v.is(), "simple API return type is array"); - is(v.get().size(), 2, "simple API array size"); - _ok(v.get()[0].is(), "simple API type #0"); - is(v.get()[0].get(), 1, "simple API value #0"); - _ok(v.get()[1].is(), "simple API type #1"); - is(v.get()[1].get(), "abc", "simple API value #1"); - } - - return done_testing(); -} diff --git a/scripts/build_picojson.bat b/scripts/build_picojson.bat deleted file mode 100644 index 38f4ffae98..0000000000 --- a/scripts/build_picojson.bat +++ /dev/null @@ -1,46 +0,0 @@ - -set picojson_version=1.3.0 -call %* - -:get-version - set version=%picojson_version% - goto :EOF - -:build -setlocal -set platform=%1 -set build_type=%2 -set vs_version=%3 -set dynamic_runtime=%4 - -set scriptdir=%~dp0 -call "%scriptdir%_init.bat" %platform% %build_type% %vs_version% -if %ERRORLEVEL% NEQ 0 goto :error -call "%scriptdir%utils.bat" :setup_visual_studio %vs_version% -if %ERRORLEVEL% NEQ 0 goto :error - -set DEPS_DIR=%scriptdir%..\deps -set PICOJSON_SOURCE_DIR=%DEPS_DIR%\picojson -set PICOJSON_INSTALL_DIR=%scriptdir%..\deps-build\%build_dir%\picojson - -cd "%PICOJSON_SOURCE_DIR%" - -rd /S /Q %PICOJSON_INSTALL_DIR% -md %PICOJSON_INSTALL_DIR% -md %PICOJSON_INSTALL_DIR%\include -copy %PICOJSON_SOURCE_DIR%\picojson.h %PICOJSON_INSTALL_DIR%\include - -cd "%curdir%" - -echo === archiving the library -call "%scriptdir%utils.bat" :zip_file picojson %picojson_version% -if %ERRORLEVEL% NEQ 0 goto :error - -goto :success - -:success -exit /b 0 - -:error -cd "%curdir%" -exit /b 1 diff --git a/scripts/build_picojson.sh b/scripts/build_picojson.sh deleted file mode 100755 index 5ded316b86..0000000000 --- a/scripts/build_picojson.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source "$DIR/_init.sh" "$@" -source "$DIR/utils.sh" - -PICOJSON_SRC_VERSION=1.3.0 -PICOJSON_BUILD_VERSION=1 -PICOJSON_VERSION=$PICOJSON_SRC_VERSION.$PICOJSON_BUILD_VERSION - -SOURCE_DIR="$DIR/../deps/picojson" -INSTALL_DIR="$DEPENDENCY_DIR/picojson" - -mkdir -p "${INSTALL_DIR}/include" -cp "$SOURCE_DIR/picojson.h" "${INSTALL_DIR}/include" - -echo === zip_file "picojson" "$PICOJSON_VERSION" "$target" -zip_file "picojson" "$PICOJSON_VERSION" "$target" From 2c285bfe46dea1e9b6449390d5f2b6079a3146e9 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 30 Oct 2024 10:13:59 -0700 Subject: [PATCH 60/74] Fix test case --- tests/test_unit_logger.c | 530 +++++++++++++++++++-------------------- 1 file changed, 265 insertions(+), 265 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 893213420f..bdeed1f5f4 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -1,265 +1,265 @@ -/* - * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. - */ - - -#include "utils/test_setup.h" -#include -#include "memory.h" - -#ifndef _WIN32 -#include -#endif - - -/** - * Tests converting a string representation of log level to the log level enum - */ -void test_log_str_to_level(void **unused) { - assert_int_equal(log_from_str_to_level("TRACE"), SF_LOG_TRACE); - assert_int_equal(log_from_str_to_level("DEBUG"), SF_LOG_DEBUG); - assert_int_equal(log_from_str_to_level("INFO"), SF_LOG_INFO); - assert_int_equal(log_from_str_to_level("wArN"), SF_LOG_WARN); - assert_int_equal(log_from_str_to_level("erroR"), SF_LOG_ERROR); - assert_int_equal(log_from_str_to_level("fatal"), SF_LOG_FATAL); - - /* negative */ - assert_int_equal(log_from_str_to_level("hahahaha"), SF_LOG_FATAL); - assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); -} - -/** - * Tests log settings with invalid client config filepath - */ -void test_invalid_client_config_path(void** unused) { - char configFilePath[] = "fakePath.json"; - - // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; - sf_bool result = load_client_config(configFilePath, &clientConfig); - assert_false(result); -} - -/** - * Tests log settings from client config file with invalid json - */ -void test_client_config_log_invalid_json(void** unused) { - char clientConfigJSON[] = "{{{\"invalid json\"}"; - char configFilePath[] = "sf_client_config.json"; - FILE* file; - file = fopen(configFilePath, "w"); - fprintf(file, "%s", clientConfigJSON); - fclose(file); - - // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; - sf_bool result = load_client_config(configFilePath, &clientConfig); - assert_false(result); - - // Cleanup - remove(configFilePath); -} - -#ifndef _WIN32 -/** - * Tests log settings from client config file - */ -void test_client_config_log(void **unused) { - char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; - char configFilePath[] = "sf_client_config.json"; - FILE *file; - file = fopen(configFilePath,"w"); - fprintf(file, "%s", clientConfigJSON); - fclose(file); - - // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; - load_client_config(configFilePath, &clientConfig); - - // Set log name and level - char logname[] = "%s/dummy.log"; - size_t log_path_size = 1 + strlen(logname); - log_path_size += strlen(clientConfig.logPath); - char* LOG_PATH = (char*)SF_CALLOC(1, log_path_size); - sf_sprintf(LOG_PATH, log_path_size, logname, clientConfig.logPath); - log_set_level(log_from_str_to_level(clientConfig.logLevel)); - log_set_path(LOG_PATH); - - // Ensure the log file doesn't exist at the beginning - remove(LOG_PATH); - - // Info log won't trigger the log file creation since log level is set to warn in config - log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, F_OK), 0); - - // Warning log will trigger the log file creation - log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, F_OK), 0); - log_close(); - - // Cleanup - remove(configFilePath); - remove(LOG_PATH); -} - -/** - * Tests log settings from client config file via global init - */ -void test_client_config_log_init(void** unused) { - char LOG_PATH[MAX_PATH] = { 0 }; - char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; - char configFilePath[] = "sf_client_config.json"; - FILE* file; - file = fopen(configFilePath, "w"); - fprintf(file, "%s", clientConfigJSON); - fclose(file); - - snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); - snowflake_global_init("./logs", SF_LOG_TRACE, NULL); - - // Get the log path determined by libsnowflakeclient - snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); - // Ensure the log file doesn't exist at the beginning - remove(LOG_PATH); - - // Info log won't trigger the log file creation since log level is set to warn in config - log_info("dummy info log"); - assert_int_not_equal(access(LOG_PATH, F_OK), 0); - - // Warning log will trigger the log file creation - log_warn("dummy warning log"); - assert_int_equal(access(LOG_PATH, F_OK), 0); - log_close(); - - // Cleanup - remove(configFilePath); - remove(LOG_PATH); -} - -/** - * Tests timing of log file creation - */ -void test_log_creation(void **unused) { - char logname[] = "dummy.log"; - - // ensure the log file doesn't exist at the beginning - remove(logname); - assert_int_not_equal(access(logname, F_OK), 0); - - log_set_lock(NULL); - log_set_level(SF_LOG_WARN); - log_set_quiet(1); - log_set_path(logname); - - // info log won't trigger the log file creation since log level is set to warning - log_info("dummy info log"); - assert_int_not_equal(access(logname, F_OK), 0); - - // warning log will trigger the log file creation - log_warn("dummy warning log"); - assert_int_equal(access(logname, F_OK), 0); - log_close(); - - remove(logname); -} - -/** - * Tests masking secret information in log - */ -void test_mask_secret_log(void **unused) { - FILE* fp = fopen("dummy.log", "w+"); - assert_non_null(fp); - log_set_lock(NULL); - log_set_level(SF_LOG_TRACE); - log_set_quiet(1); - log_set_fp(fp); - - const char * logtext[][2] = { - {//0 - "Secure log record!", - "Secure log record!" - }, - {//1 - "Token =ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", - "Token =****" - }, - {//2 - "idToken : ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", - "idToken : ****" - }, - {//3 - "sessionToken:ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", - "sessionToken:****" - }, - {//4 - "masterToken : 'ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ'", - "masterToken : '****'" - }, - {//5 - "assertion content:\"ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ\"", - "assertion content:\"****\"" - }, - {//6 - "password: random!TEST/-pwd=123++#", - "password: ****" - }, - {//7 - "pwd =\"random!TEST/-pwd=123++#", - "pwd =\"****" - }, - {//8 - "AWSAccessKeyId=ABCD%efg+1234/567", - "AWSAccessKeyId=****" - }, - {//9 - "https://sfc-fake.s3.fakeamazon.com/012345xx-012x-012x-0123-1a2b3c4d/fake/data_fake?x-amz-server-side-encryption-customer-algorithm=fakealgo&response-content-encoding=fakezip&AWSAccessKeyId=ABCD%efg+1234/567&Expires=123456789&Signature=ABCD%efg+1234/567ABCD%efg+1234/567", - "https://sfc-fake.s3.fakeamazon.com/012345xx-012x-012x-0123-1a2b3c4d/fake/data_fake?x-amz-server-side-encryption-customer-algorithm=fakealgo&response-content-encoding=fakezip&AWSAccessKeyId=****&Expires=123456789&Signature=****" - }, - {//10 - "aws_key_id='afhl124lomsafho0582'", - "aws_key_id='****'" - }, - {//11 - "aws_secret_key = 'dfhuwaojm753omsdfh30oi+fj'", - "aws_secret_key = '****'" - }, - {//12 - "\"privateKeyData\": \"abcdefghijk\"", - "\"privateKeyData\": \"XXXX\"" - }, - }; - - char * line = NULL; - size_t len = 0; - for (int i = 0; i < 13; i++) - { - fseek(fp, 0, SEEK_SET); - log_trace("%s", logtext[i][0]); - fseek(fp, 0, SEEK_SET); - getline(&line, &len, fp); - if (i != 0) - { - assert_null(strstr(line, logtext[i][0])); - } - assert_non_null(strstr(line, logtext[i][1])); - } - - free(line); - fclose(fp); -} -#endif - -int main(void) { - const struct CMUnitTest tests[] = { - cmocka_unit_test(test_log_str_to_level), - cmocka_unit_test(test_invalid_client_config_path), - cmocka_unit_test(test_client_config_log_invalid_json), -#ifndef _WIN32 - cmocka_unit_test(test_client_config_log), - cmocka_unit_test(test_client_config_log_init), - cmocka_unit_test(test_log_creation), - cmocka_unit_test(test_mask_secret_log), -#endif - }; - return cmocka_run_group_tests(tests, NULL, NULL); -} +/* + * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. + */ + + +#include "utils/test_setup.h" +#include +#include "memory.h" + +#ifndef _WIN32 +#include +#endif + + +/** + * Tests converting a string representation of log level to the log level enum + */ +void test_log_str_to_level(void **unused) { + assert_int_equal(log_from_str_to_level("TRACE"), SF_LOG_TRACE); + assert_int_equal(log_from_str_to_level("DEBUG"), SF_LOG_DEBUG); + assert_int_equal(log_from_str_to_level("INFO"), SF_LOG_INFO); + assert_int_equal(log_from_str_to_level("wArN"), SF_LOG_WARN); + assert_int_equal(log_from_str_to_level("erroR"), SF_LOG_ERROR); + assert_int_equal(log_from_str_to_level("fatal"), SF_LOG_FATAL); + + /* negative */ + assert_int_equal(log_from_str_to_level("hahahaha"), SF_LOG_FATAL); + assert_int_equal(log_from_str_to_level(NULL), SF_LOG_FATAL); +} + +/** + * Tests log settings with invalid client config filepath + */ +void test_invalid_client_config_path(void** unused) { + char configFilePath[] = "fakePath.json"; + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); +} + +/** + * Tests log settings from client config file with invalid json + */ +void test_client_config_log_invalid_json(void** unused) { + char clientConfigJSON[] = "{{{\"invalid json\"}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + sf_bool result = load_client_config(configFilePath, &clientConfig); + assert_false(result); + + // Cleanup + remove(configFilePath); +} + +#ifndef _WIN32 +/** + * Tests log settings from client config file + */ +void test_client_config_log(void **unused) { + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE *file; + file = fopen(configFilePath,"w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { .logLevel = "", .logPath = "" }; + load_client_config(configFilePath, &clientConfig); + + // Set log name and level + char logname[] = "%s/dummy.log"; + size_t log_path_size = 1 + strlen(logname); + log_path_size += strlen(clientConfig.logPath); + char* LOG_PATH = (char*)SF_CALLOC(1, log_path_size); + sf_sprintf(LOG_PATH, log_path_size, logname, clientConfig.logPath); + log_set_level(log_from_str_to_level(clientConfig.logLevel)); + log_set_path(LOG_PATH); + + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, F_OK), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + +/** + * Tests log settings from client config file via global init + */ +void test_client_config_log_init(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); + snowflake_global_init("./logs", SF_LOG_DEFAULT, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, F_OK), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + +/** + * Tests timing of log file creation + */ +void test_log_creation(void **unused) { + char logname[] = "dummy.log"; + + // ensure the log file doesn't exist at the beginning + remove(logname); + assert_int_not_equal(access(logname, F_OK), 0); + + log_set_lock(NULL); + log_set_level(SF_LOG_WARN); + log_set_quiet(1); + log_set_path(logname); + + // info log won't trigger the log file creation since log level is set to warning + log_info("dummy info log"); + assert_int_not_equal(access(logname, F_OK), 0); + + // warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(logname, F_OK), 0); + log_close(); + + remove(logname); +} + +/** + * Tests masking secret information in log + */ +void test_mask_secret_log(void **unused) { + FILE* fp = fopen("dummy.log", "w+"); + assert_non_null(fp); + log_set_lock(NULL); + log_set_level(SF_LOG_TRACE); + log_set_quiet(1); + log_set_fp(fp); + + const char * logtext[][2] = { + {//0 + "Secure log record!", + "Secure log record!" + }, + {//1 + "Token =ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", + "Token =****" + }, + {//2 + "idToken : ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", + "idToken : ****" + }, + {//3 + "sessionToken:ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ", + "sessionToken:****" + }, + {//4 + "masterToken : 'ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ'", + "masterToken : '****'" + }, + {//5 + "assertion content:\"ETMsDgAAAXI0IS9NABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwCAABAAEEb/xAQlmT+mwIx9G32E+ikAAACA/CPlEkq//+jWZnQkOj5VhjayruDsCVRGS/B6GzHUugXLc94EfEwuto94gS/oKSVrUg/JRPekypLAx4Afa1KW8n1RqXRF9Hzy1VVLmVEBMtei3yFJPNSHtfbeFHSr9eVB/OL8dOGbxQluGCh6XmaqTjyrh3fqUTWz7+n74+gu2ugAFFZ18iT+DStK0TTdmy4vBC6xUcHQ\"", + "assertion content:\"****\"" + }, + {//6 + "password: random!TEST/-pwd=123++#", + "password: ****" + }, + {//7 + "pwd =\"random!TEST/-pwd=123++#", + "pwd =\"****" + }, + {//8 + "AWSAccessKeyId=ABCD%efg+1234/567", + "AWSAccessKeyId=****" + }, + {//9 + "https://sfc-fake.s3.fakeamazon.com/012345xx-012x-012x-0123-1a2b3c4d/fake/data_fake?x-amz-server-side-encryption-customer-algorithm=fakealgo&response-content-encoding=fakezip&AWSAccessKeyId=ABCD%efg+1234/567&Expires=123456789&Signature=ABCD%efg+1234/567ABCD%efg+1234/567", + "https://sfc-fake.s3.fakeamazon.com/012345xx-012x-012x-0123-1a2b3c4d/fake/data_fake?x-amz-server-side-encryption-customer-algorithm=fakealgo&response-content-encoding=fakezip&AWSAccessKeyId=****&Expires=123456789&Signature=****" + }, + {//10 + "aws_key_id='afhl124lomsafho0582'", + "aws_key_id='****'" + }, + {//11 + "aws_secret_key = 'dfhuwaojm753omsdfh30oi+fj'", + "aws_secret_key = '****'" + }, + {//12 + "\"privateKeyData\": \"abcdefghijk\"", + "\"privateKeyData\": \"XXXX\"" + }, + }; + + char * line = NULL; + size_t len = 0; + for (int i = 0; i < 13; i++) + { + fseek(fp, 0, SEEK_SET); + log_trace("%s", logtext[i][0]); + fseek(fp, 0, SEEK_SET); + getline(&line, &len, fp); + if (i != 0) + { + assert_null(strstr(line, logtext[i][0])); + } + assert_non_null(strstr(line, logtext[i][1])); + } + + free(line); + fclose(fp); +} +#endif + +int main(void) { + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_log_str_to_level), + cmocka_unit_test(test_invalid_client_config_path), + cmocka_unit_test(test_client_config_log_invalid_json), +#ifndef _WIN32 + cmocka_unit_test(test_client_config_log), + cmocka_unit_test(test_client_config_log_init), + cmocka_unit_test(test_log_creation), + cmocka_unit_test(test_mask_secret_log), +#endif + }; + return cmocka_run_group_tests(tests, NULL, NULL); +} From 9eaadd0bc43d93c922d6dbbfd954ef05abb0a745 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 15 Nov 2024 14:57:25 -0800 Subject: [PATCH 61/74] Use picojson instead of cjson, refactor client config parser --- CMakeLists.txt | 3 + cpp/lib/ClientConfigParser.cpp | 249 ++++++++++------------- include/snowflake/ClientConfigParser.hpp | 41 ++-- include/snowflake/client_config_parser.h | 7 + lib/client.c | 7 +- tests/test_unit_logger.c | 7 +- 6 files changed, 137 insertions(+), 177 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb79dd56ff..8040e731df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ if (LINUX) deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/azure/include deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/cmocka/include deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/uuid/include + deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/picojson/include include lib) endif() @@ -422,6 +423,7 @@ if (APPLE) deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/aws/include deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/azure/include deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/cmocka/include + deps-build/${PLATFORM}/${CMAKE_BUILD_TYPE}/picojson/include include lib) endif() @@ -436,6 +438,7 @@ if (WIN32) deps-build/${PLATFORM}/${VSDIR}/${CMAKE_BUILD_TYPE}/aws/include deps-build/${PLATFORM}/${VSDIR}/${CMAKE_BUILD_TYPE}/azure/include deps-build/${PLATFORM}/${VSDIR}/${CMAKE_BUILD_TYPE}/cmocka/include + deps-build/${PLATFORM}/${VSDIR}/${CMAKE_BUILD_TYPE}/picojson/include include lib) if (CMAKE_SIZEOF_VOID_P EQUAL 8) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 335de78594..3db957991e 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -20,7 +20,7 @@ #endif using namespace Snowflake::Client; - +using namespace picojson; namespace { @@ -36,30 +36,43 @@ sf_bool load_client_config( client_config* out_clientConfig) { try { - ClientConfigParser configParser; + EasyLoggingConfigParser configParser; configParser.loadClientConfig(in_configFilePath, *out_clientConfig); } catch (std::exception e) { - sf_fprintf(stderr, e.what()); + CXX_LOG_ERROR("Using client configuration path from a connection string: %s", e.what()); return false; } return true; } +//////////////////////////////////////////////////////////////////////////////// +void free_client_config(client_config* clientConfig) +{ + if (clientConfig->logLevel != NULL) + { + SF_FREE(clientConfig->logLevel); + } + if (clientConfig->logPath != NULL) + { + SF_FREE(clientConfig->logPath); + } +} + // Public ====================================================================== //////////////////////////////////////////////////////////////////////////////// -ClientConfigParser::ClientConfigParser() +EasyLoggingConfigParser::EasyLoggingConfigParser() { ; // Do nothing. } //////////////////////////////////////////////////////////////////////////////// -ClientConfigParser::~ClientConfigParser() +EasyLoggingConfigParser::~EasyLoggingConfigParser() { ; // Do nothing. } //////////////////////////////////////////////////////////////////////////////// -void ClientConfigParser::loadClientConfig( +void EasyLoggingConfigParser::loadClientConfig( const std::string& in_configFilePath, client_config& out_clientConfig) { @@ -72,211 +85,161 @@ void ClientConfigParser::loadClientConfig( } //////////////////////////////////////////////////////////////////////////////// -std::string ClientConfigParser::resolveClientConfigPath( +std::string EasyLoggingConfigParser::resolveClientConfigPath( const std::string& in_configFilePath) { char envbuf[MAX_PATH + 1]; - std::string derivedConfigPath = ""; if (!in_configFilePath.empty()) { // 1. Try config file if it was passed in - derivedConfigPath = in_configFilePath; - CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", - "Using client configuration path from a connection string: %s", in_configFilePath.c_str()); + CXX_LOG_INFO("Using client configuration path from a connection string: %s", in_configFilePath.c_str()); + return in_configFilePath; } else if (const char* clientConfigEnv = sf_getenv_s(SF_CLIENT_CONFIG_ENV_NAME.c_str(), envbuf, sizeof(envbuf))) { // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME - derivedConfigPath = clientConfigEnv; - CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", - "Using client configuration path from an environment variable: %s", clientConfigEnv); + CXX_LOG_INFO("Using client configuration path from an environment variable: %s", clientConfigEnv); + return clientConfigEnv; } else { // 3. Try DLL binary dir std::string binaryDir = getBinaryPath(); std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(binaryDirFilePath.c_str())) + if (boost::filesystem::is_regular_file(binaryDirFilePath)) { - derivedConfigPath = binaryDirFilePath; - CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", - "Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); + CXX_LOG_INFO("Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); + return binaryDirFilePath; } else { -#if defined(WIN32) || defined(_WIN64) - // 4. Try user home dir - std::string homeDirFilePath; - char* homeDir; - if ((homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf))) && strlen(homeDir) != 0) - { - homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - } else { - // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH - char* homeDriveEnv; - char* homePathEnv; - if ((homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf))) && (strlen(homeDriveEnv) != 0) && - (homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf))) && (strlen(homePathEnv) != 0)) - { - homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - } - } - if (checkFileExists(homeDirFilePath)) - { - derivedConfigPath = homeDirFilePath; - CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", - "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); - } -#else // 4. Try user home dir - char* homeDir; - if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) - { - std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; - if (checkFileExists(homeDirFilePath)) - { - derivedConfigPath = homeDirFilePath; - CXX_LOG_INFO("sf", "ClientConfigParser", "loadClientConfig", - "Using client configuration path from home directory: %s", homeDirFilePath.c_str()); - } - } -#endif + return resolveHomeDirConfigPath(); } } - return derivedConfigPath; } // Private ===================================================================== //////////////////////////////////////////////////////////////////////////////// -bool ClientConfigParser::checkFileExists(const std::string& in_filePath) +std::string EasyLoggingConfigParser::resolveHomeDirConfigPath() { - FILE* file = sf_fopen(&file, in_filePath.c_str(), "r"); - if (file != nullptr) + char envbuf[MAX_PATH + 1]; +#if defined(WIN32) || defined(_WIN64) + std::string homeDirFilePath; + char* homeDir; + if ((homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf))) && strlen(homeDir) != 0) + { + homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + } else { + // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH + char* homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); + char* homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); + if (homeDriveEnv && strlen(homeDriveEnv) != 0 && homePathEnv && strlen(homePathEnv) != 0) + { + homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + } + } + if (boost::filesystem::is_regular_file(homeDirFilePath)) { - fclose(file); - return true; + CXX_LOG_INFO("Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + return homeDirFilePath; } - return false; +#else + char* homeDir; + if ((homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf))) && (strlen(homeDir) != 0)) + { + std::string homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::is_regular_file(homeDirFilePath)) + { + CXX_LOG_INFO("Using client configuration path from home directory: %s", homeDirFilePath.c_str()); + return homeDirFilePath; + } + } +#endif + return ""; } //////////////////////////////////////////////////////////////////////////////// -void ClientConfigParser::parseConfigFile( +void EasyLoggingConfigParser::parseConfigFile( const std::string& in_filePath, client_config& out_clientConfig) { - cJSON* jsonConfig; - FILE* configFile; - try + value jsonConfig; + std::string err; + std::ifstream configFile; + try { - configFile = sf_fopen(&configFile, in_filePath.c_str(), "r"); + configFile.open(in_filePath, std::fstream::in | std::ios::binary); if (!configFile) { - CXX_LOG_INFO("sf", "ClientConfigParser", "parseConfigFile", - "Could not open a file. The file may not exist: %s", + CXX_LOG_INFO("Could not open a file. The file may not exist: %s", in_filePath.c_str()); std::string errMsg = "Error finding client configuration file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); + throw std::exception(errMsg.c_str()); } #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); #endif - fseek(configFile, 0, SEEK_END); - long fileSize = ftell(configFile); - fseek(configFile, 0, SEEK_SET); - char* buffer = (char*)malloc(fileSize); - if (buffer) - { - size_t result = fread(buffer, 1, fileSize, configFile); - if (result != fileSize) - { - CXX_LOG_ERROR( - "sf", - "ClientConfigParser", - "parseConfigFile", - "Error in reading file: %s", in_filePath.c_str()); - } - } - jsonConfig = snowflake_cJSON_Parse(buffer); - const char* error_ptr = snowflake_cJSON_GetErrorPtr(); - if (error_ptr) + err = parse(jsonConfig, configFile); + if (!err.empty()) { - CXX_LOG_ERROR( - "sf", - "ClientConfigParser", - "parseConfigFile", - "Error in parsing JSON: %s, err: %s", in_filePath.c_str(), error_ptr); + CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); std::string errMsg = "Error parsing client configuration file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); + throw std::exception(errMsg.c_str()); } } - catch (std::exception& ex) + catch (std::exception& e) { - if (configFile) - { - fclose(configFile); - } + configFile.close(); throw; } - const cJSON* commonProps = snowflake_cJSON_GetObjectItem(jsonConfig, "common"); - checkUnknownEntries(snowflake_cJSON_Print(commonProps)); - const cJSON* logLevel = snowflake_cJSON_GetObjectItem(commonProps, "log_level"); - if (snowflake_cJSON_IsString(logLevel)) - { - out_clientConfig.logLevel = snowflake_cJSON_GetStringValue(logLevel); - } - const cJSON* logPath = snowflake_cJSON_GetObjectItem(commonProps, "log_path"); - if (snowflake_cJSON_IsString(logPath)) + value commonProps = jsonConfig.get("common"); + checkUnknownEntries(commonProps); + if (commonProps.is()) { - out_clientConfig.logPath = snowflake_cJSON_GetStringValue(logPath); - } - if (configFile) - { - fclose(configFile); + if (commonProps.contains("log_level") && commonProps.get("log_level").is()) + { + const char* logLevel = commonProps.get("log_level").get().c_str(); + size_t logLevelSize = strlen(logLevel) + 1; + out_clientConfig.logLevel = (char*)SF_CALLOC(1, logLevelSize); + sf_strcpy(out_clientConfig.logLevel, logLevelSize, logLevel); + } + if (commonProps.contains("log_path") && commonProps.get("log_path").is()) + { + const char* logPath = commonProps.get("log_path").get().c_str(); + size_t logPathSize = strlen(logPath) + 1; + out_clientConfig.logPath = (char*)SF_CALLOC(1, logPathSize); + sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); + } } + configFile.close(); } //////////////////////////////////////////////////////////////////////////////// -void ClientConfigParser::checkIfValidPermissions(const std::string& in_filePath) +void EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_filePath) { boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); boost::filesystem::perms permissions = fileStatus.permissions(); if (permissions & boost::filesystem::group_write || permissions & boost::filesystem::others_write) { - CXX_LOG_ERROR( - "sf", - "ClientConfigParser", - "checkIfValidPermissions", - "Error due to other users having permission to modify the config file: %s", + CXX_LOG_ERROR("Error due to other users having permission to modify the config file: %s", in_filePath.c_str()); std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); + throw std::exception(errMsg.c_str()); } } //////////////////////////////////////////////////////////////////////////////// -void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) +void EasyLoggingConfigParser::checkUnknownEntries(value& in_config) { - cJSON* jsonConfig = snowflake_cJSON_Parse(in_jsonString.c_str()); - const char* error_ptr = snowflake_cJSON_GetErrorPtr(); - if (error_ptr) - { - CXX_LOG_ERROR( - "sf", - "ClientConfigParser", - "checkUnknownEntries", - "Error in parsing JSON: %s, err: %s", in_jsonString.c_str(), error_ptr); - std::string errMsg = "Error parsing json: " + in_jsonString; - throw ClientConfigException(errMsg.c_str()); - } - - if (snowflake_cJSON_IsObject(jsonConfig)) + if (in_config.is()) { - cJSON* entry = NULL; - snowflake_cJSON_ArrayForEach(entry, jsonConfig) + const value::object& configObj = in_config.get(); + for (value::object::const_iterator i = configObj.begin(); i != configObj.end(); ++i) { - std::string key = entry->string; + std::string key = i->first; bool found = false; for (std::string knownEntry : KnownCommonEntries) { @@ -288,19 +251,15 @@ void ClientConfigParser::checkUnknownEntries(const std::string& in_jsonString) if (!found) { std::string warnMsg = - "Unknown configuration entry: " + key + " with value:" + entry->valuestring; - CXX_LOG_WARN( - "sf", - "ClientConfigParser", - "checkUnknownEntries", - warnMsg.c_str()); + "Unknown configuration entry: " + key + " with value:" + i->second.to_str().c_str(); + CXX_LOG_WARN(warnMsg.c_str()); } } } } //////////////////////////////////////////////////////////////////////////////// -std::string ClientConfigParser::getBinaryPath() +std::string EasyLoggingConfigParser::getBinaryPath() { std::string binaryFullPath; #if defined(WIN32) || defined(_WIN64) diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index 87a2cd2e41..a642189b55 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -2,38 +2,29 @@ * Copyright (c) 2024 Snowflake Computing */ -#ifndef SNOWFLAKE_CONFIGPARSER_HPP -#define SNOWFLAKE_CONFIGPARSER_HPP +#ifndef SNOWFLAKE_EASYLOGGINGCONFIGPARSER_HPP +#define SNOWFLAKE_EASYLOGGINGCONFIGPARSER_HPP #include #include "client_config_parser.h" +#include "picojson.h" namespace Snowflake { namespace Client { - struct ClientConfigException : public std::exception - { - ClientConfigException(const std::string& message) : message_(message) {} - const char* what() const noexcept - { - return message_.c_str(); - } - - std::string message_; - }; - class ClientConfigParser + class EasyLoggingConfigParser { // Public ================================================================== public: /** * Constructor for client config */ - ClientConfigParser(); + EasyLoggingConfigParser(); /// @brief Destructor. - ~ClientConfigParser(); + ~EasyLoggingConfigParser(); /** * Construct SFClientConfig from client config file passed by user. This method searches the @@ -53,22 +44,22 @@ namespace Client // Private ================================================================= private: - /** - * @brief Check if the file exists. - * - * @param in_filePath The file path to check. - */ - bool checkFileExists(const std::string& in_filePath); - /** * @brief Resolve the client config path. * * @param in_configFilePath The config file path passed in by the user. * - * @param The client config path + * @return The client config path */ std::string resolveClientConfigPath(const std::string& in_configFilePath); + /** + * @brief Resolve home directory config path. + * + * @return The home directory client config path if exist, else empty. + */ + std::string resolveHomeDirConfigPath(); + /** * @brief Parse JSON string. * @@ -91,7 +82,7 @@ namespace Client * * @param in_jsonString The json object to check in json config file. */ - void checkUnknownEntries(const std::string& in_jsonString); + void checkUnknownEntries(picojson::value& in_config); /** * @ brief Get the path to the binary file @@ -102,4 +93,4 @@ namespace Client } // namespace Client } // namespace Snowflake -#endif //SNOWFLAKE_CONFIGPARSER_HPP +#endif //SNOWFLAKE_EASYLOGGINGCONFIGPARSER_HPP diff --git a/include/snowflake/client_config_parser.h b/include/snowflake/client_config_parser.h index 86ae0683b6..861272b6c7 100644 --- a/include/snowflake/client_config_parser.h +++ b/include/snowflake/client_config_parser.h @@ -38,6 +38,13 @@ extern "C" { sf_bool load_client_config( const char* in_configFilePath, client_config* out_clientConfig); + + /** + * Free client config memory + * + * @param clientConfig The client_config object to be freed. + */ + void free_client_config(client_config* clientConfig); #ifdef __cplusplus } // extern "C" diff --git a/lib/client.c b/lib/client.c index bfed48f073..7a2148e42e 100644 --- a/lib/client.c +++ b/lib/client.c @@ -315,7 +315,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { char client_config_file[MAX_PATH] = {0}; snowflake_global_get_attribute( SF_GLOBAL_CLIENT_CONFIG_FILE, client_config_file, sizeof(client_config_file)); - client_config clientConfig = { .logLevel = "", .logPath = "" }; + client_config clientConfig = { 0 }; load_client_config(client_config_file, &clientConfig); size_t log_path_size = 1; //Start with 1 to include null terminator @@ -327,7 +327,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { if (sf_log_path == NULL) { if (log_path && strlen(log_path) != 0) { sf_log_path = log_path; - } else if (strlen(clientConfig.logPath) != 0) { + } else if (clientConfig.logPath != NULL) { sf_log_path = clientConfig.logPath; } } @@ -336,7 +336,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { if (sf_log_level_str != NULL) { sf_log_level = log_from_str_to_level(sf_log_level_str); } else if (sf_log_level == SF_LOG_DEFAULT) { - if (strlen(clientConfig.logLevel) != 0) { + if (clientConfig.logLevel != NULL) { sf_log_level = log_from_str_to_level(clientConfig.logLevel); } else { @@ -379,6 +379,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { ret = SF_BOOLEAN_TRUE; cleanup: + free_client_config(&clientConfig); return ret; } diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index bdeed1f5f4..bfce12985a 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -11,7 +11,6 @@ #include #endif - /** * Tests converting a string representation of log level to the log level enum */ @@ -35,7 +34,7 @@ void test_invalid_client_config_path(void** unused) { char configFilePath[] = "fakePath.json"; // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; + client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); assert_false(result); } @@ -52,7 +51,7 @@ void test_client_config_log_invalid_json(void** unused) { fclose(file); // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; + client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); assert_false(result); @@ -73,7 +72,7 @@ void test_client_config_log(void **unused) { fclose(file); // Parse client config for log details - client_config clientConfig = { .logLevel = "", .logPath = "" }; + client_config clientConfig = { 0 }; load_client_config(configFilePath, &clientConfig); // Set log name and level From 6c3aa65392a4ac9c8c1f6126c3e721c4753cda87 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 15 Nov 2024 17:32:08 -0800 Subject: [PATCH 62/74] Add exception to keep track of errors --- cpp/lib/ClientConfigParser.cpp | 6 +++--- include/snowflake/ClientConfigParser.hpp | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 3db957991e..9b796bfead 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -175,7 +175,7 @@ void EasyLoggingConfigParser::parseConfigFile( CXX_LOG_INFO("Could not open a file. The file may not exist: %s", in_filePath.c_str()); std::string errMsg = "Error finding client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } #if !defined(WIN32) && !defined(_WIN64) checkIfValidPermissions(in_filePath); @@ -185,7 +185,7 @@ void EasyLoggingConfigParser::parseConfigFile( { CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); std::string errMsg = "Error parsing client configuration file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } catch (std::exception& e) @@ -227,7 +227,7 @@ void EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_file CXX_LOG_ERROR("Error due to other users having permission to modify the config file: %s", in_filePath.c_str()); std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; - throw std::exception(errMsg.c_str()); + throw ClientConfigException(errMsg.c_str()); } } diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index a642189b55..b84dbe9e65 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -13,6 +13,16 @@ namespace Snowflake { namespace Client { + struct ClientConfigException : public std::exception + { + ClientConfigException(const std::string& message) : message_(message) {} + const char* what() const noexcept + { + return message_.c_str(); + } + + std::string message_; + }; class EasyLoggingConfigParser { From c022028c91f8b9fd45e71c6c3ae064889512547c Mon Sep 17 00:00:00 2001 From: norrislee Date: Sat, 16 Nov 2024 01:07:25 -0800 Subject: [PATCH 63/74] Revert accidental inclusions from master-2.0.0 branch --- CMakeLists.txt | 6 +- ci/build_linux.sh | 4 +- cpp/FileTransferAgent.cpp | 76 +--- cpp/StatementPutGet.cpp | 155 +------ cpp/StatementPutGet.hpp | 33 -- cpp/lib/ResultSet.cpp | 47 +-- cpp/lib/ResultSet.hpp | 58 +-- cpp/lib/ResultSetArrow.cpp | 44 +- cpp/lib/ResultSetArrow.hpp | 19 +- cpp/lib/ResultSetJson.cpp | 12 +- cpp/lib/ResultSetJson.hpp | 4 +- cpp/lib/ResultSetPutGet.cpp | 336 --------------- cpp/lib/ResultSetPutGet.hpp | 205 --------- cpp/lib/result_set.cpp | 341 ++++++++++----- include/snowflake/client.h | 63 +-- lib/chunk_downloader.c | 4 +- lib/client.c | 690 +++++++++---------------------- lib/client_int.h | 44 +- lib/connection.c | 44 +- lib/connection.h | 39 +- lib/http_perform.c | 76 +--- lib/mock_http_perform.h | 2 +- lib/result_set.h | 103 +++-- tests/CMakeLists.txt | 9 +- tests/test_multiple_statements.c | 307 -------------- tests/test_simple_put.cpp | 667 ++++++++---------------------- 26 files changed, 754 insertions(+), 2634 deletions(-) delete mode 100644 cpp/lib/ResultSetPutGet.cpp delete mode 100644 cpp/lib/ResultSetPutGet.hpp delete mode 100644 tests/test_multiple_statements.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 8040e731df..03300e9f8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,8 +191,6 @@ set (SOURCE_FILES_PUT_GET cpp/logger/SFLogger.hpp cpp/logger/SecretDetector.cpp cpp/logger/SecretDetector.hpp - cpp/lib/ResultSetPutGet.cpp - cpp/lib/ResultSetPutGet.hpp include/snowflake/IFileTransferAgent.hpp include/snowflake/ISFLogger.hpp include/snowflake/IStatementPutGet.hpp @@ -229,6 +227,8 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/lib/ClientQueryContextCache.cpp cpp/lib/ClientQueryContextCache.hpp cpp/lib/result_set.cpp + cpp/lib/result_set_arrow.cpp + cpp/lib/result_set_json.cpp cpp/lib/ResultSet.cpp cpp/lib/ResultSet.hpp cpp/lib/ResultSetArrow.cpp @@ -244,6 +244,8 @@ set(SOURCE_FILES_CPP_WRAPPER cpp/util/CurlDesc.cpp cpp/util/CurlDescPool.cpp lib/result_set.h + lib/result_set_arrow.h + lib/result_set_json.h lib/query_context_cache.h lib/curl_desc_pool.h lib/authenticator.h) diff --git a/ci/build_linux.sh b/ci/build_linux.sh index 6e58b1a987..d42ec7c84b 100755 --- a/ci/build_linux.sh +++ b/ci/build_linux.sh @@ -51,7 +51,5 @@ docker run \ #remove image to save disk space on github if [[ -n "$GITHUB_ACTIONS" ]]; then docker rm -vf $(docker ps -aq --filter ancestor=${BUILD_IMAGE_NAME}) - if [[ $CLIENT_CODE_COVERAGE -ne 1 ]] && [[ "$BUILD_TYPE" != "Debug" ]]; then - docker rmi -f "${BUILD_IMAGE_NAME}" - fi + docker rmi -f "${BUILD_IMAGE_NAME}" fi diff --git a/cpp/FileTransferAgent.cpp b/cpp/FileTransferAgent.cpp index cc0fbaee90..80066899a1 100755 --- a/cpp/FileTransferAgent.cpp +++ b/cpp/FileTransferAgent.cpp @@ -9,9 +9,8 @@ #include "FileTransferAgent.hpp" #include "snowflake/SnowflakeTransferException.hpp" #include "snowflake/IStatementPutGet.hpp" -#include "StatementPutGet.hpp" -#include "lib/ResultSetPutGet.hpp" #include "util/Base64.hpp" +#include "SnowflakeS3Client.hpp" #include "StorageClientFactory.hpp" #include "crypto/CipherStreamBuf.hpp" #include "crypto/Cryptor.hpp" @@ -19,7 +18,6 @@ #include "util/ThreadPool.hpp" #include "EncryptionProvider.hpp" #include "logger/SFLogger.hpp" -#include "error.h" #include "snowflake/platform.h" #include "snowflake/SF_CRTFunctionSafe.h" #include @@ -963,75 +961,3 @@ std::string Snowflake::Client::FileTransferAgent::getLocalFilePathFromCommand( return localFilePath; } - -using namespace Snowflake::Client; -extern "C" { - SF_STATUS STDCALL _snowflake_execute_put_get_native( - SF_STMT* sfstmt, - struct SF_QUERY_RESULT_CAPTURE* result_capture) - { - if (!sfstmt) - { - return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; - } - SF_CONNECT* sfconn = sfstmt->connection; - if (!sfconn) - { - return SF_STATUS_ERROR_CONNECTION_NOT_EXIST; - } - StatementPutGet stmtPutGet(sfstmt); - TransferConfig transConfig; - transConfig.caBundleFile = NULL; // use the one from global settings - transConfig.compressLevel = sfconn->put_compress_level; - transConfig.getSizeThreshold = sfconn->get_threshold; - transConfig.proxy = NULL; // use the one from statement - transConfig.tempDir = sfconn->put_temp_dir; - transConfig.useS3regionalUrl = sfconn->use_s3_regional_url; - string command(sfstmt->sql_text); - - FileTransferAgent agent(&stmtPutGet, &transConfig); - agent.setPutFastFail(sfconn->put_fastfail); - agent.setPutMaxRetries(sfconn->put_maxretries); - agent.setGetFastFail(sfconn->get_fastfail); - agent.setGetMaxRetries(sfconn->get_maxretries); - agent.setRandomDeviceAsUrand(sfconn->put_use_urand_dev); - - ITransferResult* result; - try - { - result = agent.execute(&command); - } - catch (std::exception& e) - { - std::string errmsg("File transfer failed: "); - errmsg += e.what(); - SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, - errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); - return SF_STATUS_ERROR_FILE_TRANSFER; - } - catch (...) - { - std::string errmsg("File transfer failed with unknown exception."); - SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_FILE_TRANSFER, - errmsg.c_str(), SF_SQLSTATE_GENERAL_ERROR); - return SF_STATUS_ERROR_FILE_TRANSFER; - } - - ResultSetPutGet * resultset = new Snowflake::Client::ResultSetPutGet(result); - if (!resultset) - { - std::string errmsg("Failed to allocate put get result set."); - SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_OUT_OF_MEMORY, - errmsg.c_str(), SF_SQLSTATE_MEMORY_ALLOCATION_ERROR); - return SF_STATUS_ERROR_OUT_OF_MEMORY; - } - - sfstmt->qrf = SF_PUTGET_FORMAT; - sfstmt->total_row_index = 0; - sfstmt->result_set = resultset; - sfstmt->chunk_rowcount = sfstmt->total_rowcount = result->getResultSize(); - sfstmt->total_fieldcount = resultset->setup_column_desc(&sfstmt->desc); - - return SF_STATUS_SUCCESS; - } -} diff --git a/cpp/StatementPutGet.cpp b/cpp/StatementPutGet.cpp index e5077854a9..989affbf89 100755 --- a/cpp/StatementPutGet.cpp +++ b/cpp/StatementPutGet.cpp @@ -3,35 +3,11 @@ */ #include -#include "connection.h" #include "snowflake/PutGetParseResponse.hpp" #include "StatementPutGet.hpp" -#include "curl_desc_pool.h" using namespace Snowflake::Client; -static size_t file_get_write_callback(char* ptr, size_t size, size_t nmemb, void* userdata) -{ - size_t data_size = size * nmemb; - std::basic_iostream* recvStream = (std::basic_iostream*)(userdata); - if (recvStream) - { - recvStream->write(static_cast(ptr), data_size); - } - - return data_size; -} - -static size_t file_put_read_callback(void* ptr, size_t size, size_t nmemb, void* userdata) -{ - std::basic_iostream* payload = (std::basic_iostream*)(userdata); - size_t data_size = size * nmemb; - - payload->read(static_cast(ptr), data_size); - size_t ret = payload->gcount(); - return payload->gcount(); -} - StatementPutGet::StatementPutGet(SF_STMT *stmt) : m_stmt(stmt), m_useProxy(false) { @@ -49,7 +25,7 @@ StatementPutGet::StatementPutGet(SF_STMT *stmt) : bool StatementPutGet::parsePutGetCommand(std::string *sql, PutGetParseResponse *putGetParseResponse) { - if (_snowflake_query_put_get_legacy(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) + if (snowflake_query(m_stmt, sql->c_str(), 0) != SF_STATUS_SUCCESS) { return false; } @@ -128,14 +104,6 @@ bool StatementPutGet::parsePutGetCommand(std::string *sql, }; putGetParseResponse->stageInfo.endPoint = response->stage_info->endPoint; - } - else if (sf_strncasecmp(response->stage_info->location_type, "gcs", 3) == 0) - { - putGetParseResponse->stageInfo.stageType = StageType::GCS; - putGetParseResponse->stageInfo.credentials = { - {"GCS_ACCESS_TOKEN", response->stage_info->stage_cred->gcs_access_token} - }; - } else if (sf_strncasecmp(response->stage_info->location_type, "local_fs", 8) == 0) { @@ -155,124 +123,3 @@ Util::Proxy* StatementPutGet::get_proxy() return &m_proxy; } } - -bool StatementPutGet::http_put(std::string const& url, - std::vector const& headers, - std::basic_iostream& payload, - size_t payloadLen, - std::string& responseHeaders) -{ - if (!m_stmt || !m_stmt->connection) - { - return false; - } - SF_CONNECT* sf = m_stmt->connection; - void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); - CURL* curl = get_curl_from_desc(curl_desc); - if (!curl) - { - return false; - } - - char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); - sf_strcpy(urlbuf, url.length() + 1, url.c_str()); - - SF_HEADER reqHeaders; - reqHeaders.header = NULL; - for (auto itr = headers.begin(); itr != headers.end(); itr++) - { - reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); - } - - PUT_PAYLOAD putPayload; - putPayload.buffer = &payload; - putPayload.length = payloadLen; - putPayload.read_callback = file_put_read_callback; - - char* respHeaders = NULL; - sf_bool success = SF_BOOLEAN_FALSE; - - success = http_perform(curl, PUT_REQUEST_TYPE, urlbuf, &reqHeaders, NULL, &putPayload, NULL, - NULL, &respHeaders, get_retry_timeout(sf), - SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode,sf->ocsp_fail_open, - sf->retry_on_curle_couldnt_connect_count, - 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, - sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); - - free_curl_desc(curl_desc); - SF_FREE(urlbuf); - curl_slist_free_all(reqHeaders.header); - if (respHeaders) - { - responseHeaders = std::string(respHeaders); - SF_FREE(respHeaders); - } - - return success; -} - -bool StatementPutGet::http_get(std::string const& url, - std::vector const& headers, - std::basic_iostream* payload, - std::string& responseHeaders, - bool headerOnly) -{ - SF_REQUEST_TYPE reqType = GET_REQUEST_TYPE; - if (headerOnly) - { - reqType = HEAD_REQUEST_TYPE; - } - - if (!m_stmt || !m_stmt->connection) - { - return false; - } - SF_CONNECT* sf = m_stmt->connection; - - void* curl_desc = get_curl_desc_from_pool(url.c_str(), sf->proxy, sf->no_proxy); - CURL* curl = get_curl_from_desc(curl_desc); - if (!curl) - { - return false; - } - - char* urlbuf = (char*)SF_CALLOC(1, url.length() + 1); - sf_strcpy(urlbuf, url.length() + 1, url.c_str()); - - SF_HEADER reqHeaders; - reqHeaders.header = NULL; - for (auto itr = headers.begin(); itr != headers.end(); itr++) - { - reqHeaders.header = curl_slist_append(reqHeaders.header, itr->c_str()); - } - - NON_JSON_RESP resp; - resp.buffer = payload; - resp.write_callback = file_get_write_callback; - - char* respHeaders = NULL; - sf_bool success = SF_BOOLEAN_FALSE; - - success = http_perform(curl, reqType, urlbuf, &reqHeaders, NULL, NULL, NULL, - &resp, &respHeaders, get_retry_timeout(sf), - SF_BOOLEAN_FALSE, &m_stmt->error, sf->insecure_mode, sf->ocsp_fail_open, - sf->retry_on_curle_couldnt_connect_count, - 0, sf->retry_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, - sf->proxy, sf->no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE); - - free_curl_desc(curl_desc); - SF_FREE(urlbuf); - curl_slist_free_all(reqHeaders.header); - if (respHeaders) - { - responseHeaders = respHeaders; - SF_FREE(respHeaders); - } - - if (payload) - { - payload->flush(); - } - - return success; -} diff --git a/cpp/StatementPutGet.hpp b/cpp/StatementPutGet.hpp index 19c00df0fd..94321fac73 100644 --- a/cpp/StatementPutGet.hpp +++ b/cpp/StatementPutGet.hpp @@ -28,39 +28,6 @@ class StatementPutGet : public Snowflake::Client::IStatementPutGet virtual Util::Proxy* get_proxy(); - /** - * PUT/GET on GCS use this interface to perform put request. - * Not implemented by default. - * @param url The url of the request. - * @param headers The headers of the request. - * @param payload The upload data. - * @param responseHeaders The headers of the response. - * - * return true if succeed otherwise false - */ - virtual bool http_put(std::string const& url, - std::vector const& headers, - std::basic_iostream& payload, - size_t payloadLen, - std::string& responseHeaders); - - /** - * PUT/GET on GCS use this interface to perform put request. - * Not implemented by default. - * @param url The url of the request. - * @param headers The headers of the request. - * @param payload The upload data. - * @param responseHeaders The headers of the response. - * @param headerOnly True if get response header only without payload body. - * - * return true if succeed otherwise false - */ - virtual bool http_get(std::string const& url, - std::vector const& headers, - std::basic_iostream* payload, - std::string& responseHeaders, - bool headerOnly); - private: SF_STMT *m_stmt; Util::Proxy m_proxy; diff --git a/cpp/lib/ResultSet.cpp b/cpp/lib/ResultSet.cpp index e72769c8f9..f304164c3d 100644 --- a/cpp/lib/ResultSet.cpp +++ b/cpp/lib/ResultSet.cpp @@ -11,15 +11,13 @@ #include "snowflake/SF_CRTFunctionSafe.h" #include "DataConversion.hpp" #include "ResultSet.hpp" -#include "ResultSetJson.hpp" -#include "ResultSetArrow.hpp" namespace Snowflake { namespace Client { -ResultSet::ResultSet(QueryResultFormat format) : +ResultSet::ResultSet() : m_binaryOutputFormat("HEX"), m_dateOutputFormat("YYYY-MM-DD"), m_timeOutputFormat("HH24:MI:SS"), @@ -30,16 +28,14 @@ ResultSet::ResultSet(QueryResultFormat format) : m_currChunkIdx(0), m_currChunkRowIdx(0), m_currColumnIdx(0), - m_currRowIdx(0), - m_queryResultFormat(format) + m_currRowIdx(0) { ; } ResultSet::ResultSet( SF_COLUMN_DESC * metadata, - const std::string& tzString, - QueryResultFormat format + std::string tzString ) : m_currChunkIdx(0), m_currChunkRowIdx(0), @@ -48,7 +44,6 @@ ResultSet::ResultSet( m_totalChunkCount(0), m_totalColumnCount(0), m_metadata(metadata), - m_queryResultFormat(format), m_isFirstChunk(true), m_tzString(tzString), m_error(SF_STATUS_SUCCESS) @@ -56,41 +51,5 @@ ResultSet::ResultSet( ; } -ResultSet* ResultSet::CreateResultFromJson(cJSON* json_rowset, - SF_COLUMN_DESC* metadata, - QueryResultFormat query_result_format, - const std::string& tz_string) -{ - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow(json_rowset, metadata, tz_string); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson(json_rowset, metadata, tz_string); - default: - return nullptr; - } -} - -ResultSet* ResultSet::CreateResultFromChunk(void* initial_chunk, - SF_COLUMN_DESC* metadata, - QueryResultFormat query_result_format, - const std::string& tz_string) -{ - switch (query_result_format) - { -#ifndef SF_WIN32 - case SF_ARROW_FORMAT: - return new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(((NON_JSON_RESP*)initial_chunk)->buffer), metadata, tz_string); -#endif - case SF_JSON_FORMAT: - return new Snowflake::Client::ResultSetJson((cJSON*)initial_chunk, metadata, tz_string); - default: - return nullptr; - } -} - } // namespace Client } // namespace Snowflake diff --git a/cpp/lib/ResultSet.hpp b/cpp/lib/ResultSet.hpp index 5e583c7848..8f3a46f059 100644 --- a/cpp/lib/ResultSet.hpp +++ b/cpp/lib/ResultSet.hpp @@ -12,21 +12,21 @@ #include "cJSON.h" #include "snowflake/basic_types.h" #include "snowflake/client.h" -#include "result_set.h" - -#define VERIFY_COLUMN_INDEX(index, total) \ - if (index < 1 || index > total) \ - { \ - setError(SF_STATUS_ERROR_OUT_OF_BOUNDS, \ - "Column index must be between 1 and snowflake_num_fields()"); \ - return SF_STATUS_ERROR_OUT_OF_BOUNDS; \ - } namespace Snowflake { namespace Client { +/** + * Enumeration over valid query result formats. + */ +enum QueryResultFormat +{ + ARROW, + JSON +}; + /** * The implementation of a base result set. * @@ -40,7 +40,7 @@ class ResultSet /** * Default constructor. */ - ResultSet(QueryResultFormat format); + ResultSet(); /** * Parameterized constructor. @@ -48,17 +48,7 @@ class ResultSet * @param metadata The metadata of the result set. * @param tzString The time zone. */ - ResultSet(SF_COLUMN_DESC * metadata, const std::string& tzString, QueryResultFormat format); - - static ResultSet* CreateResultFromJson(cJSON* json_rowset, - SF_COLUMN_DESC* metadata, - QueryResultFormat query_result_format, - const std::string& tz_string); - - static ResultSet* CreateResultFromChunk(void* initial_chunk, - SF_COLUMN_DESC* metadata, - QueryResultFormat query_result_format, - const std::string& tz_string); + ResultSet(SF_COLUMN_DESC * metadata, std::string tzString); /** * Destructor. @@ -67,20 +57,6 @@ class ResultSet // API methods ================================================================================= - /** - * Frees the previously held chunk in m_chunk and sets the current chunk to the given chunk. - * Each result type should override this function to handle chunks, - * while some result might not apply (won't be called), return error by default. - * - * @param chunkPtr The chunk to append. - * - * @return 0 if successful, otherwise an error is returned. - */ - virtual SF_STATUS STDCALL appendChunk(void* chunkPtr) - { - return SF_STATUS_ERROR_GENERAL; - } - /** * Advances to the next row. * @@ -218,13 +194,6 @@ class ResultSet */ virtual SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data) = 0; - /** - * Gets the total number of rows in the current chunk being processed. - * - * @return the number of rows in the current chunk. - */ - virtual size_t getRowCountInChunk() = 0; - // Other member getters ======================================================================== SF_STATUS getError() @@ -246,11 +215,6 @@ class ResultSet } } - QueryResultFormat getResultFormat() - { - return m_queryResultFormat; - } - protected: // Protected members =========================================================================== diff --git a/cpp/lib/ResultSetArrow.cpp b/cpp/lib/ResultSetArrow.cpp index 6b20a6b022..78779bc80a 100644 --- a/cpp/lib/ResultSetArrow.cpp +++ b/cpp/lib/ResultSetArrow.cpp @@ -10,6 +10,7 @@ #include "../logger/SFLogger.hpp" #include "ResultSetArrow.hpp" +#include "result_set_arrow.h" #include "DataConversion.hpp" #include "results.h" @@ -21,49 +22,21 @@ namespace Client ResultSetArrow::ResultSetArrow() : - Snowflake::Client::ResultSet(SF_ARROW_FORMAT) + Snowflake::Client::ResultSet() { - ; // Do nothing + m_queryResultFormat = QueryResultFormat::ARROW; } ResultSetArrow::ResultSetArrow( arrow::BufferBuilder* initialChunk, SF_COLUMN_DESC * metadata, - const std::string& tzString + std::string tzString ) : - ResultSet(metadata, tzString, SF_ARROW_FORMAT) + ResultSet(metadata, tzString) { - this->appendChunk(initialChunk); + m_queryResultFormat = QueryResultFormat::ARROW; - // Reset row indices so that they can be re-used by public API. - m_currChunkIdx = 0; - m_currChunkRowIdx = 0; - m_currRowIdx = 0; -} - -ResultSetArrow::ResultSetArrow( - cJSON * jsonRowset64, - SF_COLUMN_DESC * metadata, - const std::string& tzString -) : - ResultSet(metadata, tzString, SF_ARROW_FORMAT) -{ - NON_JSON_RESP resp; - arrow::BufferBuilder* bufferBuilder = NULL; - if (jsonRowset64) - { - const char* base64RowsetStr = snowflake_cJSON_GetStringValue(jsonRowset64); - if (base64RowsetStr && strlen(base64RowsetStr) > 0) - { - // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. - std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); - bufferBuilder = new arrow::BufferBuilder(); - (void)bufferBuilder->Append((void*)decodedRowsetStr.c_str(), decodedRowsetStr.length()); - } - } - resp.buffer = bufferBuilder; - - this->appendChunk(&resp); + this->appendChunk(initialChunk); // Reset row indices so that they can be re-used by public API. m_currChunkIdx = 0; @@ -78,9 +51,8 @@ ResultSetArrow::~ResultSetArrow() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetArrow::appendChunk(void* chunkPtr) +SF_STATUS STDCALL ResultSetArrow::appendChunk(arrow::BufferBuilder * chunk) { - arrow::BufferBuilder * chunk = (arrow::BufferBuilder*)(((NON_JSON_RESP*)chunkPtr)->buffer); if (chunk == nullptr) { if (m_isFirstChunk) diff --git a/cpp/lib/ResultSetArrow.hpp b/cpp/lib/ResultSetArrow.hpp index 68aa43456f..4c85c27501 100644 --- a/cpp/lib/ResultSetArrow.hpp +++ b/cpp/lib/ResultSetArrow.hpp @@ -62,20 +62,7 @@ class ResultSetArrow : public Snowflake::Client::ResultSet * @param metadata An array of metadata objects for each column. * @param tzString The time zone. */ - ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, const std::string& tzString); - - /** - * Parameterized constructor. - * - * This constructor will initialize m_records with the (partial) results - * contained in the initial chunk. It will also initialize m_metadata with - * the metadata in "metadata". - * - * @param jsonRowset64 A pointer to the rowset64 data in json result set. - * @param metadata An array of metadata objects for each column. - * @param tzString The time zone. - */ - ResultSetArrow(cJSON* jsonRowset64, SF_COLUMN_DESC* metadata, const std::string& tzString); + ResultSetArrow(arrow::BufferBuilder * initialChunk, SF_COLUMN_DESC * metadata, std::string tzString); /** * Destructor. @@ -87,11 +74,11 @@ class ResultSetArrow : public Snowflake::Client::ResultSet /** * Appends the given chunk to the internal result set. * - * @param chunkPtr The chunk to append. + * @param chunk The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(void* chunkPtr); + SF_STATUS STDCALL appendChunk(arrow::BufferBuilder * chunk); /** * Advances the internal iterator to the next row. diff --git a/cpp/lib/ResultSetJson.cpp b/cpp/lib/ResultSetJson.cpp index a3e5510990..9b27947495 100644 --- a/cpp/lib/ResultSetJson.cpp +++ b/cpp/lib/ResultSetJson.cpp @@ -18,18 +18,19 @@ namespace Client ResultSetJson::ResultSetJson() : - ResultSet(SF_JSON_FORMAT) + ResultSet() { - ; // Do nothing + m_queryResultFormat = QueryResultFormat::JSON; } ResultSetJson::ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - const std::string& tzString + std::string tzString ) : - ResultSet(metadata, tzString, SF_JSON_FORMAT) + ResultSet(metadata, tzString) { + m_queryResultFormat = QueryResultFormat::JSON; m_chunk = nullptr; appendChunk(rowset); } @@ -42,9 +43,8 @@ ResultSetJson::~ResultSetJson() // Public methods ================================================================================== -SF_STATUS STDCALL ResultSetJson::appendChunk(void* chunkPtr) +SF_STATUS STDCALL ResultSetJson::appendChunk(cJSON * chunk) { - cJSON * chunk = (cJSON *)chunkPtr; if (chunk == nullptr) { CXX_LOG_ERROR("appendChunk -- Received a null chunk to append."); diff --git a/cpp/lib/ResultSetJson.hpp b/cpp/lib/ResultSetJson.hpp index 5dc292dcbd..0c853e7ee2 100644 --- a/cpp/lib/ResultSetJson.hpp +++ b/cpp/lib/ResultSetJson.hpp @@ -52,7 +52,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet ResultSetJson( cJSON * rowset, SF_COLUMN_DESC * metadata, - const std::string& tzString); + std::string tzString); /** * Destructor. @@ -66,7 +66,7 @@ class ResultSetJson : public Snowflake::Client::ResultSet * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL appendChunk(void * chunkPtr); + SF_STATUS STDCALL appendChunk(cJSON * chunk); /** * Advances the internal iterator to the next row. If there are no more rows to consume, diff --git a/cpp/lib/ResultSetPutGet.cpp b/cpp/lib/ResultSetPutGet.cpp deleted file mode 100644 index 173ea18a70..0000000000 --- a/cpp/lib/ResultSetPutGet.cpp +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. - */ - -#include "ResultSetPutGet.hpp" -#include "../logger/SFLogger.hpp" -#include "memory.h" -#include "client_int.h" - -// helper functions -namespace -{ - void setup_string_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) - { - col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); - sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); - col_desc.byte_size = SF_DEFAULT_MAX_OBJECT_SIZE; - col_desc.c_type = SF_C_TYPE_STRING; - col_desc.internal_size = SF_DEFAULT_MAX_OBJECT_SIZE; - col_desc.null_ok = SF_BOOLEAN_TRUE; - col_desc.precision = 0; - col_desc.scale = 0; - col_desc.type = SF_DB_TYPE_TEXT; - col_desc.idx = idx; - } - - void setup_integer_column_desc(const std::string& name, SF_COLUMN_DESC& col_desc, size_t idx) - { - col_desc.name = (char*)SF_CALLOC(1, name.length() + 1); - sf_strncpy(col_desc.name, name.length() + 1, name.c_str(), name.length() + 1); - col_desc.byte_size = 8; - col_desc.c_type = SF_C_TYPE_UINT64; - col_desc.internal_size = 8; - col_desc.null_ok = SF_BOOLEAN_TRUE; - col_desc.precision = 38; - col_desc.scale = 0; - col_desc.type = SF_DB_TYPE_FIXED; - col_desc.idx = idx; - } - - struct putget_column - { - std::string name; - bool isInteger; - }; - - std::vector PUTGET_COLUMNS[2] = - { - // UPLOAD - { - {"source", false}, - {"target", false}, - {"source_size", true}, - {"target_size", true}, - {"source_compression", false}, - {"target_compression", false}, - {"status", false}, - {"encryption", false}, - {"message", false}, - }, - // DOWNLOAD - { - {"file", false}, - {"size", true}, - {"status", false}, - {"encryption", false}, - {"message", false}, - }, - }; -} - -namespace Snowflake -{ -namespace Client -{ -ResultSetPutGet::ResultSetPutGet(ITransferResult *result) : - ResultSet(SF_PUTGET_FORMAT), - m_cmdType(result->getCommandType()) -{ - m_values.reserve(result->getResultSize()); - while (result->next()) - { - std::vector row; - row.reserve(result->getColumnSize()); - for (unsigned int i = 0; i < result->getColumnSize(); i++) - { - row.emplace_back(); - result->getColumnAsString(i, row.back()); - } - m_values.push_back(row); - } -} - -ResultSetPutGet::~ResultSetPutGet() -{ - ; // Do nothing -} - -// Public methods ================================================================================== - -size_t ResultSetPutGet::setup_column_desc(SF_COLUMN_DESC** desc) -{ - if ((m_cmdType != CommandType::UPLOAD) && (m_cmdType != CommandType::DOWNLOAD)) - { - // impossible - return 0; - } - - SF_COLUMN_DESC * col_desc = NULL; - m_totalColumnCount = PUTGET_COLUMNS[m_cmdType].size(); - col_desc = (SF_COLUMN_DESC*)SF_CALLOC(m_totalColumnCount, sizeof(SF_COLUMN_DESC)); - for (size_t i = 0; i < m_totalColumnCount; i++) - { - if (PUTGET_COLUMNS[m_cmdType][i].isInteger) - { - setup_integer_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); - } - else - { - setup_string_column_desc(PUTGET_COLUMNS[m_cmdType][i].name, col_desc[i], i + 1); - } - } - - *desc = col_desc; - return m_totalColumnCount; -} - -SF_STATUS STDCALL ResultSetPutGet::next() -{ - if (m_currRowIdx < m_values.size()) - { - m_currRowIdx++; - return SF_STATUS_SUCCESS; - } - - return SF_STATUS_ERROR_OUT_OF_BOUNDS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsBool(size_t idx, sf_bool * out_data) -{ - setError(SF_STATUS_ERROR_CONVERSION_FAILURE, - "Value cannot be converted to boolean."); - return SF_STATUS_ERROR_CONVERSION_FAILURE; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsInt8(size_t idx, int8 * out_data) -{ - VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); - size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; - *out_data = static_cast(m_values[row_idx][idx - 1][0]); - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsInt32(size_t idx, int32 * out_data) -{ - *out_data = 0; - - uint64 value = 0; - SF_STATUS ret = getCellAsUint64(idx, &value); - - if (SF_STATUS_SUCCESS != ret) - { - return ret; - } - - if (value > SF_INT32_MAX) - { - CXX_LOG_ERROR("Value out of range for int32."); - setError(SF_STATUS_ERROR_OUT_OF_RANGE, - "Value out of range for int32."); - return SF_STATUS_ERROR_OUT_OF_RANGE; - } - - *out_data = static_cast(value); - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsInt64(size_t idx, int64 * out_data) -{ - *out_data = 0; - - uint64 value = 0; - SF_STATUS ret = getCellAsUint64(idx, &value); - - if (SF_STATUS_SUCCESS != ret) - { - return ret; - } - - if (value > SF_INT64_MAX) - { - CXX_LOG_ERROR("Value out of range for int64."); - setError(SF_STATUS_ERROR_OUT_OF_RANGE, - "Value out of range for int64."); - return SF_STATUS_ERROR_OUT_OF_RANGE; - } - - *out_data = static_cast(value); - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsUint8(size_t idx, uint8 * out_data) -{ - return getCellAsInt8(idx, (int8*)out_data); -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsUint32(size_t idx, uint32 * out_data) -{ - *out_data = 0; - - uint64 value = 0; - SF_STATUS ret = getCellAsUint64(idx, &value); - - if (SF_STATUS_SUCCESS != ret) - { - return ret; - } - - if (value > SF_UINT32_MAX) - { - CXX_LOG_ERROR("Value out of range for uint32."); - setError(SF_STATUS_ERROR_OUT_OF_RANGE, - "Value out of range for uint32."); - return SF_STATUS_ERROR_OUT_OF_RANGE; - } - - *out_data = static_cast(value); - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsUint64(size_t idx, uint64 * out_data) -{ - VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); - - *out_data = 0; - - if (!PUTGET_COLUMNS[m_cmdType][idx - 1].isInteger) - { - setError(SF_STATUS_ERROR_CONVERSION_FAILURE, - "Invalid conversion from string column to integer."); - return SF_STATUS_ERROR_CONVERSION_FAILURE; - } - - uint64 value = 0; - size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; - - try - { - value = std::stoull(m_values[row_idx][idx - 1], NULL, 10); - } - catch (...) - { - CXX_LOG_ERROR("Cannot convert value to uint64."); - setError(SF_STATUS_ERROR_CONVERSION_FAILURE, - "Cannot convert value to uint64."); - return SF_STATUS_ERROR_CONVERSION_FAILURE; - } - - *out_data = value; - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat32(size_t idx, float32 * out_data) -{ - *out_data = 0; - - uint64 value = 0; - SF_STATUS ret = getCellAsUint64(idx, &value); - - if (SF_STATUS_SUCCESS != ret) - { - return ret; - } - - *out_data = (float32)value; - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsFloat64(size_t idx, float64 * out_data) -{ - *out_data = 0; - - uint64 value = 0; - SF_STATUS ret = getCellAsUint64(idx, &value); - - if (SF_STATUS_SUCCESS != ret) - { - return ret; - } - - *out_data = (float64)value; - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsConstString(size_t idx, const char ** out_data) -{ - VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); - - size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; - - *out_data = m_values[row_idx][idx - 1].c_str(); - return SF_STATUS_SUCCESS; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data) -{ - setError(SF_STATUS_ERROR_CONVERSION_FAILURE, - "Value cannot be converted to timestamp."); - return SF_STATUS_ERROR_CONVERSION_FAILURE; -} - -SF_STATUS STDCALL ResultSetPutGet::getCellStrlen(size_t idx, size_t * out_data) -{ - VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); - - size_t row_idx = m_currRowIdx > 0 ? m_currRowIdx - 1 : 0; - - *out_data = m_values[row_idx][idx - 1].length(); - return SF_STATUS_SUCCESS; -} - -size_t ResultSetPutGet::getRowCountInChunk() -{ - return m_values.size(); -} - -SF_STATUS STDCALL ResultSetPutGet::isCellNull(size_t idx, sf_bool * out_data) -{ - VERIFY_COLUMN_INDEX(idx, m_totalColumnCount); - *out_data = SF_BOOLEAN_FALSE; - - return SF_STATUS_SUCCESS; -} - -} // namespace Client -} // namespace Snowflake diff --git a/cpp/lib/ResultSetPutGet.hpp b/cpp/lib/ResultSetPutGet.hpp deleted file mode 100644 index 7e8e8c8681..0000000000 --- a/cpp/lib/ResultSetPutGet.hpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. - */ - -#ifndef SNOWFLAKECLIENT_RESULTSETPUTGET_HPP -#define SNOWFLAKECLIENT_RESULTSETPUTGET_HPP - -#include "ResultSet.hpp" -#include "snowflake/ITransferResult.hpp" - -namespace Snowflake -{ -namespace Client -{ - -/** - * Represents a result set retrieved from PUT/GET execution. - */ -class ResultSetPutGet : public Snowflake::Client::ResultSet -{ -public: - - /** - * Parameterized constructor. - * - * @param result A pointer to the transfer result. - */ - ResultSetPutGet(ITransferResult *result); - - /** - * Destructor. - */ - ~ResultSetPutGet(); - - /** - * Setup column description of the transfer result. - * - * param desc The output parameter to return the pointer to the buffer allocated - * for the column description. - * Needs to be freed by the caller using SF_FREE. - * - * @return The number of columns. - */ - size_t setup_column_desc(SF_COLUMN_DESC** desc); - - /** - * Advances the internal iterator to the next row. If there are no more rows to consume, - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL next(); - - /** - * Writes the value of the given cell as a boolean to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsBool(size_t idx, sf_bool * out_data); - - /** - * Writes the value of the given cell as an int8 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsInt8(size_t idx, int8 * out_data); - - /** - * Writes the value of the given cell as an int32 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsInt32(size_t idx, int32 * out_data); - - /** - * Writes the value of the given cell as an int64 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsInt64(size_t idx, int64 * out_data); - - /** - * Writes the value of the given cell as a uint8 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsUint8(size_t idx, uint8 * out_data); - - /** - * Writes the value of the given cell as a uint32 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsUint32(size_t idx, uint32 * out_data); - - /** - * Writes the value of the given cell as a uint64 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsUint64(size_t idx, uint64 * out_data); - - /** - * Writes the value of the given cell as a float32 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsFloat32(size_t idx, float32 * out_data); - - /** - * Writes the value of the given cell as a float64 to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsFloat64(size_t idx, float64 * out_data); - - /** - * Writes the value of the given cell as a constant C-string to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsConstString(size_t idx, const char ** out_data); - - /** - * Writes the value of the given cell as a timestamp to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellAsTimestamp(size_t idx, SF_TIMESTAMP * out_data); - - /** - * Writes the length of the given cell to the provided buffer. - * - * @param idx The index of the column to retrieve. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL getCellStrlen(size_t idx, size_t * out_data); - - /** - * Gets the total number of rows in the current chunk being processed. - * - * @return the number of rows in the current chunk. - */ - size_t getRowCountInChunk(); - - /** - * Indicates whether the given cell is null. - * - * @param idx The index of the column to check is null. - * @param out_data The buffer to write to. - * - * @return 0 if successful, otherwise an error is returned. - */ - SF_STATUS STDCALL isCellNull(size_t idx, sf_bool * out_data); - -private: - - // Hidden default constructor. - ResultSetPutGet(); - - // the result values from transfer result - std::vector> m_values; - - // the command type of UPLOAD(PUT) or DOWNLOAD(GET) - CommandType m_cmdType; -}; - -} // namespace Client -} // namespace Snowflake - -#endif // SNOWFLAKECLIENT_RESULTSETPUTGET_HPP diff --git a/cpp/lib/result_set.cpp b/cpp/lib/result_set.cpp index 3fca6fe496..5cfcb1238a 100644 --- a/cpp/lib/result_set.cpp +++ b/cpp/lib/result_set.cpp @@ -4,239 +4,382 @@ #include "result_set.h" #include "ResultSet.hpp" -#include "ResultSetArrow.hpp" -#include "ResultSetJson.hpp" -#include "ResultSetPutGet.hpp" #ifdef __cplusplus extern "C" { #endif - result_set_ptr rs_create_with_json_result( + void * rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat query_result_format, + QueryResultFormat_t * query_result_format, const char * tz_string ) { - return Snowflake::Client::ResultSet::CreateResultFromJson( - json_rowset, metadata, query_result_format, std::string(tz_string)); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_create_with_json_result(json_rowset, metadata, tz_string); + case JSON_FORMAT: + return rs_json_create(json_rowset, metadata, tz_string); + default: + return nullptr; + } } - result_set_ptr rs_create_with_chunk( + void * rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat query_result_format, + QueryResultFormat_t * query_result_format, const char * tz_string ) { - return Snowflake::Client::ResultSet::CreateResultFromChunk( - initial_chunk, metadata, query_result_format, std::string(tz_string)); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_create_with_chunk((NON_JSON_RESP*)initial_chunk, metadata, tz_string); + case JSON_FORMAT: + return rs_json_create((cJSON*)initial_chunk, metadata, tz_string); + default: + return nullptr; + } } - void rs_destroy(result_set_ptr rs) + void rs_destroy(void * rs, QueryResultFormat_t * query_result_format) { - if (!rs) - { - return; + switch (*query_result_format){ + case ARROW_FORMAT: + rs_arrow_destroy((rs_arrow_t *) rs); + break; + case JSON_FORMAT: + rs_json_destroy((rs_json_t *) rs); + break; + default: + break; } - delete static_cast(rs); } -#define ERROR_IF_NULL(ptr) \ -{ \ - if (ptr == NULL) { return SF_STATUS_ERROR_NULL_POINTER; } \ -} - SF_STATUS STDCALL - rs_append_chunk(result_set_ptr rs, void * chunk) + rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk) { - ERROR_IF_NULL(rs); - return static_cast(rs)->appendChunk(chunk); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_append_chunk((rs_arrow_t *) rs, (NON_JSON_RESP*)chunk); + case JSON_FORMAT: + return rs_json_append_chunk((rs_json_t *) rs, (cJSON*)chunk); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } - SF_STATUS STDCALL rs_next(result_set_ptr rs) + SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format) { - ERROR_IF_NULL(rs); - return static_cast(rs)->next(); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_next((rs_arrow_t *) rs); + case JSON_FORMAT: + return rs_json_next((rs_json_t *) rs); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_bool( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, sf_bool * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsBool(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_bool((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_bool((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_int8( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int8 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsInt8(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_int8((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_int8((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_int32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int32 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsInt32(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_int32((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_int32((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_int64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int64 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsInt64(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_int64((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_int64((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_uint8( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint8 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsUint8(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_uint8((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_uint8((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_uint32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint32 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsUint32(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_uint32((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_uint32((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_uint64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint64 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsUint64(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_uint64((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_uint64((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_float32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, float32 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsFloat32(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_float32((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_float32((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_float64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, float64 * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsFloat64(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_float64((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_float64((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_const_string( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, const char ** out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsConstString(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_const_string((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_const_string((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_as_timestamp( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, SF_TIMESTAMP * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellAsTimestamp(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_as_timestamp((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_as_timestamp((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } SF_STATUS STDCALL rs_get_cell_strlen( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, size_t * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getCellStrlen(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_cell_strlen((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_get_cell_strlen((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } - size_t rs_get_row_count_in_chunk(result_set_ptr rs) + size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getRowCountInChunk(); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_get_row_count_in_chunk((rs_arrow_t *) rs); + case JSON_FORMAT: + return rs_json_get_row_count_in_chunk((rs_json_t *) rs); + default: + return 0; + } } SF_STATUS STDCALL rs_is_cell_null( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, sf_bool * out_data ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->isCellNull(idx, out_data); + switch (*query_result_format) + { + case ARROW_FORMAT: + return rs_arrow_is_cell_null((rs_arrow_t *) rs, idx, out_data); + case JSON_FORMAT: + return rs_json_is_cell_null((rs_json_t *) rs, idx, out_data); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; + } } - SF_STATUS STDCALL rs_get_error(result_set_ptr rs) + SF_STATUS STDCALL rs_get_error( + void * rs, + QueryResultFormat_t * query_result_format + ) { - ERROR_IF_NULL(rs); - return static_cast(rs)->getError(); - } + if (!rs || !query_result_format) + { + return SF_STATUS_ERROR_NULL_POINTER; + } - const char* rs_get_error_message(result_set_ptr rs) - { - if (!rs) + Snowflake::Client::ResultSet * rs_obj = NULL; + switch (*query_result_format) { - return ""; + case ARROW_FORMAT: + rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); + case JSON_FORMAT: + rs_obj = static_cast(((rs_json_t *)rs)->rs_object); + default: + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; } - return static_cast(rs)->getErrorMessage(); + + return rs_obj->getError(); } -#ifndef SF_WIN32 - size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) + const char* rs_get_error_message( + void * rs, + QueryResultFormat_t * query_result_format + ) { - size_t data_size = size * nmemb; - arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); + if (!rs || !query_result_format) + { + return ""; + } - log_debug("Curl response for arrow chunk size: %zu", data_size); - (void) arrowBufBuilder->Append(ptr, data_size); - return data_size; - } + Snowflake::Client::ResultSet * rs_obj = NULL; + switch (*query_result_format) + { + case ARROW_FORMAT: + rs_obj = static_cast(((rs_arrow_t *)rs)->rs_object); + case JSON_FORMAT: + rs_obj = static_cast(((rs_json_t *)rs)->rs_object); + default: + return ""; + } - NON_JSON_RESP* callback_create_arrow_resp(void) - { - NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; - arrow_resp->buffer = new arrow::BufferBuilder(); - arrow_resp->write_callback = arrow_write_callback; - return arrow_resp; + return rs_obj->getErrorMessage(); } -#else - NON_JSON_RESP* callback_create_arrow_resp(void) - { - log_error("Query results were fetched using Arrow"); - return NULL; - } -#endif #ifdef __cplusplus } // extern "C" diff --git a/include/snowflake/client.h b/include/snowflake/client.h index f69f1353e1..6bddfc2d6e 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -15,15 +15,7 @@ extern "C" { #include "version.h" #include "logger.h" -/** - * API Name - */ -/* TODO: Temporarily change to ODBC for now to pass the test before - * features (PUT for GCP, multiple statements etc.) unblocked - * on server side. - * Need to revert to C_API when merging to master. - */ -#define SF_API_NAME "ODBC" +#define SF_API_NAME "C API" /** * SQLState code length @@ -169,8 +161,7 @@ typedef enum SF_STATUS { SF_STATUS_ERROR_NULL_POINTER = 240022, SF_STATUS_ERROR_BUFFER_TOO_SMALL = 240023, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT = 240024, - SF_STATUS_ERROR_OTHER = 240025, - SF_STATUS_ERROR_FILE_TRANSFER = 240026 + SF_STATUS_ERROR_OTHER = 240025 } SF_STATUS; /** @@ -274,14 +265,6 @@ typedef enum SF_ATTRIBUTE { SF_CON_MAX_BINARY_SIZE, SF_CON_MAX_VARIANT_SIZE, SF_CON_OCSP_FAIL_OPEN, - SF_CON_PUT_TEMPDIR, - SF_CON_PUT_COMPRESSLV, - SF_CON_PUT_USE_URANDOM_DEV, - SF_CON_PUT_FASTFAIL, - SF_CON_PUT_MAXRETRIES, - SF_CON_GET_FASTFAIL, - SF_CON_GET_MAXRETRIES, - SF_CON_GET_THRESHOLD, SF_DIR_QUERY_URL, SF_DIR_QUERY_URL_PARAM, SF_DIR_QUERY_TOKEN, @@ -308,11 +291,8 @@ typedef enum SF_GLOBAL_ATTRIBUTE { * Attributes for Snowflake statement context. */ typedef enum SF_STMT_ATTRIBUTE { - SF_STMT_USER_REALLOC_FUNC, - SF_STMT_MULTI_STMT_COUNT + SF_STMT_USER_REALLOC_FUNC } SF_STMT_ATTRIBUTE; -#define SF_MULTI_STMT_COUNT_UNSET (-1) -#define SF_MULTI_STMT_COUNT_UNLIMITED 0 /** * Snowflake Error @@ -424,17 +404,6 @@ typedef struct SF_CONNECT { //token for OAuth authentication char *oauth_token; - - // put get configurations - sf_bool use_s3_regional_url; - sf_bool put_use_urand_dev; - int8 put_compress_level; - char* put_temp_dir; - sf_bool put_fastfail; - int8 put_maxretries; - sf_bool get_fastfail; - int8 get_maxretries; - int64 get_threshold; } SF_CONNECT; /** @@ -484,16 +453,6 @@ typedef struct SF_CHUNK_DOWNLOADER SF_CHUNK_DOWNLOADER; */ typedef struct SF_PUT_GET_RESPONSE SF_PUT_GET_RESPONSE; -typedef void* result_set_ptr; - -/** - * An enumeration over all supported query result formats. - */ -typedef enum QueryResultFormat_e -{ - SF_ARROW_FORMAT, SF_JSON_FORMAT, SF_PUTGET_FORMAT, SF_FORMAT_UNKNOWN -} QueryResultFormat; - /** * Statement context */ @@ -503,9 +462,9 @@ typedef struct SF_STMT { char request_id[SF_UUID4_LEN]; SF_ERROR_STRUCT error; SF_CONNECT *connection; - QueryResultFormat qrf; + void* qrf; char *sql_text; - result_set_ptr result_set; + void* result_set; int64 chunk_rowcount; int64 total_rowcount; int64 total_fieldcount; @@ -517,9 +476,6 @@ typedef struct SF_STMT { SF_STATS *stats; void *stmt_attrs; sf_bool is_dml; - sf_bool is_multi_stmt; - void* multi_stmt_result_ids; - int64 multi_stmt_count; /** * User realloc function used in snowflake_fetch @@ -837,15 +793,6 @@ SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture); -/** - * Determines whether more results are available and, if so, - * initializes processing for the next one. - * @param sfstmt SNOWFLAKE_STMT context. - * - * @return 0 if success, otherwise an errno is returned. - */ -SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt); - /** * Fetches the next row for the statement and stores on the bound buffer * if any. Noop if no buffer is bound. diff --git a/lib/chunk_downloader.c b/lib/chunk_downloader.c index 5657cb4465..6f6e923cb2 100644 --- a/lib/chunk_downloader.c +++ b/lib/chunk_downloader.c @@ -217,8 +217,8 @@ sf_bool STDCALL download_chunk(char *url, SF_HEADER *headers, CURL *curl = get_curl_from_desc(curl_desc); if (!curl || - !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, NULL, chunk, - non_json_resp, NULL, network_timeout, + !http_perform(curl, GET_REQUEST_TYPE, url, headers, NULL, chunk, + non_json_resp, network_timeout, SF_BOOLEAN_TRUE, error, insecure_mode, fail_open, 0, 0, retry_max_count, NULL, NULL, NULL, SF_BOOLEAN_FALSE, proxy, no_proxy, SF_BOOLEAN_FALSE, SF_BOOLEAN_FALSE)) { diff --git a/lib/client.c b/lib/client.c index 7a2148e42e..8ee4b58567 100644 --- a/lib/client.c +++ b/lib/client.c @@ -37,7 +37,7 @@ sf_bool DEBUG; sf_bool SF_OCSP_CHECK; char *SF_HEADER_USER_AGENT = NULL; -static char *CLIENT_CONFIG_FILE = NULL; +static char* CLIENT_CONFIG_FILE = NULL; static char *LOG_PATH = NULL; static FILE *LOG_FP = NULL; @@ -68,7 +68,6 @@ sf_bool validate_application(const char *application); #define _SF_STMT_TYPE_DELETE (_SF_STMT_TYPE_DML + 0x300) #define _SF_STMT_TYPE_MERGE (_SF_STMT_TYPE_DML + 0x400) #define _SF_STMT_TYPE_MULTI_TABLE_INSERT (_SF_STMT_TYPE_DML + 0x500) -#define _SF_STMT_TYPE_MULTI_STMT 0xA000 /** * Detects statement type is DML @@ -185,9 +184,6 @@ static SF_STATUS STDCALL _reset_connection_parameters( else if (strcmp(name->valuestring, "VARIANT_MAX_SIZE_IN_RESULT") == 0) { sf->max_variant_size = snowflake_cJSON_GetUint64Value(value); } - else if (strcmp(name->valuestring, "ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1") == 0) { - sf->use_s3_regional_url = snowflake_cJSON_IsTrue(value) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; - } } } SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -312,7 +308,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { SF_LOG_LEVEL sf_log_level = log_level; char strerror_buf[SF_ERROR_BUFSIZE]; - char client_config_file[MAX_PATH] = {0}; + char client_config_file[MAX_PATH] = { 0 }; snowflake_global_get_attribute( SF_GLOBAL_CLIENT_CONFIG_FILE, client_config_file, sizeof(client_config_file)); client_config clientConfig = { 0 }; @@ -327,7 +323,8 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { if (sf_log_path == NULL) { if (log_path && strlen(log_path) != 0) { sf_log_path = log_path; - } else if (clientConfig.logPath != NULL) { + } + else if (clientConfig.logPath != NULL) { sf_log_path = clientConfig.logPath; } } @@ -335,7 +332,8 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { sf_log_level_str = sf_getenv_s("SNOWFLAKE_LOG_LEVEL", log_level_buf, sizeof(log_level_buf)); if (sf_log_level_str != NULL) { sf_log_level = log_from_str_to_level(sf_log_level_str); - } else if (sf_log_level == SF_LOG_DEFAULT) { + } + else if (sf_log_level == SF_LOG_DEFAULT) { if (clientConfig.logLevel != NULL) { sf_log_level = log_from_str_to_level(clientConfig.logLevel); } @@ -378,7 +376,7 @@ static sf_bool STDCALL log_init(const char *log_path, SF_LOG_LEVEL log_level) { ret = SF_BOOLEAN_TRUE; -cleanup: + cleanup: free_client_config(&clientConfig); return ret; } @@ -574,19 +572,11 @@ _snowflake_check_connection_parameters(SF_CONNECT *sf) { log_debug("retry_count: %d", sf->retry_count); log_debug("qcc_disable: %s", sf->qcc_disable ? "true" : "false"); log_debug("include_retry_reason: %s", sf->include_retry_reason ? "true" : "false"); - log_debug("use_s3_regional_url: %s", sf->use_s3_regional_url ? "true" : "false"); - log_debug("put_use_urand_dev: %s", sf->put_use_urand_dev ? "true" : "false"); - log_debug("put_compress_level: %d", sf->put_compress_level); - log_debug("put_temp_dir: %s", sf->put_temp_dir ? sf->put_temp_dir : ""); - log_debug("put_fastfail: %s", sf->put_fastfail ? "true" : "false"); - log_debug("put_maxretries: %d", sf->put_maxretries); - log_debug("get_fastfail: %s", sf->get_fastfail ? "true" : "false"); - log_debug("get_maxretries: %d", sf->get_maxretries); - log_debug("get_threshold: %d", sf->get_threshold); return SF_STATUS_SUCCESS; } + SF_STATUS STDCALL snowflake_global_init( const char *log_path, SF_LOG_LEVEL log_level, SF_USER_MEM_HOOKS *hooks) { SF_STATUS ret = SF_STATUS_ERROR_GENERAL; @@ -602,13 +592,11 @@ SF_STATUS STDCALL snowflake_global_init( _snowflake_memory_hooks_setup(hooks); sf_memory_init(); sf_error_init(); - if (!log_init(log_path, log_level)) { // no way to log error because log_init failed. sf_fprintf(stderr, "Error during log initialization"); goto cleanup; } - CURLcode curl_ret = curl_global_init(CURL_GLOBAL_DEFAULT); if (curl_ret != CURLE_OK) { log_fatal("curl_global_init() failed: %s", @@ -803,18 +791,7 @@ SF_CONNECT *STDCALL snowflake_init() { sf->max_varchar_size = SF_DEFAULT_MAX_OBJECT_SIZE; sf->max_binary_size = SF_DEFAULT_MAX_OBJECT_SIZE / 2; sf->max_variant_size = SF_DEFAULT_MAX_OBJECT_SIZE; - sf->oauth_token = NULL; - - sf->use_s3_regional_url = SF_BOOLEAN_FALSE; - sf->put_use_urand_dev = SF_BOOLEAN_FALSE; - sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; - sf->put_temp_dir = NULL; - sf->put_fastfail = SF_BOOLEAN_FALSE; - sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; - sf->get_fastfail = SF_BOOLEAN_FALSE; - sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; - sf->get_threshold = SF_DEFAULT_GET_THRESHOLD; } return sf; @@ -1239,45 +1216,6 @@ SF_STATUS STDCALL snowflake_set_attribute( case SF_CON_INCLUDE_RETRY_REASON: sf->include_retry_reason = value ? *((sf_bool *)value) : SF_BOOLEAN_TRUE; break; - case SF_CON_PUT_TEMPDIR: - alloc_buffer_and_copy(&sf->put_temp_dir, value); - break; - case SF_CON_PUT_COMPRESSLV: - sf->put_compress_level = value ? *((int8 *)value) : SF_DEFAULT_PUT_COMPRESS_LEVEL; - if ((sf->put_compress_level > SF_MAX_PUT_COMPRESS_LEVEL) || - (sf->put_compress_level < 0)) - { - sf->put_compress_level = SF_DEFAULT_PUT_COMPRESS_LEVEL; - } - break; - case SF_CON_PUT_USE_URANDOM_DEV: - sf->put_use_urand_dev = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; - break; - case SF_CON_PUT_FASTFAIL: - sf->put_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; - break; - case SF_CON_PUT_MAXRETRIES: - sf->put_maxretries = value ? *((int8 *)value) : SF_DEFAULT_PUT_MAX_RETRIES; - if ((sf->put_maxretries > SF_MAX_PUT_MAX_RETRIES) || - (sf->put_maxretries < 0)) - { - sf->put_maxretries = SF_DEFAULT_PUT_MAX_RETRIES; - } - break; - case SF_CON_GET_FASTFAIL: - sf->get_fastfail = value ? *((sf_bool *)value) : SF_BOOLEAN_FALSE; - break; - case SF_CON_GET_MAXRETRIES: - sf->get_maxretries = value ? *((int8 *)value) : SF_DEFAULT_GET_MAX_RETRIES; - if ((sf->get_maxretries > SF_MAX_GET_MAX_RETRIES) || - (sf->get_maxretries < 0)) - { - sf->get_maxretries = SF_DEFAULT_GET_MAX_RETRIES; - } - break; - case SF_CON_GET_THRESHOLD: - sf->get_threshold = value ? *((int64 *)value) : SF_DEFAULT_GET_THRESHOLD; - break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -1427,30 +1365,6 @@ SF_STATUS STDCALL snowflake_get_attribute( case SF_CON_MAX_VARIANT_SIZE: *value = &sf->max_variant_size; break; - case SF_CON_PUT_TEMPDIR: - *value = sf->put_temp_dir; - break; - case SF_CON_PUT_COMPRESSLV: - *value = &sf->put_compress_level; - break; - case SF_CON_PUT_USE_URANDOM_DEV: - *value = &sf->put_use_urand_dev; - break; - case SF_CON_PUT_FASTFAIL: - *value = &sf->put_fastfail; - break; - case SF_CON_PUT_MAXRETRIES: - *value = &sf->put_maxretries; - break; - case SF_CON_GET_FASTFAIL: - *value = &sf->get_fastfail; - break; - case SF_CON_GET_MAXRETRIES: - *value = &sf->get_maxretries; - break; - case SF_CON_GET_THRESHOLD: - *value = &sf->get_threshold; - break; default: SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, "Invalid attribute type", @@ -1487,316 +1401,6 @@ static void STDCALL _snowflake_stmt_row_metadata_reset(SF_STMT *sfstmt) { sfstmt->stats = NULL; } -/** - * Setup result set from json response. - * could be result of a regular query or one of the results - * in multiple statements - */ -static sf_bool setup_result_with_json_resp(SF_STMT* sfstmt, cJSON* data) -{ - // Set Database info - _mutex_lock(&sfstmt->connection->mutex_parameters); - /* Set other parameters. Ignore the status */ - _set_current_objects(sfstmt, data); - _set_parameters_session_info(sfstmt->connection, data); - qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); - _mutex_unlock(&sfstmt->connection->mutex_parameters); - - // clean up from preivous result - sfstmt->chunk_rowcount = -1; - sfstmt->total_rowcount = -1; - sfstmt->total_fieldcount = -1; - sfstmt->total_row_index = -1; - - // Destroy chunk downloader - chunk_downloader_term(sfstmt->chunk_downloader); - sfstmt->chunk_downloader = NULL; - - int64 stmt_type_id; - if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { - /* failed to get statement type id */ - sfstmt->is_dml = SF_BOOLEAN_FALSE; - } else { - sfstmt->is_dml = detect_stmt_type(stmt_type_id); - } - cJSON* rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); - if (snowflake_cJSON_IsArray(rowtype)) { - _snowflake_stmt_desc_reset(sfstmt); - sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( - rowtype); - sfstmt->desc = set_description(rowtype); - } - cJSON* stats = snowflake_cJSON_GetObjectItem(data, "stats"); - if (snowflake_cJSON_IsObject(stats)) { - _snowflake_stmt_row_metadata_reset(sfstmt); - sfstmt->stats = set_stats(stats); - } else { - sfstmt->stats = NULL; - } - - // Determine query result format and detach rowset object from data. - cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); - char * qrf_str = snowflake_cJSON_GetStringValue(qrf); - cJSON * rowset = NULL; - - if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { -#ifdef SF_WIN32 - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, - "Query results were fetched using Arrow, " - "but the client library does not yet support decoding Arrow results", "", - sfstmt->sfqid); - - return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; -#endif - sfstmt->qrf = SF_ARROW_FORMAT; - rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); - if (!rowset) - { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); - return SF_BOOLEAN_FALSE; - } - } - else if (strcmp(qrf_str, "json") == 0) { - sfstmt->qrf = SF_JSON_FORMAT; - if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) - { - log_error("No valid rowset found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_JSON, - "Missing rowset from response. No results found.", - SF_SQLSTATE_APP_REJECT_CONNECTION, - sfstmt->sfqid); - return SF_BOOLEAN_FALSE; - } - } - else { - log_error("Unsupported query result format: %s", qrf_str); - return SF_BOOLEAN_FALSE; - } - // Index starts at 0 and incremented each fetch - sfstmt->total_row_index = 0; - cJSON* chunks = NULL; - cJSON* chunk_headers = NULL; - char* qrmk = NULL; - // When the result set is sufficient large, the server response will contain - // an empty "rowset" object. Instead, it will have a "chunks" object that contains, - // among other fields, a URL from which the result set can be downloaded in chunks. - // In this case, we initialize the chunk downloader, which will download in the - // background as calls to snowflake_fetch() are made. - if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { - // We don't care if there is no qrmk, so ignore return code - json_copy_string(&qrmk, data, "qrmk"); - chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); - NON_JSON_RESP* (*callback_create_resp)(void) = NULL; - if (SF_ARROW_FORMAT == sfstmt->qrf) { - callback_create_resp = callback_create_arrow_resp; - } - sfstmt->chunk_downloader = chunk_downloader_init( - qrmk, - chunk_headers, - chunks, - 2, // thread count - 4, // fetch slot - &sfstmt->error, - sfstmt->connection->insecure_mode, - sfstmt->connection->ocsp_fail_open, - callback_create_resp, - sfstmt->connection->proxy, - sfstmt->connection->no_proxy, - get_retry_timeout(sfstmt->connection), - sfstmt->connection->retry_count); - SF_FREE(qrmk); - if (!sfstmt->chunk_downloader) { - // Unable to create chunk downloader. - // Error is set in chunk_downloader_init function. - return SF_BOOLEAN_FALSE; - } - // Even when the result set is split into chunks, JSON format will still - // response with the first chunk in "rowset", so be sure to include it. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - sfstmt->qrf, - sfstmt->connection->timezone); - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set); - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } else { - // Create a result set object and update the total rowcount. - sfstmt->result_set = rs_create_with_json_result( - rowset, - sfstmt->desc, - sfstmt->qrf, - sfstmt->connection->timezone); - // Update chunk row count. Controls the chunk downloader. - sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( - sfstmt->result_set); - // Update total row count. Used in snowflake_num_rows(). - if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { - log_warn( - "No total count found in response. Reverting to using array size of results"); - sfstmt->total_rowcount = sfstmt->chunk_rowcount; - } - } - - return SF_BOOLEAN_TRUE; -} - -/** - * Setup result set from json response. - * could be result of a regular query or one of the results - * in multiple statements - */ -static sf_bool setup_multi_stmt_result(SF_STMT* sfstmt, cJSON* data) -{ - // Set Database info - _mutex_lock(&sfstmt->connection->mutex_parameters); - /* Set other parameters. Ignore the status */ - _set_current_objects(sfstmt, data); - _set_parameters_session_info(sfstmt->connection, data); - qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); - _mutex_unlock(&sfstmt->connection->mutex_parameters); - - if (sfstmt->multi_stmt_result_ids) - { - snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); - sfstmt->multi_stmt_result_ids = NULL; - } - char* result_ids = NULL; - if (json_copy_string(&result_ids, data, "resultIds")) - { - log_error("No valid resultIds found in response"); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_RESPONSE, - "No valid resultIds found in multiple statements response.", - SF_SQLSTATE_GENERAL_ERROR, - sfstmt->sfqid); - return SF_BOOLEAN_FALSE; - } - - // split result ids with comma(,) - cJSON* result_ids_json = snowflake_cJSON_CreateArray(); - char* start = result_ids; - char* end = NULL; - size_t len = strlen(result_ids); - while ((start - result_ids) < len) - { - end = strchr(start, ','); - if (!end) - { - // last part, set to end of the entire string - end = result_ids + len; - } - *end = '\0'; - snowflake_cJSON_AddItemToArray(result_ids_json, snowflake_cJSON_CreateString(start)); - start = end + 1; - } - - sfstmt->multi_stmt_result_ids = result_ids_json; - - return SF_STATUS_SUCCESS == snowflake_next_result(sfstmt); -} - -SF_STATUS STDCALL snowflake_next_result(SF_STMT* sfstmt) -{ - cJSON* result_id_json = NULL; - if (!sfstmt || !sfstmt->is_multi_stmt || !sfstmt->multi_stmt_result_ids || - !snowflake_cJSON_IsArray(sfstmt->multi_stmt_result_ids) || - !(result_id_json = snowflake_cJSON_DetachItemFromArray(sfstmt->multi_stmt_result_ids, 0))) - { - // no more results available. - return SF_STATUS_EOF; - } - - char* result_id = snowflake_cJSON_GetStringValue(result_id_json); - if (!result_id || (strlen(result_id) == 0)) - { - log_error("Empty result id found for multiple statements."); - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_BAD_RESPONSE, - "Empty result id found in multiple statements response.", - SF_SQLSTATE_GENERAL_ERROR, - sfstmt->sfqid); - snowflake_cJSON_Delete(result_id_json); - return SF_STATUS_ERROR_BAD_RESPONSE; - } - - char* result_url = NULL; - size_t url_size = strlen(QUERY_RESULT_URL_FORMAT) - 2 + strlen(result_id) + 1; - result_url = (char*)SF_CALLOC(1, url_size); - if (!result_url) - { - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, - SF_STATUS_ERROR_OUT_OF_MEMORY, - "Run out of memory trying to create result url.", - SF_SQLSTATE_MEMORY_ALLOCATION_ERROR, - sfstmt->sfqid); - snowflake_cJSON_Delete(result_id_json); - return SF_STATUS_ERROR_OUT_OF_MEMORY; - } - sf_sprintf(result_url, url_size, QUERY_RESULT_URL_FORMAT, result_id); - snowflake_cJSON_Delete(result_id_json); - - cJSON* result = NULL; - if (!request(sfstmt->connection, &result, result_url, NULL, 0, NULL, NULL, - GET_REQUEST_TYPE, &sfstmt->error, SF_BOOLEAN_FALSE, - 0, sfstmt->connection->retry_count, get_retry_timeout(sfstmt->connection), - NULL, NULL, NULL, SF_BOOLEAN_FALSE)) - { - SF_FREE(result_url); - return sfstmt->error.error_code; - } - SF_FREE(result_url); - - cJSON* data = snowflake_cJSON_GetObjectItem(result, "data"); - - sf_bool success = SF_BOOLEAN_FALSE; - if ((json_copy_bool(&success, result, "success") != SF_JSON_ERROR_NONE) || !success) - { - cJSON *messageJson = NULL; - char *message = NULL; - cJSON *codeJson = NULL; - int64 code = -1; - if (json_copy_string_no_alloc(sfstmt->error.sqlstate, data, - "sqlState", SF_SQLSTATE_LEN)) { - log_debug("No valid sqlstate found in response"); - } - messageJson = snowflake_cJSON_GetObjectItem(result, "message"); - if (messageJson) { - message = messageJson->valuestring; - } - codeJson = snowflake_cJSON_GetObjectItem(result, "code"); - if (codeJson) { - code = (int64) strtol(codeJson->valuestring, NULL, 10); - } else { - log_debug("no code element."); - } - SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, code, - message ? message - : "Query was not successful", - NULL, sfstmt->sfqid); - - snowflake_cJSON_Delete(result); - return sfstmt->error.error_code; - } - - setup_result_with_json_resp(sfstmt, data); - snowflake_cJSON_Delete(result); - - return SF_STATUS_SUCCESS; -} - /** * Returns what kind of params are being bound - Named / Positional * based on the first bind input entry. Should be used only if @@ -1921,18 +1525,14 @@ static void STDCALL _snowflake_stmt_reset(SF_STMT *sfstmt) { sfstmt->sql_text = NULL; if (sfstmt->result_set) { - rs_destroy(sfstmt->result_set); + rs_destroy(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); } sfstmt->result_set = NULL; - sfstmt->qrf = SF_FORMAT_UNKNOWN; - sfstmt->is_dml = SF_BOOLEAN_FALSE; - sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; - if (sfstmt->multi_stmt_result_ids) - { - snowflake_cJSON_Delete(sfstmt->multi_stmt_result_ids); + if (sfstmt->qrf) { + SF_FREE(sfstmt->qrf); } - sfstmt->multi_stmt_result_ids = NULL; + sfstmt->qrf = NULL; if (_snowflake_get_current_param_style(sfstmt) == NAMED) { @@ -2023,7 +1623,7 @@ SF_STMT *STDCALL snowflake_stmt(SF_CONNECT *sf) { if (sfstmt) { _snowflake_stmt_reset(sfstmt); sfstmt->connection = sf; - sfstmt->multi_stmt_count = SF_MULTI_STMT_COUNT_UNSET; + } return sfstmt; } @@ -2198,29 +1798,6 @@ SF_STATUS STDCALL snowflake_query( return SF_STATUS_SUCCESS; } -SF_STATUS STDCALL _snowflake_query_put_get_legacy( - SF_STMT *sfstmt, const char *command, size_t command_size) { - if (!sfstmt) { - return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; - } - clear_snowflake_error(&sfstmt->error); - SF_STATUS ret = snowflake_prepare(sfstmt, command, command_size); - if (ret != SF_STATUS_SUCCESS) { - return ret; - } - if (!_is_put_get_command(sfstmt->sql_text)) - { - // this should never happen as this function should only be - // called internally for put/get command. - SET_SNOWFLAKE_ERROR(&sfstmt->error, SF_STATUS_ERROR_GENERAL, - "Invalid query type, can be used for put get only", - SF_SQLSTATE_GENERAL_ERROR); - return SF_STATUS_ERROR_GENERAL; - } - - return _snowflake_execute_ex(sfstmt, SF_BOOLEAN_TRUE, SF_BOOLEAN_FALSE, NULL, SF_BOOLEAN_FALSE); -} - SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; @@ -2276,11 +1853,12 @@ SF_STATUS STDCALL snowflake_fetch(SF_STMT *sfstmt) { sfstmt->result_set = rs_create_with_chunk( sfstmt->chunk_downloader->queue[index].chunk, sfstmt->desc, - sfstmt->qrf, + (QueryResultFormat_t *) sfstmt->qrf, sfstmt->connection->timezone); } else { rs_append_chunk( sfstmt->result_set, + (QueryResultFormat_t *) sfstmt->qrf, sfstmt->chunk_downloader->queue[index].chunk); } @@ -2428,39 +2006,36 @@ snowflake_prepare(SF_STMT *sfstmt, const char *command, size_t command_size) { SF_STATUS STDCALL snowflake_describe_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_FALSE, result_capture, SF_BOOLEAN_TRUE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_TRUE); } SF_STATUS STDCALL snowflake_execute(SF_STMT *sfstmt) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, NULL, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), NULL, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL snowflake_execute_with_capture(SF_STMT *sfstmt, SF_QUERY_RESULT_CAPTURE *result_capture) { - return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), SF_BOOLEAN_TRUE, result_capture, SF_BOOLEAN_FALSE); + return _snowflake_execute_ex(sfstmt, _is_put_get_command(sfstmt->sql_text), result_capture, SF_BOOLEAN_FALSE); } SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, sf_bool is_put_get_command, - sf_bool is_native_put_get, SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only) { if (!sfstmt) { return SF_STATUS_ERROR_STATEMENT_NOT_EXIST; } - - if (is_put_get_command && is_native_put_get && !is_describe_only) - { - _snowflake_stmt_desc_reset(sfstmt); - return _snowflake_execute_put_get_native(sfstmt, result_capture); - } - clear_snowflake_error(&sfstmt->error); SF_STATUS ret = SF_STATUS_ERROR_GENERAL; SF_JSON_ERROR json_error; const char *error_msg; cJSON *body = NULL; cJSON *data = NULL; + cJSON *rowtype = NULL; + cJSON *stats = NULL; cJSON *resp = NULL; + cJSON *chunks = NULL; + cJSON *chunk_headers = NULL; + char *qrmk = NULL; char *s_body = NULL; char *s_resp = NULL; sf_bool success = SF_BOOLEAN_FALSE; @@ -2549,8 +2124,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, // Create Body body = create_query_json_body(sfstmt->sql_text, sfstmt->sequence_counter, is_string_empty(sfstmt->connection->directURL) ? - NULL : sfstmt->request_id, is_describe_only, - sfstmt->multi_stmt_count); + NULL : sfstmt->request_id, is_describe_only); if (bindings != NULL) { /* binding parameters if exists */ snowflake_cJSON_AddItemToObject(body, "bindings", bindings); @@ -2678,37 +2252,161 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, json_copy_string( &sfstmt->put_get_response->stage_info->stage_cred->azure_sas_token, stage_cred, "AZURE_SAS_TOKEN"); - json_copy_string( - &sfstmt->put_get_response->stage_info->stage_cred->gcs_access_token, - stage_cred, "GCS_ACCESS_TOKEN"); json_copy_string( &sfstmt->put_get_response->localLocation, data, "localLocation"); } else { + // Set Database info + _mutex_lock(&sfstmt->connection->mutex_parameters); + /* Set other parameters. Ignore the status */ + _set_current_objects(sfstmt, data); + _set_parameters_session_info(sfstmt->connection, data); + qcc_deserialize(sfstmt->connection, snowflake_cJSON_GetObjectItem(data, SF_QCC_RSP_KEY)); + _mutex_unlock(&sfstmt->connection->mutex_parameters); int64 stmt_type_id; if (json_copy_int(&stmt_type_id, data, "statementTypeId")) { - /* failed to get statement type id */ - sfstmt->is_multi_stmt = SF_BOOLEAN_FALSE; + /* failed to get statement type id */ + sfstmt->is_dml = SF_BOOLEAN_FALSE; + } else { + sfstmt->is_dml = detect_stmt_type(stmt_type_id); } - else { - sfstmt->is_multi_stmt = (_SF_STMT_TYPE_MULTI_STMT == stmt_type_id); + rowtype = snowflake_cJSON_GetObjectItem(data, "rowtype"); + if (snowflake_cJSON_IsArray(rowtype)) { + sfstmt->total_fieldcount = snowflake_cJSON_GetArraySize( + rowtype); + _snowflake_stmt_desc_reset(sfstmt); + sfstmt->desc = set_description(rowtype); + } + stats = snowflake_cJSON_GetObjectItem(data, "stats"); + if (snowflake_cJSON_IsObject(stats)) { + _snowflake_stmt_row_metadata_reset(sfstmt); + sfstmt->stats = set_stats(stats); + } else { + sfstmt->stats = NULL; } - if (sfstmt->is_multi_stmt) - { - if (!setup_multi_stmt_result(sfstmt, data)) + // Determine query result format and detach rowset object from data. + cJSON * qrf = snowflake_cJSON_GetObjectItem(data, "queryResultFormat"); + char * qrf_str = snowflake_cJSON_GetStringValue(qrf); + sfstmt->qrf = SF_CALLOC(1, sizeof(QueryResultFormat_t)); + cJSON * rowset = NULL; + + if (strcmp(qrf_str, "arrow") == 0 || strcmp(qrf_str, "arrow_force") == 0) { +#ifdef SF_WIN32 + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT, + "Query results were fetched using Arrow, " + "but the client library does not yet support decoding Arrow results", "", + sfstmt->sfqid); + + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +#endif + *((QueryResultFormat_t *) sfstmt->qrf) = ARROW_FORMAT; + rowset = snowflake_cJSON_DetachItemFromObject(data, "rowsetBase64"); + if (!rowset) { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); goto cleanup; } } - else - { - if (!setup_result_with_json_resp(sfstmt, data)) + else if (strcmp(qrf_str, "json") == 0) { + *((QueryResultFormat_t *) sfstmt->qrf) = JSON_FORMAT; + if (json_detach_array_from_object((cJSON **)(&rowset), data, "rowset")) { + log_error("No valid rowset found in response"); + SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, + SF_STATUS_ERROR_BAD_JSON, + "Missing rowset from response. No results found.", + SF_SQLSTATE_APP_REJECT_CONNECTION, + sfstmt->sfqid); goto cleanup; } } + else { + log_error("Unsupported query result format: %s", qrf_str); + } + + // Index starts at 0 and incremented each fetch + sfstmt->total_row_index = 0; + + // When the result set is sufficient large, the server response will contain + // an empty "rowset" object. Instead, it will have a "chunks" object that contains, + // among other fields, a URL from which the result set can be downloaded in chunks. + // In this case, we initialize the chunk downloader, which will download in the + // background as calls to snowflake_fetch() are made. + if ((chunks = snowflake_cJSON_GetObjectItem(data, "chunks")) != NULL) { + // We don't care if there is no qrmk, so ignore return code + json_copy_string(&qrmk, data, "qrmk"); + chunk_headers = snowflake_cJSON_GetObjectItem(data, "chunkHeaders"); + NON_JSON_RESP* (*callback_create_resp)(void) = NULL; + if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) { + callback_create_resp = callback_create_arrow_resp; + } + + sfstmt->chunk_downloader = chunk_downloader_init( + qrmk, + chunk_headers, + chunks, + 2, // thread count + 4, // fetch slot + &sfstmt->error, + sfstmt->connection->insecure_mode, + sfstmt->connection->ocsp_fail_open, + callback_create_resp, + sfstmt->connection->proxy, + sfstmt->connection->no_proxy, + get_retry_timeout(sfstmt->connection), + sfstmt->connection->retry_count); + if (!sfstmt->chunk_downloader) { + // Unable to create chunk downloader. + // Error is set in chunk_downloader_init function. + goto cleanup; + } + + // Even when the result set is split into chunks, JSON format will still + // response with the first chunk in "rowset", so be sure to include it. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + (QueryResultFormat_t *)sfstmt->qrf, + sfstmt->connection->timezone); + + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set, + (QueryResultFormat_t *) sfstmt->qrf); + + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } else { + // Create a result set object and update the total rowcount. + sfstmt->result_set = rs_create_with_json_result( + rowset, + sfstmt->desc, + (QueryResultFormat_t *) sfstmt->qrf, + sfstmt->connection->timezone); + + // Update chunk row count. Controls the chunk downloader. + sfstmt->chunk_rowcount = rs_get_row_count_in_chunk( + sfstmt->result_set, + (QueryResultFormat_t *) sfstmt->qrf); + + // Update total row count. Used in snowflake_num_rows(). + if (json_copy_int(&sfstmt->total_rowcount, data, "total")) { + log_warn( + "No total count found in response. Reverting to using array size of results"); + sfstmt->total_rowcount = sfstmt->chunk_rowcount; + } + } } } else if (json_error != SF_JSON_ERROR_NONE) { JSON_ERROR_MSG(json_error, error_msg, "Success code"); @@ -2756,6 +2454,7 @@ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, snowflake_cJSON_Delete(body); snowflake_cJSON_Delete(resp); SF_FREE(s_body); + SF_FREE(qrmk); if (result_capture == NULL) { // If no result capture, we always free s_resp SF_FREE(s_resp); @@ -2835,9 +2534,6 @@ SF_STATUS STDCALL snowflake_stmt_get_attr( case SF_STMT_USER_REALLOC_FUNC: *value = sfstmt->user_realloc_func; break; - case SF_STMT_MULTI_STMT_COUNT: - *value = &sfstmt->multi_stmt_count; - break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2858,9 +2554,6 @@ SF_STATUS STDCALL snowflake_stmt_set_attr( case SF_STMT_USER_REALLOC_FUNC: sfstmt->user_realloc_func = (void*(*)(void*, size_t))value; break; - case SF_STMT_MULTI_STMT_COUNT: - sfstmt->multi_stmt_count = value ? *((int64*)value) : SF_MULTI_STMT_COUNT_UNSET; - break; default: SET_SNOWFLAKE_ERROR( &sfstmt->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE, @@ -2915,7 +2608,7 @@ SF_STATUS STDCALL _snowflake_column_null_checks(SF_STMT *sfstmt, void *value_ptr } SF_STATUS STDCALL _snowflake_next(SF_STMT *sfstmt) { - return rs_next(sfstmt->result_set); + return rs_next(sfstmt->result_set, (QueryResultFormat_t *) sfstmt->qrf); } SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool *value_ptr) { @@ -2926,9 +2619,9 @@ SF_STATUS STDCALL snowflake_column_as_boolean(SF_STMT *sfstmt, int idx, sf_bool } if ((status = rs_get_cell_as_bool( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -2941,9 +2634,9 @@ SF_STATUS STDCALL snowflake_column_as_uint8(SF_STMT *sfstmt, int idx, uint8 *val } if ((status = rs_get_cell_as_uint8( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -2956,9 +2649,9 @@ SF_STATUS STDCALL snowflake_column_as_uint32(SF_STMT *sfstmt, int idx, uint32 *v } if ((status = rs_get_cell_as_uint32( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -2971,9 +2664,9 @@ SF_STATUS STDCALL snowflake_column_as_uint64(SF_STMT *sfstmt, int idx, uint64 *v } if ((status = rs_get_cell_as_uint64( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -2986,9 +2679,9 @@ SF_STATUS STDCALL snowflake_column_as_int8(SF_STMT *sfstmt, int idx, int8 *value } if ((status = rs_get_cell_as_int8( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3001,9 +2694,9 @@ SF_STATUS STDCALL snowflake_column_as_int32(SF_STMT *sfstmt, int idx, int32 *val } if ((status = rs_get_cell_as_int32( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3016,9 +2709,9 @@ SF_STATUS STDCALL snowflake_column_as_int64(SF_STMT *sfstmt, int idx, int64 *val } if ((status = rs_get_cell_as_int64( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3031,9 +2724,9 @@ SF_STATUS STDCALL snowflake_column_as_float32(SF_STMT *sfstmt, int idx, float32 } if ((status = rs_get_cell_as_float32( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3046,9 +2739,9 @@ SF_STATUS STDCALL snowflake_column_as_float64(SF_STMT *sfstmt, int idx, float64 } if ((status = rs_get_cell_as_float64( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3061,9 +2754,9 @@ SF_STATUS STDCALL snowflake_column_as_timestamp(SF_STMT *sfstmt, int idx, SF_TIM } if ((status = rs_get_cell_as_timestamp( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3076,9 +2769,9 @@ SF_STATUS STDCALL snowflake_column_as_const_str(SF_STMT *sfstmt, int idx, const } if ((status = rs_get_cell_as_const_string( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3299,15 +2992,16 @@ SF_STATUS STDCALL snowflake_column_as_str(SF_STMT *sfstmt, int idx, char **value const char* str_val = NULL; if ((status = rs_get_cell_as_const_string( sfstmt->result_set, + sfstmt->qrf, idx, &str_val)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); return status; } - if (SF_ARROW_FORMAT == sfstmt->qrf) + if (ARROW_FORMAT == *((QueryResultFormat_t *)sfstmt->qrf)) { // For Arrow the const string is formatted already return snowflake_raw_value_to_str_rep(sfstmt, str_val, @@ -3338,9 +3032,9 @@ SF_STATUS STDCALL snowflake_column_strlen(SF_STMT *sfstmt, int idx, size_t *valu } if ((status = rs_get_cell_strlen( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } @@ -3353,9 +3047,9 @@ SF_STATUS STDCALL snowflake_column_is_null(SF_STMT *sfstmt, int idx, sf_bool *va } if ((status = rs_is_cell_null( - sfstmt->result_set, idx, value_ptr)) != SF_STATUS_SUCCESS) { + sfstmt->result_set, sfstmt->qrf, idx, value_ptr)) != SF_STATUS_SUCCESS) { SET_SNOWFLAKE_STMT_ERROR(&sfstmt->error, status, - rs_get_error_message(sfstmt->result_set), "", sfstmt->sfqid); + rs_get_error_message(sfstmt->result_set, sfstmt->qrf), "", sfstmt->sfqid); } return status; } diff --git a/lib/client_int.h b/lib/client_int.h index 47db68c595..ed6b46c5a2 100644 --- a/lib/client_int.h +++ b/lib/client_int.h @@ -25,20 +25,10 @@ #define SF_DEFAULT_MAX_OBJECT_SIZE 16777216 -// defaults for put get configurations -#define SF_DEFAULT_PUT_COMPRESS_LEVEL (-1) -#define SF_MAX_PUT_COMPRESS_LEVEL 9 -#define SF_DEFAULT_PUT_MAX_RETRIES 5 -#define SF_MAX_PUT_MAX_RETRIES 100 -#define SF_DEFAULT_GET_MAX_RETRIES 5 -#define SF_MAX_GET_MAX_RETRIES 100 -#define SF_DEFAULT_GET_THRESHOLD 5 - #define SESSION_URL "/session/v1/login-request" #define QUERY_URL "/queries/v1/query-request" #define RENEW_SESSION_URL "/session/token-request" #define DELETE_SESSION_URL "/session" -#define QUERY_RESULT_URL_FORMAT "/queries/%s/result" // not used for now but add for URL checking on connection requests #define AUTHENTICATOR_URL "/session/authenticator-request" @@ -95,7 +85,6 @@ typedef struct SF_STAGE_CRED { char *aws_secret_key; char *aws_token; char *azure_sas_token; - char* gcs_access_token; } SF_STAGE_CRED; typedef struct SF_STAGE_INFO { @@ -150,7 +139,7 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); /** * Executes a statement. * @param sfstmt SNOWFLAKE_STMT context. - * @param is_put_get_command type true if this is a put/get command + * @param sf_use_application_json_accept type true if this is a put/get command * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, * @param is_describe_only should the statement be executed in describe only mode * if the query response is to be captured. @@ -158,8 +147,7 @@ SF_PUT_GET_RESPONSE *STDCALL sf_put_get_response_allocate(); * @return 0 if success, otherwise an errno is returned. */ SF_STATUS STDCALL _snowflake_execute_ex(SF_STMT *sfstmt, - sf_bool is_put_get_command, - sf_bool is_native_put_get, + sf_bool use_application_json_accept_type, struct SF_QUERY_RESULT_CAPTURE* result_capture, sf_bool is_describe_only); @@ -173,32 +161,4 @@ sf_bool STDCALL _is_put_get_command(char* sql_text); */ PARAM_TYPE STDCALL _snowflake_get_param_style(const SF_BIND_INPUT *input); -#ifdef __cplusplus -extern "C" { -#endif -/** - * Legacy approach of Executing a query, not to execute put/get natively. - * Should only be called internally for put/get queries. - * - * @param sf SNOWFLAKE_STMT context. - * @param command a query or command that returns results. - * @return 0 if success, otherwise an errno is returned. - */ -SF_STATUS STDCALL -_snowflake_query_put_get_legacy(SF_STMT* sfstmt, const char* command, size_t command_size); - -/** - * Executes put get command natively. - * @param sfstmt SNOWFLAKE_STMT context. - * @param raw_response_buffer optional pointer to an SF_QUERY_RESULT_CAPTURE, - * - * @return 0 if success, otherwise an errno is returned. - */ -SF_STATUS STDCALL _snowflake_execute_put_get_native( - SF_STMT *sfstmt, - struct SF_QUERY_RESULT_CAPTURE* result_capture); -#ifdef __cplusplus -} // extern "C" -#endif - #endif //SNOWFLAKE_CLIENT_INT_H diff --git a/lib/connection.c b/lib/connection.c index cf1916cbdf..94661ac6d9 100644 --- a/lib/connection.c +++ b/lib/connection.c @@ -201,12 +201,7 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, return body; } -cJSON *STDCALL create_query_json_body(const char *sql_text, - int64 sequence_id, - const char *request_id, - sf_bool is_describe_only, - int64 multi_stmt_count) -{ +cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only) { cJSON *body; double submission_time; // Create body @@ -226,26 +221,11 @@ cJSON *STDCALL create_query_json_body(const char *sql_text, snowflake_cJSON_AddStringToObject(body, "requestId", request_id); } - cJSON* parameters = NULL; - if (multi_stmt_count >= 0) - { - parameters = snowflake_cJSON_CreateObject(); - snowflake_cJSON_AddNumberToObject(parameters, "MULTI_STATEMENT_COUNT", (double)multi_stmt_count); - } - #ifdef SF_WIN32 - if (!parameters) - { - parameters = snowflake_cJSON_CreateObject(); - } + cJSON * parameters = snowflake_cJSON_CreateObject(); snowflake_cJSON_AddStringToObject(parameters, "C_API_QUERY_RESULT_FORMAT", "JSON"); - // temporary code to fake as ODBC to have multiple statements enabled - snowflake_cJSON_AddStringToObject(parameters, "ODBC_QUERY_RESULT_FORMAT", "JSON"); + snowflake_cJSON_AddItemToObject(body, "parameters", parameters); #endif - if (parameters) - { - snowflake_cJSON_AddItemToObject(body, "parameters", parameters); - } return body; } @@ -375,7 +355,7 @@ sf_bool STDCALL curl_post_call(SF_CONNECT *sf, } do { - if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, NULL, json, NULL, NULL, + if (!http_perform(curl, POST_REQUEST_TYPE, url, header, body, json, NULL, retry_timeout, SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -502,7 +482,7 @@ sf_bool STDCALL curl_get_call(SF_CONNECT *sf, memset(query_code, 0, QUERYCODE_LEN); do { - if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, NULL, json, NULL, NULL, + if (!http_perform(curl, GET_REQUEST_TYPE, url, header, NULL, json, NULL, get_retry_timeout(sf), SF_BOOLEAN_FALSE, error, sf->insecure_mode, sf->ocsp_fail_open, sf->retry_on_curle_couldnt_connect_count, @@ -905,16 +885,16 @@ ARRAY_LIST *json_get_object_keys(const cJSON *item) { } size_t -char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf) { +json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json) { size_t data_size = size * nmemb; log_debug("Curl response size: %zu", data_size); - raw_buf->buffer = (char *) SF_REALLOC(raw_buf->buffer, - raw_buf->size + data_size + 1); + raw_json->buffer = (char *) SF_REALLOC(raw_json->buffer, + raw_json->size + data_size + 1); // Start copying where last null terminator existed - sf_memcpy(&raw_buf->buffer[raw_buf->size], data_size, data, data_size); - raw_buf->size += data_size; - // Set null raw_buf - raw_buf->buffer[raw_buf->size] = '\0'; + sf_memcpy(&raw_json->buffer[raw_json->size], data_size, data, data_size); + raw_json->size += data_size; + // Set null terminator + raw_json->buffer[raw_json->size] = '\0'; return data_size; } diff --git a/lib/connection.h b/lib/connection.h index 04ef521ee8..fe605526d0 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -56,9 +56,6 @@ typedef enum SF_REQUEST_TYPE { /** we are doing a http delete */ DELETE_REQUEST_TYPE, - - /** we are doing a http head */ - HEAD_REQUEST_TYPE, } SF_REQUEST_TYPE; /** @@ -84,12 +81,12 @@ typedef enum SF_JSON_ERROR { /** * Dynamically growing char buffer to hold retrieved in cURL call. */ -typedef struct RAW_CHAR_BUFFER { +typedef struct RAW_JSON_BUFFER { // Char buffer char *buffer; // Number of characters in char buffer size_t size; -} RAW_CHAR_BUFFER; +} RAW_JSON_BUFFER; /** * URL Parameter struct used to construct an encoded URL. @@ -170,15 +167,6 @@ typedef struct non_json_response { void * buffer; } NON_JSON_RESP; -/** -* payload struct for put request -*/ -typedef struct put_payload { - size_t (*read_callback)(void* ptr, size_t size, size_t nmemb, void* userdata); - void * buffer; - size_t length; -} PUT_PAYLOAD; - /** * Macro to get a custom error message to pass to the Snowflake Error object. */ @@ -213,14 +201,9 @@ cJSON *STDCALL create_auth_json_body(SF_CONNECT *sf, const char *application, co * @param sequence_id Sequence ID from the Snowflake Connection object. * @param request_id requestId to be passed as a part of body instead of header. * @param is_describe_only is the query describe only. - * @param multi_stmt_count The value of MULTI_STATEMENT_COUNT set with the query. No setting if < 0. * @return Query cJSON Body. */ -cJSON *STDCALL create_query_json_body(const char *sql_text, - int64 sequence_id, - const char *request_id, - sf_bool is_describe_only, - int64 multi_stmt_count); +cJSON *STDCALL create_query_json_body(const char *sql_text, int64 sequence_id, const char *request_id, sf_bool is_describe_only); /** * Creates a cJSON blob that is used to renew a session with Snowflake. cJSON blob must be freed by the caller using @@ -410,16 +393,16 @@ SF_JSON_ERROR STDCALL json_detach_object_from_array(cJSON **dest, cJSON *data, i ARRAY_LIST *json_get_object_keys(const cJSON *item); /** - * A write callback function to use to write the response text received from the cURL response. The raw CHAR buffer + * A write callback function to use to write the response text received from the cURL response. The raw JSON buffer * will grow in size until * * @param data The data to copy in the buffer. * @param size The size (in bytes) of each data member. * @param nmemb The number of data members. - * @param raw_buf The Raw CHAR Buffer object that grows in size to copy multiple writes for a single cURL call. + * @param raw_json The Raw JSON Buffer object that grows in size to copy multiple writes for a single cURL call. * @return The number of bytes copied into the buffer. */ -size_t char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_buf); +size_t json_resp_cb(char *data, size_t size, size_t nmemb, RAW_JSON_BUFFER *raw_json); /** * Performs an HTTP request with retry. @@ -428,13 +411,10 @@ size_t char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_ * @param request_type The type of HTTP request. * @param url The fully qualified URL to use for the HTTP request. * @param header The header to use for the HTTP request. - * @param body The body to send over the HTTP request. If running GET/PUT request, set this to NULL. - * @param put_payload The payload to send over the PUT HTTP request. If not running PUT request, set this to NULL. + * @param body The body to send over the HTTP request. If running GET request, set this to NULL. * @param json A reference to a cJSON pointer where we should store a successful request. * @param non_json_resp A reference to a non-json response to retrieve response in non-json format. * Used only when json is set to NULL. - * @param resp_headers A reference to retrieve response headers. Needs to be freed with SF_FREE. - * Set to NULL if it's not needed. * @param network_timeout The network request timeout to use for each request try. * @param chunk_downloader A boolean value determining whether or not we are running this request from the chunk * downloader. Each chunk that we download from AWS is invalid JSON so we need to add an @@ -465,9 +445,8 @@ size_t char_resp_cb(char *data, size_t size, size_t nmemb, RAW_CHAR_BUFFER *raw_ * @return Success/failure status of http request call. 1 = Success; 0 = Failure/renew timeout */ sf_bool STDCALL http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP* non_json_resp, - char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, - SF_ERROR_STRUCT* error, sf_bool insecure_mode, sf_bool fail_open, + char *body, cJSON **json, NON_JSON_RESP* non_json_resp, int64 network_timeout, sf_bool chunk_downloader, + SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open, int8 retry_on_curle_couldnt_connect_count, int64 renew_timeout, int8 retry_max_count, int64 *elapsed_time, int8 *retried_count, diff --git a/lib/http_perform.c b/lib/http_perform.c index 56d093967d..b7291e2034 100644 --- a/lib/http_perform.c +++ b/lib/http_perform.c @@ -143,10 +143,8 @@ sf_bool STDCALL http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, - PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, - char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, @@ -192,8 +190,7 @@ sf_bool STDCALL http_perform(CURL *curl, sf_get_current_time_millis() // start time }; time_t elapsedRetryTime = time(NULL); - RAW_CHAR_BUFFER buffer = {NULL, 0}; - RAW_CHAR_BUFFER headerBuffer = { NULL, 0 }; + RAW_JSON_BUFFER buffer = {NULL, 0}; struct data config; config.trace_ascii = 1; @@ -207,8 +204,6 @@ sf_bool STDCALL http_perform(CURL *curl, // Reset buffer since this may not be our first rodeo SF_FREE(buffer.buffer); buffer.size = 0; - SF_FREE(headerBuffer.buffer); - headerBuffer.size = 0; // Generate new request guid, if request guid exists in url if (SF_BOOLEAN_TRUE != retry_ctx_update_url(&curl_retry_ctx, url, include_retry_reason)) { @@ -274,46 +269,6 @@ sf_bool STDCALL http_perform(CURL *curl, break; } } - else if (request_type == HEAD_REQUEST_TYPE) - { - /** we want response header only */ - curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); - } - else if (request_type == PUT_REQUEST_TYPE) - { - // we need to upload the data - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - - if (!put_payload) - { - log_error("Invalid payload for put request"); - break; - } - /** set read callback function */ - res = curl_easy_setopt(curl, CURLOPT_READFUNCTION, put_payload->read_callback); - if (res != CURLE_OK) { - log_error("Failed to set read function [%s]", curl_easy_strerror(res)); - break; - } - - /** set data object to pass to callback function */ - res = curl_easy_setopt(curl, CURLOPT_READDATA, put_payload->buffer); - if (res != CURLE_OK) { - log_error("Failed to set read data [%s]", curl_easy_strerror(res)); - break; - } - - /** set size of put */ - if (put_payload->length <= SF_INT32_MAX) - { - res = curl_easy_setopt(curl, CURLOPT_INFILESIZE, (long)put_payload->length); - } - else - { - res = curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)put_payload->length); - } - } - if (!json && non_json_resp) { @@ -321,7 +276,7 @@ sf_bool STDCALL http_perform(CURL *curl, } else { - res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&char_resp_cb); + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (void*)&json_resp_cb); } if (res != CURLE_OK) { log_error("Failed to set writer [%s]", curl_easy_strerror(res)); @@ -337,20 +292,8 @@ sf_bool STDCALL http_perform(CURL *curl, res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buffer); } if (res != CURLE_OK) { - log_error("Failed to set write data [%s]", curl_easy_strerror(res)); - break; - } - - res = curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)&headerBuffer); - if (res != CURLE_OK) { - log_error("Failed to set header data [%s]", curl_easy_strerror(res)); - break; - } - - res = curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (void*)&char_resp_cb); - if (res != CURLE_OK) { - log_error("Failed to set header function [%s]", curl_easy_strerror(res)); - break; + log_error("Failed to set write data [%s]", curl_easy_strerror(res)); + break; } if (DISABLE_VERIFY_PEER) { @@ -558,15 +501,6 @@ sf_bool STDCALL http_perform(CURL *curl, SF_FREE(buffer.buffer); - if (resp_headers) - { - *resp_headers = headerBuffer.buffer; - } - else - { - SF_FREE(headerBuffer.buffer); - } - return ret; } @@ -577,10 +511,8 @@ sf_bool STDCALL __wrap_http_perform(CURL *curl, char *url, SF_HEADER *header, char *body, - PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, - char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, diff --git a/lib/mock_http_perform.h b/lib/mock_http_perform.h index 71ea0c8b79..ce6cad20c3 100644 --- a/lib/mock_http_perform.h +++ b/lib/mock_http_perform.h @@ -17,7 +17,7 @@ extern "C" { // The parameters for this are identical to http_perform located in connection.h // This is just the mock interface sf_bool STDCALL __wrap_http_perform(CURL *curl, SF_REQUEST_TYPE request_type, char *url, SF_HEADER *header, - char *body, PUT_PAYLOAD* put_payload, cJSON **json, NON_JSON_RESP *non_json_resp, char** resp_headers, int64 network_timeout, sf_bool chunk_downloader, + char *body, cJSON **json, NON_JSON_RESP *non_json_resp, int64 network_timeout, sf_bool chunk_downloader, SF_ERROR_STRUCT *error, sf_bool insecure_mode, sf_bool fail_open); #endif diff --git a/lib/result_set.h b/lib/result_set.h index c5faac33ff..865d8eba69 100644 --- a/lib/result_set.h +++ b/lib/result_set.h @@ -6,12 +6,26 @@ #define SNOWFLAKE_RESULTSET_H #include "snowflake/client.h" -#include "connection.h" +#include "result_set_arrow.h" +#include "result_set_json.h" #ifdef __cplusplus extern "C" { #endif + // Utility ===================================================================================== + + /** + * An enumeration over all supported query result formats. + * This is used to help deciding which result set to create. + * + * NOTE: Keep the order consistent with rowset_key_map! + */ + typedef enum QueryResultFormat + { + ARROW_FORMAT, JSON_FORMAT, FORMAT_MAX + } QueryResultFormat_t; + // Result Set API ============================================================================== /** @@ -25,10 +39,10 @@ extern "C" { * * @return the created result set. */ - result_set_ptr rs_create_with_json_result( + void * rs_create_with_json_result( cJSON * json_rowset, SF_COLUMN_DESC * metadata, - QueryResultFormat query_result_format, + QueryResultFormat_t * query_result_format, const char * tz_string); /** @@ -42,50 +56,55 @@ extern "C" { * * @return the created result set. */ - result_set_ptr rs_create_with_chunk( + void * rs_create_with_chunk( void * initial_chunk, SF_COLUMN_DESC * metadata, - QueryResultFormat query_result_format, + QueryResultFormat_t * query_result_format, const char * tz_string); /** * Destructor. * * @param rs The ResultSet object. + * @param query_result_format The query result format. */ - void rs_destroy(result_set_ptr rs); + void rs_destroy(void * rs, QueryResultFormat_t * query_result_format); /** * Appends the given chunk to the internal result set. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param chunk The chunk to append. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL - rs_append_chunk(result_set_ptr rs, void * chunk); + rs_append_chunk(void * rs, QueryResultFormat_t * query_result_format, void * chunk); /** * Advances to the next row. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * * @return 0 if successful, otherwise an error is returned. */ - SF_STATUS STDCALL rs_next(result_set_ptr rs); + SF_STATUS STDCALL rs_next(void * rs, QueryResultFormat_t * query_result_format); /** * Writes the value of the current cell as a boolean to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_bool( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, sf_bool * out_data); @@ -93,13 +112,15 @@ extern "C" { * Writes the value of the current cell as an int8 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int8( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int8 * out_data); @@ -107,13 +128,15 @@ extern "C" { * Writes the value of the current cell as an int32 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int32 * out_data); @@ -121,13 +144,15 @@ extern "C" { * Writes the value of the current cell as an int64 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_int64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, int64 * out_data); @@ -135,13 +160,15 @@ extern "C" { * Writes the value of the current cell as a uint8 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint8( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint8 * out_data); @@ -149,13 +176,15 @@ extern "C" { * Writes the value of the current cell as a uint32 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint32 * out_data); @@ -163,13 +192,15 @@ extern "C" { * Writes the value of the current cell as a uint64 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_uint64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, uint64 * out_data); @@ -177,13 +208,15 @@ extern "C" { * Writes the value of the current cell as a float32 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float32( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, float32 * out_data); @@ -191,13 +224,15 @@ extern "C" { * Writes the value of the current cell as a float64 to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_float64( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, float64 * out_data); @@ -205,13 +240,15 @@ extern "C" { * Writes the value of the current cell as a constant C-string to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_const_string( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, const char ** out_data); @@ -219,13 +256,15 @@ extern "C" { * Writes the value of the current cell as a timestamp to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_as_timestamp( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, SF_TIMESTAMP * out_data); @@ -233,13 +272,15 @@ extern "C" { * Writes the length of the current cell to the provided buffer. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_get_cell_strlen( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, size_t * out_data); @@ -247,22 +288,25 @@ extern "C" { * Gets the number of rows in the current chunk being processed. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * * @return the number of rows in the current chunk. */ - size_t rs_get_row_count_in_chunk(result_set_ptr rs); + size_t rs_get_row_count_in_chunk(void * rs, QueryResultFormat_t * query_result_format); /** * Indiciates whether the current cell is null or not. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * @param idx The index of the column or row to retrieve. * @param out_data The buffer to write to. * * @return 0 if successful, otherwise an error is returned. */ SF_STATUS STDCALL rs_is_cell_null( - result_set_ptr rs, + void * rs, + QueryResultFormat_t * query_result_format, size_t idx, sf_bool * out_data); @@ -270,22 +314,27 @@ extern "C" { * Get the latest error code. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * * @return the latest error code. 0 if no error. */ - SF_STATUS STDCALL rs_get_error(result_set_ptr rs); + SF_STATUS STDCALL rs_get_error( + void * rs, + QueryResultFormat_t * query_result_format + ); /** * Get the latest error code. * * @param rs The ResultSet object. + * @param query_result_format The query result format. * * @return the latest error message. empty string if no error. */ - const char* rs_get_error_message(result_set_ptr rs); - - // return callback struct for arrow chunk downloading - NON_JSON_RESP* callback_create_arrow_resp(void); + const char* rs_get_error_message( + void * rs, + QueryResultFormat_t * query_result_format + ); #ifdef __cplusplus } // extern "C" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 35d42ed177..b4a8fe9fc4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -44,7 +44,6 @@ SET(TESTS_C test_unit_oauth test_unit_mfa_auth test_ocsp_fail_open - test_multiple_statements # FEATURE_INCREASED_MAX_LOB_SIZE_IN_MEMORY is internal switch # will enable lob test when the change on server side will be published # test_lob @@ -224,7 +223,7 @@ if (LINUX) endif () message("valgrind suppression file is located at " ${VALGRIND_SUPPRESSION}) if (LINUX) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} -Wl,--whole-archive telemetry curl ssl crypto uuid -Wl,--no-whole-archive pthread -Wl,--as-needed -static-libgcc -static-libstdc++) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} -Wl,--whole-archive telemetry curl ssl crypto uuid pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++) @@ -238,14 +237,12 @@ endif() if (WIN32) if (WIN32_DEBUG) if(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} - Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ucrtd.lib) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${AWS_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} ${AZURE_STORAGE_LITE_LIB} Version.lib Userenv.lib Bcrypt.lib ucrtd.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() else() - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} - Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB}) set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${AWS_ALL_LIBS} ${AZURE_STORAGE_LITE_LIB} ${ARROW_ALL_LIBS} ${CURL_LIB} ${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} ${ZLIB_LIB} Version.lib Userenv.lib Bcrypt.lib Secur32.lib Ncrypt.lib Shlwapi.lib) endif() diff --git a/tests/test_multiple_statements.c b/tests/test_multiple_statements.c deleted file mode 100644 index f4a2cf37e3..0000000000 --- a/tests/test_multiple_statements.c +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright (c) 2024 Snowflake Computing, Inc. All rights reserved. - */ -#include -#include "utils/test_setup.h" - -void test_multi_stmt_transaction(void **unused) -{ - SF_CONNECT *sf = setup_snowflake_connection(); - SF_STATUS status = snowflake_connect(sf); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sf->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - /* query */ - SF_STMT *sfstmt = snowflake_stmt(sf); - status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); - assert_int_equal(status, SF_STATUS_SUCCESS); - - int64 multi_stmt_count = 5; - status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); - assert_int_equal(status, SF_STATUS_SUCCESS); - - status = snowflake_query(sfstmt, - "begin;\n" - "delete from test_multi_txn;\n" - "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" - "commit;\n" - "select count(*) from test_multi_txn", - 0); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sfstmt->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - // first statement (begin) - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // second statement (delete) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // third statement (insert) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 2); - - // fourth statement (commit) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // fifth statement (select) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - - int counter = 0; - int64 out; - while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { - snowflake_column_as_int64(sfstmt, 1, &out); - assert_int_equal(out, 2); - ++counter; - } - assert_int_equal(status, SF_STATUS_EOF); - assert_int_equal(counter, 1); - - // no more result - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); - - snowflake_stmt_term(sfstmt); - snowflake_term(sf); -} - -void test_multi_stmt_transaction_rollback(void **unused) -{ - SF_CONNECT *sf = setup_snowflake_connection(); - SF_STATUS status = snowflake_connect(sf); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sf->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - /* query */ - SF_STMT *sfstmt = snowflake_stmt(sf); - status = snowflake_query(sfstmt, "create or replace temporary table test_multi_txn(c1 number, c2 string) as select 10, 'z'", 0); - assert_int_equal(status, SF_STATUS_SUCCESS); - - int64 multi_stmt_count = 5; - status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); - assert_int_equal(status, SF_STATUS_SUCCESS); - - status = snowflake_query(sfstmt, - "begin;\n" - "delete from test_multi_txn;\n" - "insert into test_multi_txn values (1, 'a'), (2, 'b');\n" - "rollback;\n" - "select count(*) from test_multi_txn", - 0); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sfstmt->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - // first statement (begin) - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // second statement (delete) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // third statement (insert) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 2); - - // fourth statement (rollback) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // fifth statement (select) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - - int counter = 0; - int64 out; - while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { - snowflake_column_as_int64(sfstmt, 1, &out); - assert_int_equal(out, 1); - ++counter; - } - assert_int_equal(status, SF_STATUS_EOF); - assert_int_equal(counter, 1); - - // no more result - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); - - snowflake_stmt_term(sfstmt); - snowflake_term(sf); -} - -void test_multi_stmt_with_large_result(void **unused) -{ - const int rownum = 100000; - SF_CONNECT *sf = setup_snowflake_connection(); - - // TODO SNOW-1526335 - // Sometime we can't get OCSP response from cache server or responder - // Usually happen on GCP and should be ignored by FAIL_OPEN - // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now - // so we have to disable OCSP validation to around it. - // Will remove this code when adding support for FAIL_OPEN (which is - // the default behavior for all other drivers) - char *cenv = getenv("CLOUD_PROVIDER"); - if (cenv && !strncmp(cenv, "GCP", 4)) { - sf_bool insecure_mode = SF_BOOLEAN_TRUE; - snowflake_set_attribute(sf, SF_CON_INSECURE_MODE, &insecure_mode); - } - - SF_STATUS status = snowflake_connect(sf); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sf->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - /* query */ - SF_STMT *sfstmt = snowflake_stmt(sf); - int64 multi_stmt_count = 3; - status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); - assert_int_equal(status, SF_STATUS_SUCCESS); - - status = snowflake_query(sfstmt, - "create or replace temporary table test_multi_large(c1 number, c2 number);\n" - "insert into test_multi_large select seq4(), TO_VARCHAR(seq4()) from table(generator(rowcount => 100000));\n" - "select * from test_multi_large", - 0); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sfstmt->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - // first statement (begin) - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), 1); - - // second statement (insert) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - assert_int_equal(snowflake_affected_rows(sfstmt), rownum); - - // third statement (select) - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_SUCCESS); - assert_int_equal(snowflake_num_rows(sfstmt), rownum); - - int counter = 0; - int64 intout; - const char* strout; - char strexp[64]; - while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) { - snowflake_column_as_int64(sfstmt, 1, &intout); - assert_int_equal(intout, counter); - snowflake_column_as_const_str(sfstmt, 2, &strout); - sprintf(strexp, "%d", counter); - assert_string_equal(strout, strexp); - ++counter; - } - assert_int_equal(status, SF_STATUS_EOF); - assert_int_equal(counter, rownum); - - // no more result - assert_int_equal(snowflake_next_result(sfstmt), SF_STATUS_EOF); - - snowflake_stmt_term(sfstmt); - snowflake_term(sf); -} - -/* helper function for testing multi_stmt_count, running a query with - * multiple statements of 3. - * @param use_session_param Whethter to set MULTI_STATEMENT_COUNT through - * session parameter or statement attribute. - * @param count The count number to be set - * @return True if the query succeeded, otherwise false. - */ -sf_bool test_multi_stmt_core(sf_bool use_session_param, int count) -{ - SF_CONNECT *sf = setup_snowflake_connection(); - SF_STATUS status = snowflake_connect(sf); - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sf->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - SF_STMT* sfstmt = snowflake_stmt(sf); - - char query[1024]; - int64 multi_stmt_count = count; - if (SF_BOOLEAN_TRUE == use_session_param) - { - sprintf(query, "alter session set MULTI_STATEMENT_COUNT=%d", count); - status = snowflake_query(sfstmt, query, 0); - } - else - { - status = snowflake_stmt_set_attr(sfstmt, SF_STMT_MULTI_STMT_COUNT, &multi_stmt_count); - } - if (status != SF_STATUS_SUCCESS) { - dump_error(&(sfstmt->error)); - } - assert_int_equal(status, SF_STATUS_SUCCESS); - - sprintf(query, "%s", "select 1; select 2; select 3"); - status = snowflake_query(sfstmt, query, 0); - - snowflake_stmt_term(sfstmt); - snowflake_term(sf); - - return (status == SF_STATUS_SUCCESS) ? SF_BOOLEAN_TRUE : SF_BOOLEAN_FALSE; -} - -void test_multi_stmt_count_session_param_off(void** unused) -{ - // disable multiple statements by setting session parameter to 1 - // the query is expected to fail - assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 1), SF_BOOLEAN_FALSE); -} - -void test_multi_stmt_count_session_param_on(void** unused) -{ - // enable multiple statements by setting session parameter to 0 - // the query should work - assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_TRUE, 0), SF_BOOLEAN_TRUE); -} - -void test_multi_stmt_count_stmt_attr_match(void** unused) -{ - // set statement attribute with match number - // the query should work - assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 3), SF_BOOLEAN_TRUE); -} - -void test_multi_stmt_count_stmt_attr_mismatch(void** unused) -{ - // set statement attribute with mismatch number - // the query is expected to fail - assert_int_equal(test_multi_stmt_core(SF_BOOLEAN_FALSE, 2), SF_BOOLEAN_FALSE); -} - -int main(void) { - initialize_test(SF_BOOLEAN_FALSE); - const struct CMUnitTest tests[] = { - cmocka_unit_test(test_multi_stmt_transaction), - cmocka_unit_test(test_multi_stmt_transaction_rollback), - cmocka_unit_test(test_multi_stmt_with_large_result), - cmocka_unit_test(test_multi_stmt_count_session_param_off), - cmocka_unit_test(test_multi_stmt_count_session_param_on), - cmocka_unit_test(test_multi_stmt_count_stmt_attr_match), - cmocka_unit_test(test_multi_stmt_count_stmt_attr_mismatch), - }; - int ret = cmocka_run_group_tests(tests, NULL, NULL); - snowflake_global_term(); - return ret; -} diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index 4096263221..e592371e46 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -118,8 +118,7 @@ void test_simple_put_core(const char * fileName, int compressLevel = -1, bool overwrite = false, SF_CONNECT * connection = nullptr, - bool testUnicode = false, - bool native = false) + bool testUnicode = false) { /* init */ SF_STATUS status; @@ -195,140 +194,71 @@ void test_simple_put_core(const char * fileName, putCommand += " overwrite=true"; } std::unique_ptr stmtPutGet; - if (!native) + if (testUnicode) { - if (testUnicode) - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); - } + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (tmpDir) { - if (native) - { - snowflake_set_attribute(sf, SF_CON_PUT_TEMPDIR, tmpDir); - } - else - { transConfig.tempDir = tmpDir; transConfigPtr = &transConfig; - } } if(useS3regionalUrl) { - if (native) - { - std::string cmd = "alter session set ENABLE_STAGE_S3_PRIVATELINK_FOR_US_EAST_1=true"; - snowflake_query(sfstmt, cmd.c_str(), cmd.size()); - } - else - { - transConfig.useS3regionalUrl = true; - transConfigPtr = &transConfig; - } + transConfig.useS3regionalUrl = true; + transConfigPtr = &transConfig; } if(compressLevel > 0) { - if (native) - { - int8 lv = (int8)compressLevel; - snowflake_set_attribute(sf, SF_CON_PUT_COMPRESSLV, &lv); - } - else - { - transConfig.compressLevel = compressLevel; - transConfigPtr = &transConfig; - } + transConfig.compressLevel = compressLevel; + transConfigPtr = &transConfig; } Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); if(useDevUrand){ - if (native) - { - sf_bool use_urand = SF_BOOLEAN_TRUE; - snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &use_urand); - } - else - { - agent.setRandomDeviceAsUrand(true); - } + agent.setRandomDeviceAsUrand(true); } - std::string expectedSrc = sf_filename_from_path(fileName); - std::string expectedTarget = (autoCompress && !strstr(expectedSrc.c_str(), ".gz")) ? - expectedSrc + ".gz" : expectedSrc; - std::string expectedSourceCompression = !strstr(fileName, ".gz") ? "none" : "gzip"; - std::string expectedTargetCompression = (!autoCompress && !strstr(fileName, ".gz")) ? "none" : "gzip"; - - if (native) - { - ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); - assert_int_equal(SF_STATUS_SUCCESS, ret); - - assert_int_equal(snowflake_num_rows(sfstmt), 1); - - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_SUCCESS, ret); + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); - const char *out; - // source - snowflake_column_as_const_str(sfstmt, 1, &out); - assert_string_equal(expectedSrc.c_str(), out); - // target - snowflake_column_as_const_str(sfstmt, 2, &out); - assert_string_equal(expectedTarget.c_str(), out); - // source comparession - snowflake_column_as_const_str(sfstmt, 5, &out); - assert_string_equal(expectedSourceCompression.c_str(), out); - // target compression - snowflake_column_as_const_str(sfstmt, 6, &out); - assert_string_equal(expectedTargetCompression.c_str(), out); - snowflake_column_as_const_str(sfstmt, 7, &out); - // status - assert_string_equal("UPLOADED", out); - // encryption - snowflake_column_as_const_str(sfstmt, 8, &out); - assert_string_equal("ENCRYPTED", out); - - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_EOF, ret); - } - else + while(results->next()) { - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); - - while(results->next()) - { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal(expectedSrc.c_str(), value.c_str()); - - results->getColumnAsString(1, value); // get target - assert_string_equal(expectedTarget.c_str(), value.c_str()); - - results->getColumnAsString(4, value); // get source_compression - assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); - - results->getColumnAsString(5, value); // get target_compression - assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); - - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); - - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); - } + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal( sf_filename_from_path(fileName), value.c_str()); + + std::string expectedTarget = (autoCompress && !strstr(fileName, ".gz")) ? + std::string(fileName) + ".gz" : + std::string(fileName); + results->getColumnAsString(1, value); // get target + assert_string_equal(sf_filename_from_path(expectedTarget.c_str()), value.c_str()); + + std::string expectedSourceCompression = !strstr(fileName, ".gz") ? + "none" : "gzip"; + results->getColumnAsString(4, value); // get source_compression + assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); + + std::string expectedTargetCompression = (!autoCompress && + !strstr(fileName, ".gz")) ? "none" : "gzip"; + results->getColumnAsString(5, value); // get target_compression + assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); + + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); + + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); } if (copyUploadFile) @@ -422,8 +352,7 @@ static int teardown(void **unused) } void test_simple_get_data(const char *getCommand, const char *size, - long getThreshold = 0, bool testUnicode = false, - bool native = false) + long getThreshold = 0, bool testUnicode = false) { /* init */ SF_STATUS status; @@ -438,72 +367,39 @@ void test_simple_get_data(const char *getCommand, const char *size, sfstmt = snowflake_stmt(sf); std::unique_ptr stmtPutGet; - if (!native) + if (testUnicode) { - if (testUnicode) - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); - } - else - { - stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); - } + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGetUnicode(sfstmt)); + } + else + { + stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); } TransferConfig transConfig; TransferConfig * transConfigPtr = nullptr; if (getThreshold > 0) { - if (native) - { - int64 threshold = getThreshold; - status = snowflake_set_attribute(sf, SF_CON_GET_THRESHOLD, &threshold); - assert_int_equal(SF_STATUS_SUCCESS, status); - } - else - { transConfig.getSizeThreshold = getThreshold; transConfigPtr = &transConfig; - } } - if (native) - { - status = snowflake_query(sfstmt, getCommand, strlen(getCommand)); - assert_int_equal(SF_STATUS_SUCCESS, status); + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); - while ((status = snowflake_fetch(sfstmt)) == SF_STATUS_SUCCESS) - { - const char *out; - // source - snowflake_column_as_const_str(sfstmt, 2, &out); - //Compressed File sizes vary on Windows/Linux, So not verifying size. - snowflake_column_as_const_str(sfstmt, 3, &out); - assert_string_equal("DOWNLOADED", out); - snowflake_column_as_const_str(sfstmt, 4, &out); - assert_string_equal("DECRYPTED", out); - } - assert_int_equal(SF_STATUS_EOF, status); - } - else + // load first time should return uploaded + std::string get_status; + std::string getcmd(getCommand); + ITransferResult * results = agent.execute(&getcmd); + while(results && results->next()) { - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get(), transConfigPtr); - - // load first time should return uploaded - std::string get_status; - std::string getcmd(getCommand); - ITransferResult * results = agent.execute(&getcmd); - while(results && results->next()) - { - results->getColumnAsString(1, get_status); - //Compressed File sizes vary on Windows/Linux, So not verifying size. - results->getColumnAsString(2, get_status); - assert_string_equal("DOWNLOADED", get_status.c_str()); - results->getColumnAsString(3, get_status); - assert_string_equal("DECRYPTED", get_status.c_str()); - } + results->getColumnAsString(1, get_status); + //Compressed File sizes vary on Windows/Linux, So not verifying size. + results->getColumnAsString(2, get_status); + assert_string_equal("DOWNLOADED", get_status.c_str()); + results->getColumnAsString(3, get_status); + assert_string_equal("DECRYPTED", get_status.c_str()); } snowflake_stmt_term(sfstmt); @@ -513,7 +409,7 @@ void test_simple_get_data(const char *getCommand, const char *size, } -void test_large_put_auto_compress_core(bool native) +void test_large_put_auto_compress(void **unused) { char *cenv = getenv("CLOUD_PROVIDER"); if ( cenv && !strncmp(cenv, "AWS", 4) ) { @@ -527,32 +423,10 @@ void test_large_put_auto_compress_core(bool native) false, // auto compress true, // Load data into table false, // Run select * on loaded table (Not good for large data set) - true, // copy data from Table to Staging. - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - nullptr, // tmpDir - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native + true // copy data from Table to Staging. ); } -void test_large_put_auto_compress(void **unused) -{ - test_large_put_auto_compress_core(false); -} - -void test_large_put_auto_compress_native(void **unused) -{ - test_large_put_auto_compress_core(true); -} - void test_large_put_threshold(void **unused) { char *cenv = getenv("CLOUD_PROVIDER"); @@ -574,10 +448,10 @@ void test_large_put_threshold(void **unused) ); } -void test_large_reupload_core(bool native) +void test_large_reupload(void **unused) { - char *cenv = getenv("CLOUD_PROVIDER"); - if (cenv && !strncmp(cenv, "AWS", 4)) { + char *cenv = getenv("CLOUD_PROVIDER"); + if (cenv && !strncmp(cenv, "AWS", 4)) { errno = 0; return; } @@ -602,7 +476,7 @@ void test_large_reupload_core(bool native) char tempFile[MAX_BUF_SIZE] ={0}; sf_get_tmp_dir(tempDir); #ifdef _WIN32 - getLongTempPath(tempDir); + getLongTempPath(tempDir); #endif for(const std::string s : fileList) { tempFile[0] = 0; @@ -613,30 +487,10 @@ void test_large_reupload_core(bool native) true, // Load data into table false, // Run select * on loaded table (Not good for large data set) false, // copy data from Table to Staging. - true, //Creates a dup table to compare uploaded data. - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - nullptr, // tmpDir - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native + true //Creates a dup table to compare uploaded data. ); } -} - -void test_large_reupload(void** unused) -{ - test_large_reupload_core(false); -} -void test_large_reupload_native(void** unused) -{ - test_large_reupload_core(true); } /* @@ -684,7 +538,7 @@ void test_verify_upload(void **unused) } -void test_simple_put_use_dev_urandom_core(bool native) +void test_simple_put_use_dev_urandom(void **unused) { std::string dataDir = TestSetup::getDataDir(); std::string file = dataDir + "medium_file.csv"; @@ -713,67 +567,32 @@ void test_simple_put_use_dev_urandom_core(bool native) std::string putCommand = "put file://" + file + " @%test_small_put auto_compress=true overwrite=true"; - if (native) - { - sf_bool useUrand = SF_BOOLEAN_TRUE; - ret = snowflake_set_attribute(sf, SF_CON_PUT_USE_URANDOM_DEV, &useUrand); - assert_int_equal(SF_STATUS_SUCCESS, ret); - ret = snowflake_query(sfstmt, putCommand.c_str(), putCommand.size()); - assert_int_equal(SF_STATUS_SUCCESS, ret); - assert_int_equal(snowflake_num_rows(sfstmt), 1); - - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_SUCCESS, ret); - - const char* out = NULL; - // source - snowflake_column_as_const_str(sfstmt, 1, &out); - assert_string_equal(sf_filename_from_path(file.c_str()), out); - // target - snowflake_column_as_const_str(sfstmt, 2, &out); - assert_string_equal("medium_file.csv.gz", out); - // source compression - snowflake_column_as_const_str(sfstmt, 5, &out); - assert_string_equal("none", out); - // status - snowflake_column_as_const_str(sfstmt, 7, &out); - assert_string_equal("UPLOADED", out); - // encryption - snowflake_column_as_const_str(sfstmt, 8, &out); - assert_string_equal("ENCRYPTED", out); - - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_EOF, ret); - } - else - { - std::unique_ptr stmtPutGet = std::unique_ptr - (new Snowflake::Client::StatementPutGet(sfstmt)); + std::unique_ptr stmtPutGet = std::unique_ptr + (new Snowflake::Client::StatementPutGet(sfstmt)); - Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); - agent.setRandomDeviceAsUrand(true); + Snowflake::Client::FileTransferAgent agent(stmtPutGet.get()); + agent.setRandomDeviceAsUrand(true); - ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + ITransferResult * results = agent.execute(&putCommand); + assert_int_equal(1, results->getResultSize()); - while (results->next()) - { - std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); + while (results->next()) + { + std::string value; + results->getColumnAsString(0, value); // source + assert_string_equal(sf_filename_from_path(file.c_str()), value.c_str()); - results->getColumnAsString(1, value); // get target - assert_string_equal("medium_file.csv.gz", value.c_str()); + results->getColumnAsString(1, value); // get target + assert_string_equal("medium_file.csv.gz", value.c_str()); - results->getColumnAsString(4, value); // get source_compression - assert_string_equal("none", value.c_str()); + results->getColumnAsString(4, value); // get source_compression + assert_string_equal("none", value.c_str()); - results->getColumnAsString(6, value); // get encryption - assert_string_equal("UPLOADED", value.c_str()); + results->getColumnAsString(6, value); // get encryption + assert_string_equal("UPLOADED", value.c_str()); - results->getColumnAsString(7, value); // get encryption - assert_string_equal("ENCRYPTED", value.c_str()); - } + results->getColumnAsString(7, value); // get encryption + assert_string_equal("ENCRYPTED", value.c_str()); } std::string copyCommand = "copy into test_small_put from @%test_small_put"; @@ -791,56 +610,30 @@ void test_simple_put_use_dev_urandom_core(bool native) for(int i = 0; i < 200000; ++i) { - ret = snowflake_fetch(sfstmt); - assert_int_equal(SF_STATUS_SUCCESS, ret); - snowflake_column_as_const_str(sfstmt, 1, &out_c1); - snowflake_column_as_const_str(sfstmt, 2, &out_c2); - snowflake_column_as_const_str(sfstmt, 3, &out_c3); - std::string c1 = std::to_string(i); - std::string c2 = std::to_string(i + 1); - assert_string_equal(out_c1, c1.c_str()); - assert_string_equal(out_c2, c2.c_str()); - assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - out_c1 = NULL; - out_c2 = NULL; - out_c3 = NULL; + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + snowflake_column_as_const_str(sfstmt, 1, &out_c1); + snowflake_column_as_const_str(sfstmt, 2, &out_c2); + snowflake_column_as_const_str(sfstmt, 3, &out_c3); + std::string c1 = std::to_string(i); + std::string c2 = std::to_string(i + 1); + assert_string_equal(out_c1, c1.c_str()); + assert_string_equal(out_c2, c2.c_str()); + assert_string_equal(out_c3, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + out_c1 = NULL; + out_c2 = NULL; + out_c3 = NULL; } ret = snowflake_fetch(sfstmt); assert_int_equal(SF_STATUS_EOF, ret); } -void test_simple_put_use_dev_urandom(void **unused) -{ - test_simple_put_use_dev_urandom_core(false); -} - -void test_simple_put_use_dev_urandom_native(void **unused) -{ - test_simple_put_use_dev_urandom_core(true); -} - - -void test_simple_put_auto_compress_core(bool native) +void test_simple_put_auto_compress(void **unused) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true, // auto compress - true, // copyUploadFile - true, // verifyCopyUploadFile - false, // copyTableToStaging - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - nullptr, // tmpDir - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native + true // auto compress ); test_simple_put_core("small_file.csv", // filename @@ -857,11 +650,7 @@ void test_simple_put_auto_compress_core(bool native) nullptr, false, 1, - true, // overwrite - nullptr, // connection - false, // testUnicode - native - ); + true); test_simple_put_core("small_file.csv", // filename "auto", //source compression @@ -877,24 +666,10 @@ void test_simple_put_auto_compress_core(bool native) nullptr, false, 9, - true, // overwrite - nullptr, // connection - false, // testUnicode - native - ); -} - -void test_simple_put_auto_compress(void **unused) -{ - test_simple_put_auto_compress_core(false); + true); } -void test_simple_put_auto_compress_native(void **unused) -{ - test_simple_put_auto_compress_core(true); -} - -void test_simple_put_config_temp_dir_core(bool native) +void test_simple_put_config_temp_dir(void **unused) { char tmpDir[MAX_PATH] = {0}; char tmpDirInjection[MAX_PATH] = {0}; @@ -921,13 +696,7 @@ void test_simple_put_config_temp_dir_core(bool native) 64*1024*1024, // customThreshold false, // useDevUrand false, // createSubfolder - tmpDir, - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native + tmpDir ); assert_true(sf_is_directory_exist(tmpDir)); @@ -952,51 +721,31 @@ void test_simple_put_config_temp_dir_core(bool native) sf_delete_directory_if_exists(tmpDir); } - // native execution doesn't throw exception - if (!native) + // try injection the command for folder deletion like + // rm -rf xxx ; mkdir /injection ; xxx + sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", + CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); + try { - // try injection the command for folder deletion like - // rm -rf xxx ; mkdir /injection ; xxx - sprintf(tmpDirInjection, "xxx %s mkdir %s %s xxx", - CMD_SEPARATOR, tmpDir, CMD_SEPARATOR); - try - { - test_simple_put_core("small_file.csv", // filename - "auto", //source compression - true, // auto compress - true, // copyUploadFile - true, // verifyCopyUploadFile - false, // copyTableToStaging - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - tmpDirInjection, - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native - ); - } - catch (...) - { - //ignore exception as the failure is expected. - } - assert_false(sf_is_directory_exist(tmpDir)); - } -} - -void test_simple_put_config_temp_dir(void **unused) -{ - test_simple_put_config_temp_dir_core(false); -} - -void test_simple_put_config_temp_dir_native(void **unused) -{ - test_simple_put_config_temp_dir_core(true); + test_simple_put_core("small_file.csv", // filename + "auto", //source compression + true, // auto compress + true, // copyUploadFile + true, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64*1024*1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + tmpDirInjection + ); + } + catch (...) + { + //ignore exception as the failure is expected. + } + assert_false(sf_is_directory_exist(tmpDir)); } void test_simple_put_auto_detect_gzip(void ** unused) @@ -1040,7 +789,7 @@ void test_simple_put_create_subfolder(void **unused) test_simple_put_core("small_file.csv.gz", "gzip", false, false, false, false, false, false, 100*1024*1024, false, true); } -void test_simple_put_use_s3_regionalURL_core(bool native) +void test_simple_put_use_s3_regionalURL(void **unused) { test_simple_put_core("small_file.csv.gz", "gzip", false,false, false, @@ -1051,76 +800,30 @@ void test_simple_put_use_s3_regionalURL_core(bool native) false, false, nullptr, - true, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native - ); -} - -void test_simple_put_use_s3_regionalURL(void **unused) -{ - test_simple_put_use_s3_regionalURL_core(false); + true); } -void test_simple_put_use_s3_regionalURL_native(void **unused) -{ - test_simple_put_use_s3_regionalURL_core(true); -} - -void test_simple_get_core(bool native) +void test_simple_get(void **unused) { test_simple_put_core("small_file.csv", // filename "auto", //source compression - true, // auto compress - true, // copyUploadFile - true, // verifyCopyUploadFile - false, // copyTableToStaging - false, // createDupTable - false, // setCustomThreshold - 64*1024*1024, // customThreshold - false, // useDevUrand - false, // createSubfolder - nullptr, // tmpDir - false, // useS3regionalUrl - -1, //compressLevel - false, // overwrite - nullptr, // connection - false, // testUnicode - native + true // auto compress ); char tempDir[MAX_BUF_SIZE] = { 0 }; - char command[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; + char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/small_file.csv.gz file://"; sf_get_tmp_dir(tempDir); #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(command, tempDir); - test_simple_get_data(command, // getCommand - "48", // size - 0, // getThreshold - false, // testUnicode - native - ); -} - -void test_simple_get(void **unused) -{ - test_simple_get_core(false); -} - -void test_simple_get_native(void **unused) -{ - test_simple_get_core(true); + strcat(tempPath, tempDir); + test_simple_get_data(tempPath, "48"); } -void test_large_get_core(bool native) +void test_large_get(void **unused) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -1129,29 +832,14 @@ void test_large_get_core(bool native) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(command, tempDir); - test_simple_get_data(command, // getCommand - "5166848", // size - 0, // getThreshold - false, // testUnicode - native - ); -} - -void test_large_get(void **unused) -{ - test_large_get_core(false); -} - -void test_large_get_native(void **unused) -{ - test_large_get_core(true); + strcat(tempPath, tempDir); + test_simple_get_data(tempPath, "5166848"); } -void test_large_get_threshold_core(bool native) +void test_large_get_threshold(void **unused) { char tempDir[MAX_BUF_SIZE] = { 0 }; - char command[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; + char tempPath[MAX_BUF_SIZE] = "get @%test_small_put/bigFile.csv.gz file://"; if ( ! strncmp(getenv("CLOUD_PROVIDER"), "AWS", 6) ) { errno = 0; return; @@ -1160,42 +848,14 @@ void test_large_get_threshold_core(bool native) #ifdef _WIN32 getLongTempPath(tempDir); #endif - strcat(command, tempDir); - test_simple_get_data(command, // getCommand - "5166848", // size - 1000000, // getThreshold - false, // testUnicode - native - ); -} - -void test_large_get_threshold(void **unused) -{ - test_large_get_threshold_core(false); -} - -void test_large_get_threshold_native(void **unused) -{ - test_large_get_threshold_core(true); + strcat(tempPath, tempDir); + test_simple_get_data(tempPath, "5166848", 1000000); } static int gr_setup(void **unused) { initialize_test(SF_BOOLEAN_FALSE); - // TODO SNOW-1526335 - // Sometime we can't get OCSP response from cache server or responder - // Usually happen on GCP and should be ignored by FAIL_OPEN - // Unfortunately libsnowflakeclient doesn't support FAIL_OPEN for now - // so we have to disable OCSP validation to around it. - // Will remove this code when adding support for FAIL_OPEN (which is - // the default behavior for all other drivers) - char *cenv = getenv("CLOUD_PROVIDER"); - if (cenv && !strncmp(cenv, "GCP", 4)) { - sf_bool value = SF_BOOLEAN_FALSE; - snowflake_global_set_attribute(SF_GLOBAL_OCSP_CHECK, &value); - } - if(!setup_random_database()) { std::cout << "Failed to setup random database, fallback to use regular one." << std::endl; } @@ -1571,6 +1231,12 @@ void test_2GBlarge_put(void **unused) return; } } + // put/get for GCP is not supported in libsnowflakeclient + // will test that in odbc. + if (cenv && !strncmp(cenv, "GCP", 4)) { + errno = 0; + return; + } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1604,6 +1270,12 @@ void test_2GBlarge_get(void **unused) return; } } + // put/get for GCP is not supported in libsnowflakeclient + // will test that in odbc. + if (cenv && !strncmp(cenv, "GCP", 4)) { + errno = 0; + return; + } // Jenkins node on Mac has issue with large file. #ifdef __APPLE__ @@ -1965,7 +1637,7 @@ int main(void) { }); if(testAccount.find("GCP") != std::string::npos) { - setenv("CLOUD_PROVIDER", "GCP", 1); + setenv("CLOUD_PROVIDER", "GCP", 1); } else if(testAccount.find("AZURE") != std::string::npos) { @@ -1979,9 +1651,13 @@ int main(void) { char *cp = getenv("CLOUD_PROVIDER"); std::cout << "Cloud provider is " << cp << std::endl; #endif + const char *cloud_provider = std::getenv("CLOUD_PROVIDER"); + if(cloud_provider && ( strcmp(cloud_provider, "GCP") == 0 ) ) { + std::cout << "GCP put/get feature is not available in libsnowflakeclient." << std::endl; + return 0; + } const struct CMUnitTest tests[] = { -/* cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), @@ -2013,17 +1689,6 @@ int main(void) { cmocka_unit_test_teardown(test_simple_put_with_noproxy_fromenv, teardown), cmocka_unit_test_teardown(test_upload_file_to_stage_using_stream, donothing), cmocka_unit_test_teardown(test_put_get_with_unicode, teardown), -*/ - cmocka_unit_test_teardown(test_simple_put_auto_compress_native, teardown), - cmocka_unit_test_teardown(test_simple_put_config_temp_dir_native, teardown), - cmocka_unit_test_teardown(test_simple_get_native, teardown), - cmocka_unit_test_teardown(test_large_put_auto_compress_native, donothing), - cmocka_unit_test_teardown(test_large_get_native, donothing), - cmocka_unit_test_teardown(test_large_get_threshold_native, donothing), - cmocka_unit_test_teardown(test_large_reupload_native, donothing), - cmocka_unit_test_teardown(test_verify_upload, teardown), - cmocka_unit_test_teardown(test_simple_put_use_dev_urandom_native, teardown), - cmocka_unit_test_teardown(test_simple_put_use_s3_regionalURL_native, teardown), }; int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown); return ret; From b03491a44c87d964cbf3abc8adf3f012ae50f412 Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 18 Nov 2024 09:38:40 -0800 Subject: [PATCH 64/74] Add back missing files, fix styling change --- cpp/lib/result_set_arrow.cpp | 480 +++++++++++++++++++++++++++++++++++ cpp/lib/result_set_json.cpp | 260 +++++++++++++++++++ include/snowflake/client.h | 4 +- lib/result_set_arrow.h | 268 +++++++++++++++++++ lib/result_set_json.h | 259 +++++++++++++++++++ 5 files changed, 1269 insertions(+), 2 deletions(-) create mode 100644 cpp/lib/result_set_arrow.cpp create mode 100644 cpp/lib/result_set_json.cpp create mode 100644 lib/result_set_arrow.h create mode 100644 lib/result_set_json.h diff --git a/cpp/lib/result_set_arrow.cpp b/cpp/lib/result_set_arrow.cpp new file mode 100644 index 0000000000..090cbe1788 --- /dev/null +++ b/cpp/lib/result_set_arrow.cpp @@ -0,0 +1,480 @@ +/* + * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. + */ + +#include "arrowheaders.hpp" + +#include + +#include "memory.h" +#include "result_set_arrow.h" +#include "ResultSetArrow.hpp" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SF_WIN32 + rs_arrow_t * rs_arrow_create_with_json_result( + cJSON * json_rowset64, + SF_COLUMN_DESC * metadata, + const char * tz_string + ) + { + arrow::BufferBuilder* bufferBuilder = NULL; + if (json_rowset64) + { + const char * base64RowsetStr = snowflake_cJSON_GetStringValue(json_rowset64); + if (base64RowsetStr && strlen(base64RowsetStr) > 0) + { + // Decode Base64-encoded Arrow-format rowset of the chunk and build a buffer builder from it. + std::string decodedRowsetStr = arrow::util::base64_decode(std::string(base64RowsetStr)); + bufferBuilder = new arrow::BufferBuilder(); + (void) bufferBuilder->Append((void *)decodedRowsetStr.c_str(), decodedRowsetStr.length()); + } + } + + rs_arrow_t * rs_struct = (rs_arrow_t *) SF_MALLOC(sizeof(rs_arrow_t)); + Snowflake::Client::ResultSetArrow * rs_obj = + new Snowflake::Client::ResultSetArrow(bufferBuilder, metadata, std::string(tz_string)); + rs_struct->rs_object = rs_obj; + + return rs_struct; + } + + rs_arrow_t * rs_arrow_create_with_chunk( + NON_JSON_RESP * initial_chunk, + SF_COLUMN_DESC * metadata, + const char * tz_string + ) + { + rs_arrow_t * rs_struct = (rs_arrow_t *)SF_MALLOC(sizeof(rs_arrow_t)); + Snowflake::Client::ResultSetArrow * rs_obj = + new Snowflake::Client::ResultSetArrow((arrow::BufferBuilder*)(initial_chunk->buffer), metadata, std::string(tz_string)); + rs_struct->rs_object = rs_obj; + + // the buffer is passed in to result set so the response is no longer needed + delete initial_chunk; + + return rs_struct; + } + + void rs_arrow_destroy(rs_arrow_t * rs) + { + if (rs == NULL) + { + return; + } + + delete static_cast(rs->rs_object); + SF_FREE(rs); + } + + SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + + arrow::BufferBuilder * buffer = (arrow::BufferBuilder*)(chunk->buffer); + delete chunk; + return rs_obj->appendChunk(buffer); + } + + SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->next(); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsBool(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt8(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt32(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt64(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint8(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint32(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint64(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsFloat32(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsFloat64(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( + rs_arrow_t * rs, + size_t idx, + const char ** out_data + ) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsConstString(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( + rs_arrow_t * rs, + size_t idx, + SF_TIMESTAMP * out_data + ) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsTimestamp(idx, out_data); + } + + SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellStrlen(idx, out_data); + } + + size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return 0; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getRowCountInChunk(); + } + + SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) + { + Snowflake::Client::ResultSetArrow * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->isCellNull(idx, out_data); + } + + size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) + { + size_t data_size = size * nmemb; + arrow::BufferBuilder * arrowBufBuilder = (arrow::BufferBuilder*)(userdata); + + log_debug("Curl response for arrow chunk size: %zu", data_size); + (void) arrowBufBuilder->Append(ptr, data_size); + return data_size; + } + + NON_JSON_RESP* callback_create_arrow_resp(void) + { + NON_JSON_RESP* arrow_resp = new NON_JSON_RESP; + arrow_resp->buffer = new arrow::BufferBuilder(); + arrow_resp->write_callback = arrow_write_callback; + return arrow_resp; + } + +#else // SF_WIN32 +rs_arrow_t * rs_arrow_create_with_json_result( + cJSON * json_rowset64, + SF_COLUMN_DESC * metadata, + const char * tz_string +) +{ + log_error("Query results were fetched using Arrow"); + return NULL; +} + +rs_arrow_t * rs_arrow_create_with_chunk( + NON_JSON_RESP * initial_chunk, + SF_COLUMN_DESC * metadata, + const char * tz_string +) +{ + log_error("Query results were fetched using Arrow"); + return NULL; +} + +void rs_arrow_destroy(rs_arrow_t * rs) +{ + log_error("Query results were fetched using Arrow"); +} + +SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_finish_result_set(rs_arrow_t * rs) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_bool(rs_arrow_t * rs, size_t idx, sf_bool * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_int8(rs_arrow_t * rs, size_t idx, int8 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_int32(rs_arrow_t * rs, size_t idx, int32 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_int64(rs_arrow_t * rs, size_t idx, int64 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_uint8(rs_arrow_t * rs, size_t idx, uint8 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_uint32(rs_arrow_t * rs, size_t idx, uint32 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_uint64(rs_arrow_t * rs, size_t idx, uint64 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_float32(rs_arrow_t * rs, size_t idx, float32 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_float64(rs_arrow_t * rs, size_t idx, float64 * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( + rs_arrow_t * rs, + size_t idx, + const char ** out_data +) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_string( + rs_arrow_t * rs, + size_t idx, + char ** out_data, + size_t * io_len, + size_t * io_capacity +) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( + rs_arrow_t * rs, + size_t idx, + SF_TIMESTAMP * out_data +) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs) +{ + log_error("Query results were fetched using Arrow"); + return 0; +} + +size_t rs_arrow_get_total_row_count(rs_arrow_t * rs) +{ + log_error("Query results were fetched using Arrow"); + return 0; +} + +SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data) +{ + log_error("Query results were fetched using Arrow"); + return SF_STATUS_ERROR_UNSUPPORTED_QUERY_RESULT_FORMAT; +} + +size_t arrow_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + log_error("Query results were fetched using Arrow"); + return 0; +} + +NON_JSON_RESP* callback_create_arrow_resp(void) +{ + log_error("Query results were fetched using Arrow"); + return NULL; +} + +#endif // SF_WIN32 + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/cpp/lib/result_set_json.cpp b/cpp/lib/result_set_json.cpp new file mode 100644 index 0000000000..b276b865f5 --- /dev/null +++ b/cpp/lib/result_set_json.cpp @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. + */ + +#include + +#include "memory.h" +#include "result_set_json.h" +#include "ResultSetJson.hpp" + + +#ifdef __cplusplus +extern "C" { +#endif + + rs_json_t * rs_json_create( + cJSON * rowset, + SF_COLUMN_DESC * metadata, + const char * tz_string + ) + { + rs_json_t * rs_struct = (rs_json_t *) SF_MALLOC(sizeof(rs_json_t)); + Snowflake::Client::ResultSetJson * rs_obj = new Snowflake::Client::ResultSetJson( + rowset, metadata, std::string(tz_string)); + rs_struct->rs_object = rs_obj; + + return rs_struct; + } + + void rs_json_destroy(rs_json_t * rs) + { + if (rs == NULL) + { + return; + } + + delete static_cast(rs->rs_object); + SF_FREE(rs); + } + + SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->appendChunk(chunk); + } + + SF_STATUS STDCALL rs_json_next(rs_json_t * rs) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->next(); + } + + SF_STATUS STDCALL rs_json_get_cell_as_bool(rs_json_t * rs, size_t idx, sf_bool * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsBool(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_int8(rs_json_t * rs, size_t idx, int8 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt8(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_int32(rs_json_t * rs, size_t idx, int32 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt32(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_int64(rs_json_t * rs, size_t idx, int64 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsInt64(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_uint8(rs_json_t * rs, size_t idx, uint8 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint8(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_uint32(rs_json_t * rs, size_t idx, uint32 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint32(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_uint64(rs_json_t * rs, size_t idx, uint64 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsUint64(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_float32(rs_json_t * rs, size_t idx, float32 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsFloat32(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_float64(rs_json_t * rs, size_t idx, float64 * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsFloat64(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_const_string( + rs_json_t * rs, + size_t idx, + const char ** out_data + ) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsConstString(idx, out_data); + } + + SF_STATUS STDCALL rs_json_get_cell_as_timestamp( + rs_json_t * rs, + size_t idx, + SF_TIMESTAMP * out_data + ) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellAsTimestamp(idx, out_data); + } + + + SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getCellStrlen(idx, out_data); + } + + size_t rs_json_get_row_count_in_chunk(rs_json_t * rs) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return 0; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->getRowCountInChunk(); + } + + SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data) + { + Snowflake::Client::ResultSetJson * rs_obj; + + if (rs == NULL) + { + return SF_STATUS_ERROR_NULL_POINTER; + } + + rs_obj = static_cast (rs->rs_object); + return rs_obj->isCellNull(idx, out_data); + } + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/include/snowflake/client.h b/include/snowflake/client.h index 6bddfc2d6e..883d6f31bf 100644 --- a/include/snowflake/client.h +++ b/include/snowflake/client.h @@ -462,9 +462,9 @@ typedef struct SF_STMT { char request_id[SF_UUID4_LEN]; SF_ERROR_STRUCT error; SF_CONNECT *connection; - void* qrf; + void *qrf; char *sql_text; - void* result_set; + void *result_set; int64 chunk_rowcount; int64 total_rowcount; int64 total_fieldcount; diff --git a/lib/result_set_arrow.h b/lib/result_set_arrow.h new file mode 100644 index 0000000000..3250aad46f --- /dev/null +++ b/lib/result_set_arrow.h @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. + */ + +#ifndef SNOWFLAKE_RESULTSETARROW_H +#define SNOWFLAKE_RESULTSETARROW_H + +#include "cJSON.h" +#include "snowflake/basic_types.h" +#include "snowflake/client.h" +#include "connection.h" + +#ifdef __cplusplus +extern "C" { +#endif + + /** + * A result set interface for Arrow result format. + * + * @see cpp/lib/ResultSet.hpp + * @see cpp/lib/ResultSetArrow.hpp + */ + typedef struct rs_arrow { + void * rs_object; + } rs_arrow_t; + + /** + * Parameterized constructor. + * Initializes the result set with rowset64 data in json result set. + * + * @param json_rowset64 A pointer to the rowset64 data in json result set. + * @param metadata A pointer to the metadata for the result set. + * @param tz_string The time zone. + */ + rs_arrow_t * rs_arrow_create_with_json_result( + cJSON * json_rowset64, + SF_COLUMN_DESC * metadata, + const char * tz_string); + + /** + * Parameterized constructor. + * Initializes the result set with the first arrow chunk of the result set. + * + * @param initial_chunk A pointer to the first chunk of the result set. + * @param metadata A pointer to the metadata for the result set. + * @param tz_string The time zone. + */ + rs_arrow_t * rs_arrow_create_with_chunk( + NON_JSON_RESP * initial_chunk, + SF_COLUMN_DESC * metadata, + const char * tz_string); + + /** + * Destructor. + */ + void rs_arrow_destroy(rs_arrow_t * rs); + + /** + * Appends the given chunk to the internal result set. + * + * @param rs The ResultSetArrow object. + * @param chunk The chunk to append. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_append_chunk(rs_arrow_t * rs, NON_JSON_RESP * chunk); + + /** + * Advances to the next row. + * + * @param rs The ResultSetArrow object. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_next(rs_arrow_t * rs); + + /** + * Writes the value of the current cell as a boolean to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_bool( + rs_arrow_t * rs, + size_t idx, + sf_bool * out_data); + + /** + * Writes the value of the current cell as an int8 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_int8( + rs_arrow_t * rs, + size_t idx, + int8 * out_data); + + /** + * Writes the value of the current cell as an int32 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_int32( + rs_arrow_t * rs, + size_t idx, + int32 * out_data); + + /** + * Writes the value of the current cell as an int64 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_int64( + rs_arrow_t * rs, + size_t idx, + int64 * out_data); + + /** + * Writes the value of the current cell as a uint8 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_uint8( + rs_arrow_t * rs, + size_t idx, + uint8 * out_data); + + /** + * Writes the value of the current cell as a uint32 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_uint32( + rs_arrow_t * rs, + size_t idx, + uint32 * out_data); + + /** + * Writes the value of the current cell as a uint64 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_uint64( + rs_arrow_t * rs, + size_t idx, + uint64 * out_data); + + /** + * Writes the value of the current cell as a float32 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_float32( + rs_arrow_t * rs, + size_t idx, + float32 * out_data); + + /** + * Writes the value of the current cell as a float64 to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_float64( + rs_arrow_t * rs, + size_t idx, + float64 * out_data); + + /** + * Writes the value of the current cell as a constant C-string to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_const_string( + rs_arrow_t * rs, + size_t idx, + const char ** out_data); + + /** + * Writes the value of the current cell as a timestamp to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_as_timestamp( + rs_arrow_t * rs, + size_t idx, + SF_TIMESTAMP * out_data); + + /** + * Writes the length of the current cell to the provided buffer. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_get_cell_strlen(rs_arrow_t * rs, size_t idx, size_t * out_data); + + /** + * Gets the number of rows in the current chunk being processed. + * + * @param rs The ResultSetArrow object. + * + * @return the number of rows in the current chunk. + */ + size_t rs_arrow_get_row_count_in_chunk(rs_arrow_t * rs); + + /** + * Indiciates whether the current cell is null or not. + * + * @param rs The ResultSetArrow object. + * @param idx The index of the row to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_arrow_is_cell_null(rs_arrow_t * rs, size_t idx, sf_bool * out_data); + + NON_JSON_RESP* callback_create_arrow_resp(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SNOWFLAKE_RESULTSETARROW_H diff --git a/lib/result_set_json.h b/lib/result_set_json.h new file mode 100644 index 0000000000..c2a1d28d20 --- /dev/null +++ b/lib/result_set_json.h @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2021 Snowflake Computing, Inc. All rights reserved. + */ + +#ifndef SNOWFLAKE_RESULTSETJSON_H +#define SNOWFLAKE_RESULTSETJSON_H + +#include "cJSON.h" +#include "snowflake/basic_types.h" +#include "snowflake/client.h" + +#ifdef __cplusplus +extern "C" { +#endif + + /** + * A result set interface for JSON result format. + * + * @see cpp/lib/ResultSet.hpp + * @see cpp/lib/ResultSetJson.hpp + */ + typedef struct rs_json { + void * rs_object; + } rs_json_t; + + /** + * Parameterized constructor. + * Initializes the result set with required information as well as data. + * + * @param rowset A pointer to the result set data. + * @param metadata A pointer to the metadata for the result set. + * @param tz_string The time zone. + */ + rs_json_t * rs_json_create( + cJSON * rowset, + SF_COLUMN_DESC * metadata, + const char * tz_string); + + /** + * Destructor. + */ + void rs_json_destroy(rs_json_t * rs); + + /** + * Appends the given chunk to the internal result set. + * + * @param rs The ResultSetJson object. + * @param chunk The chunk to append. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_append_chunk(rs_json_t * rs, cJSON * chunk); + + /** + * Advances to next row. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_next(rs_json_t * rs); + + /** + * Writes the value of the current cell as a boolean to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_bool( + rs_json_t * rs, + size_t idx, + sf_bool * out_data); + + /** + * Writes the value of the current cell as an int8 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_int8( + rs_json_t * rs, + size_t idx, + int8 * out_data); + + /** + * Writes the value of the current cell as an int32 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_int32( + rs_json_t * rs, + size_t idx, + int32 * out_data); + + /** + * Writes the value of the current cell as an int64 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_int64( + rs_json_t * rs, + size_t idx, + int64 * out_data); + + /** + * Writes the value of the current cell as a uint8 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_uint8( + rs_json_t * rs, + size_t idx, + uint8 * out_data); + + /** + * Writes the value of the current cell as a uint32 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_uint32( + rs_json_t * rs, + size_t idx, + uint32 * out_data); + + /** + * Writes the value of the current cell as a uint64 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_uint64( + rs_json_t * rs, + size_t idx, + uint64 * out_data); + + /** + * Writes the value of the current cell as a float32 to the provided buffer. + * + * @param rs The ResultSet object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_float32( + rs_json_t * rs, + size_t idx, + float32 * out_data); + + /** + * Writes the value of the current cell as a float64 to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_float64( + rs_json_t * rs, + size_t idx, + float64 * out_data); + + /** + * Writes the value of the current cell as a constant C-string to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_const_string( + rs_json_t * rs, + size_t idx, + const char ** out_data); + + /** + * Writes the value of the current cell as a timestamp to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_as_timestamp( + rs_json_t * rs, + size_t idx, + SF_TIMESTAMP * out_data); + + /** + * Writes the length of the current cell to the provided buffer. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_get_cell_strlen(rs_json_t * rs, size_t idx, size_t * out_data); + + /** + * Gets the number of rows in the current chunk being processed. + * + * @param rs The ResultSetJson object. + * + * @return the number of rows in the current chunk. + */ + size_t rs_json_get_row_count_in_chunk(rs_json_t * rs); + + /** + * Gets the total number of rows in the entire result set. + * + * @param rs The ResultSetJson object. + * + * @return the number of rows in the result set. + */ + size_t rs_json_get_total_row_count(rs_json_t * rs); + + /** + * Indiciates whether the current cell is null or not. + * + * @param rs The ResultSetJson object. + * @param idx The index of the column to retrieve. + * @param out_data The buffer to write to. + * + * @return 0 if successful, otherwise an error is returned. + */ + SF_STATUS STDCALL rs_json_is_cell_null(rs_json_t * rs, size_t idx, sf_bool * out_data); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SNOWFLAKE_RESULTSETJSON_H From 0d2e977ac6cafca0a382597af3ec65010f617ef3 Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 18 Nov 2024 15:05:35 -0800 Subject: [PATCH 65/74] Disable test case for windows 32-bit debug --- tests/test_unit_logger.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index bfce12985a..0af4bc3a9b 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -38,7 +38,7 @@ void test_invalid_client_config_path(void** unused) { sf_bool result = load_client_config(configFilePath, &clientConfig); assert_false(result); } - +#if !defined(_WIN32) && !defined(_DEBUG) /** * Tests log settings from client config file with invalid json */ @@ -58,6 +58,7 @@ void test_client_config_log_invalid_json(void** unused) { // Cleanup remove(configFilePath); } +#endif #ifndef _WIN32 /** @@ -252,7 +253,9 @@ int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), cmocka_unit_test(test_invalid_client_config_path), +#if !defined(_WIN32) && !defined(_DEBUG) cmocka_unit_test(test_client_config_log_invalid_json), +#endif #ifndef _WIN32 cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), From 36faf0779b9f7cade55e411e589202dadbd70f07 Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 18 Nov 2024 17:17:24 -0800 Subject: [PATCH 66/74] Ignore client config loading on win32 debug --- cpp/lib/ClientConfigParser.cpp | 3 +++ tests/test_unit_logger.c | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 9b796bfead..bb73a3522e 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -35,6 +35,7 @@ sf_bool load_client_config( const char* in_configFilePath, client_config* out_clientConfig) { +#if !defined(_WIN32) && !defined(_DEBUG) try { EasyLoggingConfigParser configParser; configParser.loadClientConfig(in_configFilePath, *out_clientConfig); @@ -42,6 +43,7 @@ sf_bool load_client_config( CXX_LOG_ERROR("Using client configuration path from a connection string: %s", e.what()); return false; } +#endif return true; } @@ -181,6 +183,7 @@ void EasyLoggingConfigParser::parseConfigFile( checkIfValidPermissions(in_filePath); #endif err = parse(jsonConfig, configFile); + if (!err.empty()) { CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 0af4bc3a9b..0b4795b08e 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -36,9 +36,13 @@ void test_invalid_client_config_path(void** unused) { // Parse client config for log details client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); +#if !defined(_WIN32) && !defined(_DEBUG) assert_false(result); +#else + assert_true(result); +#endif } -#if !defined(_WIN32) && !defined(_DEBUG) + /** * Tests log settings from client config file with invalid json */ @@ -53,12 +57,15 @@ void test_client_config_log_invalid_json(void** unused) { // Parse client config for log details client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); +#if !defined(_WIN32) && !defined(_DEBUG) assert_false(result); +#else + assert_true(result); +#endif // Cleanup remove(configFilePath); } -#endif #ifndef _WIN32 /** @@ -253,9 +260,7 @@ int main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_log_str_to_level), cmocka_unit_test(test_invalid_client_config_path), -#if !defined(_WIN32) && !defined(_DEBUG) cmocka_unit_test(test_client_config_log_invalid_json), -#endif #ifndef _WIN32 cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), From be2a7b3c19e8bfe7d6ac2dc26386d77054ba2d76 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 20 Nov 2024 11:45:58 -0800 Subject: [PATCH 67/74] Add comment explaining the ifdef --- cpp/lib/ClientConfigParser.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index bb73a3522e..d4d14dcc0a 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -35,6 +35,8 @@ sf_bool load_client_config( const char* in_configFilePath, client_config* out_clientConfig) { +// Disable easy logging for 32-bit windows debug build due to linking issues +// with _osfile causing hanging/assertions until dynamic linking is available #if !defined(_WIN32) && !defined(_DEBUG) try { EasyLoggingConfigParser configParser; From d6a00fed38f1e1a35b7d876686b85bb8320eaccd Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 21 Nov 2024 08:54:41 -0800 Subject: [PATCH 68/74] Fix log message --- cpp/lib/ClientConfigParser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index d4d14dcc0a..aa02d89f3b 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -42,7 +42,7 @@ sf_bool load_client_config( EasyLoggingConfigParser configParser; configParser.loadClientConfig(in_configFilePath, *out_clientConfig); } catch (std::exception e) { - CXX_LOG_ERROR("Using client configuration path from a connection string: %s", e.what()); + CXX_LOG_ERROR("Error loading client configuration: %s", e.what()); return false; } #endif From 726b897208c6d5359290bbb423ef177507aca811 Mon Sep 17 00:00:00 2001 From: norrislee Date: Mon, 25 Nov 2024 11:11:43 -0800 Subject: [PATCH 69/74] Remove unncessary else clauses and removed ifstream.close --- cpp/lib/ClientConfigParser.cpp | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index aa02d89f3b..89f9052737 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -93,34 +93,32 @@ std::string EasyLoggingConfigParser::resolveClientConfigPath( const std::string& in_configFilePath) { char envbuf[MAX_PATH + 1]; + + // 1. Try config file if it was passed in if (!in_configFilePath.empty()) { - // 1. Try config file if it was passed in CXX_LOG_INFO("Using client configuration path from a connection string: %s", in_configFilePath.c_str()); return in_configFilePath; } - else if (const char* clientConfigEnv = sf_getenv_s(SF_CLIENT_CONFIG_ENV_NAME.c_str(), envbuf, sizeof(envbuf))) + + // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME + if (const char* clientConfigEnv = sf_getenv_s(SF_CLIENT_CONFIG_ENV_NAME.c_str(), envbuf, sizeof(envbuf))) { - // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME CXX_LOG_INFO("Using client configuration path from an environment variable: %s", clientConfigEnv); return clientConfigEnv; } - else + + // 3. Try DLL binary dir + std::string binaryDir = getBinaryPath(); + std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; + if (boost::filesystem::is_regular_file(binaryDirFilePath)) { - // 3. Try DLL binary dir - std::string binaryDir = getBinaryPath(); - std::string binaryDirFilePath = binaryDir + SF_CLIENT_CONFIG_FILE_NAME; - if (boost::filesystem::is_regular_file(binaryDirFilePath)) - { - CXX_LOG_INFO("Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); - return binaryDirFilePath; - } - else - { - // 4. Try user home dir - return resolveHomeDirConfigPath(); - } + CXX_LOG_INFO("Using client configuration path from binary directory: %s", binaryDirFilePath.c_str()); + return binaryDirFilePath; } + + // 4. Try user home dir + return resolveHomeDirConfigPath(); } // Private ===================================================================== @@ -195,7 +193,6 @@ void EasyLoggingConfigParser::parseConfigFile( } catch (std::exception& e) { - configFile.close(); throw; } @@ -218,7 +215,6 @@ void EasyLoggingConfigParser::parseConfigFile( sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); } } - configFile.close(); } //////////////////////////////////////////////////////////////////////////////// From a6e6ca030eea4c3b504e57dd8aed0872e3ebd381 Mon Sep 17 00:00:00 2001 From: norrislee Date: Thu, 28 Nov 2024 22:13:07 -0800 Subject: [PATCH 70/74] Remove exceptions, add test case, fix home env bug --- cpp/lib/ClientConfigParser.cpp | 71 +++++++++++------------- include/snowflake/ClientConfigParser.hpp | 23 +++----- tests/test_unit_logger.c | 49 +++++++++++++++- 3 files changed, 88 insertions(+), 55 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 89f9052737..c0dd26e81c 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -37,16 +37,12 @@ sf_bool load_client_config( { // Disable easy logging for 32-bit windows debug build due to linking issues // with _osfile causing hanging/assertions until dynamic linking is available -#if !defined(_WIN32) && !defined(_DEBUG) - try { - EasyLoggingConfigParser configParser; - configParser.loadClientConfig(in_configFilePath, *out_clientConfig); - } catch (std::exception e) { - CXX_LOG_ERROR("Error loading client configuration: %s", e.what()); - return false; - } -#endif +#if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) + EasyLoggingConfigParser configParser; + return configParser.loadClientConfig(in_configFilePath, *out_clientConfig); +#else return true; +#endif } //////////////////////////////////////////////////////////////////////////////// @@ -76,7 +72,7 @@ EasyLoggingConfigParser::~EasyLoggingConfigParser() } //////////////////////////////////////////////////////////////////////////////// -void EasyLoggingConfigParser::loadClientConfig( +sf_bool EasyLoggingConfigParser::loadClientConfig( const std::string& in_configFilePath, client_config& out_clientConfig) { @@ -84,8 +80,9 @@ void EasyLoggingConfigParser::loadClientConfig( if (!derivedConfigPath.empty()) { - parseConfigFile(derivedConfigPath, out_clientConfig); + return parseConfigFile(derivedConfigPath, out_clientConfig); } + return false; } //////////////////////////////////////////////////////////////////////////////// @@ -128,14 +125,15 @@ std::string EasyLoggingConfigParser::resolveHomeDirConfigPath() char envbuf[MAX_PATH + 1]; #if defined(WIN32) || defined(_WIN64) std::string homeDirFilePath; - char* homeDir; - if ((homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf))) && strlen(homeDir) != 0) + char* homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); + if (homeDir && strlen(homeDir) != 0) { - homeDirFilePath = homeDir + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; + homeDirFilePath = std::string(homeDir) + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; } else { // USERPROFILE is empty, try HOMEDRIVE and HOMEPATH + char envbuf2[MAX_PATH + 1]; char* homeDriveEnv = sf_getenv_s("HOMEDRIVE", envbuf, sizeof(envbuf)); - char* homePathEnv = sf_getenv_s("HOMEPATH", envbuf, sizeof(envbuf)); + char* homePathEnv = sf_getenv_s("HOMEPATH", envbuf2, sizeof(envbuf2)); if (homeDriveEnv && strlen(homeDriveEnv) != 0 && homePathEnv && strlen(homePathEnv) != 0) { homeDirFilePath = std::string(homeDriveEnv) + homePathEnv + PATH_SEP + SF_CLIENT_CONFIG_FILE_NAME; @@ -162,38 +160,32 @@ std::string EasyLoggingConfigParser::resolveHomeDirConfigPath() } //////////////////////////////////////////////////////////////////////////////// -void EasyLoggingConfigParser::parseConfigFile( +sf_bool EasyLoggingConfigParser::parseConfigFile( const std::string& in_filePath, client_config& out_clientConfig) { value jsonConfig; std::string err; std::ifstream configFile; - try + configFile.open(in_filePath, std::fstream::in | std::ios::binary); + if (!configFile) { - configFile.open(in_filePath, std::fstream::in | std::ios::binary); - if (!configFile) - { - CXX_LOG_INFO("Could not open a file. The file may not exist: %s", - in_filePath.c_str()); - std::string errMsg = "Error finding client configuration file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); - } + CXX_LOG_INFO("Could not open a file. The file may not exist: %s", + in_filePath.c_str()); + return false; + } #if !defined(WIN32) && !defined(_WIN64) - checkIfValidPermissions(in_filePath); + if (!checkIfValidPermissions(in_filePath)) + { + return false; + } #endif - err = parse(jsonConfig, configFile); + err = parse(jsonConfig, configFile); - if (!err.empty()) - { - CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); - std::string errMsg = "Error parsing client configuration file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); - } - } - catch (std::exception& e) + if (!err.empty()) { - throw; + CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); + return false; } value commonProps = jsonConfig.get("common"); @@ -215,10 +207,11 @@ void EasyLoggingConfigParser::parseConfigFile( sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); } } + return true; } //////////////////////////////////////////////////////////////////////////////// -void EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_filePath) +sf_bool EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_filePath) { boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); boost::filesystem::perms permissions = fileStatus.permissions(); @@ -227,9 +220,9 @@ void EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_file { CXX_LOG_ERROR("Error due to other users having permission to modify the config file: %s", in_filePath.c_str()); - std::string errMsg = "Error due to other users having permission to modify the config file: " + in_filePath; - throw ClientConfigException(errMsg.c_str()); + return false; } + return true; } //////////////////////////////////////////////////////////////////////////////// diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index b84dbe9e65..b27fc56373 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -13,17 +13,6 @@ namespace Snowflake { namespace Client { - struct ClientConfigException : public std::exception - { - ClientConfigException(const std::string& message) : message_(message) {} - const char* what() const noexcept - { - return message_.c_str(); - } - - std::string message_; - }; - class EasyLoggingConfigParser { // Public ================================================================== @@ -47,8 +36,10 @@ namespace Client * * @param in_configFilePath The config file path passed in by the user. * @param out_clientConfig The SFClientConfig object to be filled. + * + * @return whether the client config was loaded properly */ - void loadClientConfig( + sf_bool loadClientConfig( const std::string& in_configFilePath, client_config& out_clientConfig); @@ -75,8 +66,10 @@ namespace Client * * @param in_filePath The filePath of the config file to parse. * @param out_clientConfig The SFClientConfig object to be filled. + * + * @return whether parsing the client config file was successful. */ - void parseConfigFile( + sf_bool parseConfigFile( const std::string& in_filePath, client_config& out_clientConfig); @@ -84,8 +77,10 @@ namespace Client * @ brief Check if other have permission to modify file * * @param in_filePath The file path of the config file to check permissions. + * + * @return whether the file has valid permissions */ - void checkIfValidPermissions(const std::string& in_filePath); + sf_bool checkIfValidPermissions(const std::string& in_filePath); /** * @ brief Check if there are unknown entries in config file diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 0b4795b08e..856ae5c1e1 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -36,7 +36,7 @@ void test_invalid_client_config_path(void** unused) { // Parse client config for log details client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); -#if !defined(_WIN32) && !defined(_DEBUG) +#if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) assert_false(result); #else assert_true(result); @@ -57,7 +57,7 @@ void test_client_config_log_invalid_json(void** unused) { // Parse client config for log details client_config clientConfig = { 0 }; sf_bool result = load_client_config(configFilePath, &clientConfig); -#if !defined(_WIN32) && !defined(_DEBUG) +#if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) assert_false(result); #else assert_true(result); @@ -107,6 +107,7 @@ void test_client_config_log(void **unused) { // Cleanup remove(configFilePath); remove(LOG_PATH); + SF_FREE(LOG_PATH); } /** @@ -143,6 +144,49 @@ void test_client_config_log_init(void** unused) { remove(LOG_PATH); } +/** + * Tests log settings from client config file via global init in home dir + */ +void test_client_config_log_init_home_config(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; + + char envbuf[MAX_PATH + 1]; + char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf)); + char configFile[] = "/sf_client_config.json"; + size_t log_path_size = strlen(homeDir) + strlen(configFile) + 1; + char *configFilePath = (char*)SF_CALLOC(1, log_path_size); + sf_strcat(configFilePath, log_path_size, homeDir); + sf_strcat(configFilePath, log_path_size, configFile); + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, ""); + snowflake_global_init("./logs", SF_LOG_DEFAULT, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, F_OK), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); + SF_FREE(configFilePath); +} + /** * Tests timing of log file creation */ @@ -264,6 +308,7 @@ int main(void) { #ifndef _WIN32 cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), + cmocka_unit_test(test_client_config_log_init_home_config), cmocka_unit_test(test_log_creation), cmocka_unit_test(test_mask_secret_log), #endif From 21f7b13610561408c198400820a2490b8e821fd8 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 11 Dec 2024 15:53:27 -0800 Subject: [PATCH 71/74] Add test cases, fix config parsing return when json is malformed --- cpp/lib/ClientConfigParser.cpp | 37 ++++++----- tests/test_unit_logger.c | 117 ++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 18 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index c0dd26e81c..6ddf76d4d7 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -188,26 +188,31 @@ sf_bool EasyLoggingConfigParser::parseConfigFile( return false; } - value commonProps = jsonConfig.get("common"); - checkUnknownEntries(commonProps); - if (commonProps.is()) + if (jsonConfig.is()) { - if (commonProps.contains("log_level") && commonProps.get("log_level").is()) + value commonProps = jsonConfig.get("common"); + if (commonProps.is()) { - const char* logLevel = commonProps.get("log_level").get().c_str(); - size_t logLevelSize = strlen(logLevel) + 1; - out_clientConfig.logLevel = (char*)SF_CALLOC(1, logLevelSize); - sf_strcpy(out_clientConfig.logLevel, logLevelSize, logLevel); - } - if (commonProps.contains("log_path") && commonProps.get("log_path").is()) - { - const char* logPath = commonProps.get("log_path").get().c_str(); - size_t logPathSize = strlen(logPath) + 1; - out_clientConfig.logPath = (char*)SF_CALLOC(1, logPathSize); - sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); + checkUnknownEntries(commonProps); + if (commonProps.contains("log_level") && commonProps.get("log_level").is()) + { + const char* logLevel = commonProps.get("log_level").get().c_str(); + size_t logLevelSize = strlen(logLevel) + 1; + out_clientConfig.logLevel = (char*)SF_CALLOC(1, logLevelSize); + sf_strcpy(out_clientConfig.logLevel, logLevelSize, logLevel); + } + if (commonProps.contains("log_path") && commonProps.get("log_path").is()) + { + const char* logPath = commonProps.get("log_path").get().c_str(); + size_t logPathSize = strlen(logPath) + 1; + out_clientConfig.logPath = (char*)SF_CALLOC(1, logPathSize); + sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); + } + return true; } } - return true; + CXX_LOG_ERROR("Malformed client config file: %s", in_filePath.c_str()); + return false; } //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 856ae5c1e1..04dd1e2614 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -9,6 +9,11 @@ #ifndef _WIN32 #include +#else +#define F_OK 0 +inline int access(const char* pathname, int mode) { + return _access(pathname, mode); +} #endif /** @@ -67,7 +72,30 @@ void test_client_config_log_invalid_json(void** unused) { remove(configFilePath); } -#ifndef _WIN32 +/** + * Tests log settings from client config file with malformed json + */ +void test_client_config_log_malformed_json(void** unused) { + char clientConfigJSON[] = "[]"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + // Parse client config for log details + client_config clientConfig = { 0 }; + sf_bool result = load_client_config(configFilePath, &clientConfig); +#if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) + assert_false(result); +#else + assert_true(result); +#endif + + // Cleanup + remove(configFilePath); +} + /** * Tests log settings from client config file */ @@ -86,6 +114,7 @@ void test_client_config_log(void **unused) { // Set log name and level char logname[] = "%s/dummy.log"; size_t log_path_size = 1 + strlen(logname); + assert_non_null(clientConfig.logPath); log_path_size += strlen(clientConfig.logPath); char* LOG_PATH = (char*)SF_CALLOC(1, log_path_size); sf_sprintf(LOG_PATH, log_path_size, logname, clientConfig.logPath); @@ -153,7 +182,11 @@ void test_client_config_log_init_home_config(void** unused) { char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; char envbuf[MAX_PATH + 1]; +#ifdef _WIN32 + char* homeDir = sf_getenv_s("USERPROFILE", envbuf, sizeof(envbuf)); +#else char* homeDir = sf_getenv_s("HOME", envbuf, sizeof(envbuf)); +#endif char configFile[] = "/sf_client_config.json"; size_t log_path_size = strlen(homeDir) + strlen(configFile) + 1; char *configFilePath = (char*)SF_CALLOC(1, log_path_size); @@ -187,6 +220,82 @@ void test_client_config_log_init_home_config(void** unused) { SF_FREE(configFilePath); } +/** + * Tests log settings from client config file without log_path + */ +void test_client_config_log_no_level(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + char LOG_SUBPATH[MAX_PATH] = { 0 }; + char clientConfigJSON[] = "{\"common\":{\"log_path\":\"./test/\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); + snowflake_global_init("./logs", SF_LOG_DEFAULT, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level default is fatal + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Warning log won't trigger the log file creation since log level default is fatal + log_warn("dummy warning log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Fatal log will trigger the log file creation + log_fatal("dummy fatal log"); + assert_int_equal(access(LOG_PATH, F_OK), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + +/** + * Tests log settings from client config file without log_level + */ +void test_client_config_log_no_path(void** unused) { + char LOG_PATH[MAX_PATH] = { 0 }; + char LOG_SUBPATH[MAX_PATH] = { 0 }; + char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\"}}"; + char configFilePath[] = "sf_client_config.json"; + FILE* file; + file = fopen(configFilePath, "w"); + fprintf(file, "%s", clientConfigJSON); + fclose(file); + + snowflake_global_set_attribute(SF_GLOBAL_CLIENT_CONFIG_FILE, configFilePath); + snowflake_global_init("./logs", SF_LOG_DEFAULT, NULL); + + // Get the log path determined by libsnowflakeclient + snowflake_global_get_attribute(SF_GLOBAL_LOG_PATH, LOG_PATH, MAX_PATH); + memcpy(LOG_SUBPATH, &LOG_PATH[0], 6); + assert_string_equal(LOG_SUBPATH, "./logs"); + // Ensure the log file doesn't exist at the beginning + remove(LOG_PATH); + + // Info log won't trigger the log file creation since log level is set to warn in config + log_info("dummy info log"); + assert_int_not_equal(access(LOG_PATH, F_OK), 0); + + // Warning log will trigger the log file creation + log_warn("dummy warning log"); + assert_int_equal(access(LOG_PATH, F_OK), 0); + log_close(); + + // Cleanup + remove(configFilePath); + remove(LOG_PATH); +} + /** * Tests timing of log file creation */ @@ -214,6 +323,7 @@ void test_log_creation(void **unused) { remove(logname); } +#ifndef _WIN32 /** * Tests masking secret information in log */ @@ -305,11 +415,14 @@ int main(void) { cmocka_unit_test(test_log_str_to_level), cmocka_unit_test(test_invalid_client_config_path), cmocka_unit_test(test_client_config_log_invalid_json), -#ifndef _WIN32 + cmocka_unit_test(test_client_config_log_malformed_json), cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), cmocka_unit_test(test_client_config_log_init_home_config), + cmocka_unit_test(test_client_config_log_no_level), + cmocka_unit_test(test_client_config_log_no_path), cmocka_unit_test(test_log_creation), +#ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), #endif }; From 31001f30629fa19ab837c40629ee1b030f227c95 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 11 Dec 2024 16:03:46 -0800 Subject: [PATCH 72/74] Fix code quality --- tests/test_unit_logger.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 04dd1e2614..038ca0abab 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -19,7 +19,7 @@ inline int access(const char* pathname, int mode) { /** * Tests converting a string representation of log level to the log level enum */ -void test_log_str_to_level(void **unused) { +void test_log_str_to_level() { assert_int_equal(log_from_str_to_level("TRACE"), SF_LOG_TRACE); assert_int_equal(log_from_str_to_level("DEBUG"), SF_LOG_DEBUG); assert_int_equal(log_from_str_to_level("INFO"), SF_LOG_INFO); @@ -35,7 +35,7 @@ void test_log_str_to_level(void **unused) { /** * Tests log settings with invalid client config filepath */ -void test_invalid_client_config_path(void** unused) { +void test_invalid_client_config_path() { char configFilePath[] = "fakePath.json"; // Parse client config for log details @@ -51,7 +51,7 @@ void test_invalid_client_config_path(void** unused) { /** * Tests log settings from client config file with invalid json */ -void test_client_config_log_invalid_json(void** unused) { +void test_client_config_log_invalid_json() { char clientConfigJSON[] = "{{{\"invalid json\"}"; char configFilePath[] = "sf_client_config.json"; FILE* file; @@ -75,7 +75,7 @@ void test_client_config_log_invalid_json(void** unused) { /** * Tests log settings from client config file with malformed json */ -void test_client_config_log_malformed_json(void** unused) { +void test_client_config_log_malformed_json() { char clientConfigJSON[] = "[]"; char configFilePath[] = "sf_client_config.json"; FILE* file; @@ -99,7 +99,7 @@ void test_client_config_log_malformed_json(void** unused) { /** * Tests log settings from client config file */ -void test_client_config_log(void **unused) { +void test_client_config_log() { char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; char configFilePath[] = "sf_client_config.json"; FILE *file; @@ -142,7 +142,7 @@ void test_client_config_log(void **unused) { /** * Tests log settings from client config file via global init */ -void test_client_config_log_init(void** unused) { +void test_client_config_log_init() { char LOG_PATH[MAX_PATH] = { 0 }; char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; char configFilePath[] = "sf_client_config.json"; @@ -176,7 +176,7 @@ void test_client_config_log_init(void** unused) { /** * Tests log settings from client config file via global init in home dir */ -void test_client_config_log_init_home_config(void** unused) { +void test_client_config_log_init_home_config() { char LOG_PATH[MAX_PATH] = { 0 }; char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\",\"log_path\":\"./test/\"}}"; @@ -223,9 +223,8 @@ void test_client_config_log_init_home_config(void** unused) { /** * Tests log settings from client config file without log_path */ -void test_client_config_log_no_level(void** unused) { +void test_client_config_log_no_level() { char LOG_PATH[MAX_PATH] = { 0 }; - char LOG_SUBPATH[MAX_PATH] = { 0 }; char clientConfigJSON[] = "{\"common\":{\"log_path\":\"./test/\"}}"; char configFilePath[] = "sf_client_config.json"; FILE* file; @@ -262,7 +261,7 @@ void test_client_config_log_no_level(void** unused) { /** * Tests log settings from client config file without log_level */ -void test_client_config_log_no_path(void** unused) { +void test_client_config_log_no_path() { char LOG_PATH[MAX_PATH] = { 0 }; char LOG_SUBPATH[MAX_PATH] = { 0 }; char clientConfigJSON[] = "{\"common\":{\"log_level\":\"warn\"}}"; @@ -299,7 +298,7 @@ void test_client_config_log_no_path(void** unused) { /** * Tests timing of log file creation */ -void test_log_creation(void **unused) { +void test_log_creation() { char logname[] = "dummy.log"; // ensure the log file doesn't exist at the beginning @@ -327,7 +326,7 @@ void test_log_creation(void **unused) { /** * Tests masking secret information in log */ -void test_mask_secret_log(void **unused) { +void test_mask_secret_log() { FILE* fp = fopen("dummy.log", "w+"); assert_non_null(fp); log_set_lock(NULL); From 905bc6d8db8cb57f12aa0be29ae7576d6d344667 Mon Sep 17 00:00:00 2001 From: norrislee Date: Wed, 11 Dec 2024 17:03:04 -0800 Subject: [PATCH 73/74] Remove client config logger tests for Windows 32-bit debug --- tests/test_unit_logger.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index 038ca0abab..862c0c8709 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -415,11 +415,13 @@ int main(void) { cmocka_unit_test(test_invalid_client_config_path), cmocka_unit_test(test_client_config_log_invalid_json), cmocka_unit_test(test_client_config_log_malformed_json), +#if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) cmocka_unit_test(test_client_config_log), cmocka_unit_test(test_client_config_log_init), cmocka_unit_test(test_client_config_log_init_home_config), cmocka_unit_test(test_client_config_log_no_level), cmocka_unit_test(test_client_config_log_no_path), +#endif cmocka_unit_test(test_log_creation), #ifndef _WIN32 cmocka_unit_test(test_mask_secret_log), From 6a53849cc7e857095a29692f4698ca2f0fb803b3 Mon Sep 17 00:00:00 2001 From: norrislee Date: Fri, 20 Dec 2024 10:06:33 -0800 Subject: [PATCH 74/74] Remove in_ and out_ from function parameters, update copyright date --- cpp/lib/ClientConfigParser.cpp | 56 ++++++++++++------------ include/snowflake/ClientConfigParser.hpp | 28 ++++++------ include/snowflake/client_config_parser.h | 8 ++-- tests/test_unit_logger.c | 2 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/cpp/lib/ClientConfigParser.cpp b/cpp/lib/ClientConfigParser.cpp index 6ddf76d4d7..2b696a28df 100644 --- a/cpp/lib/ClientConfigParser.cpp +++ b/cpp/lib/ClientConfigParser.cpp @@ -32,14 +32,14 @@ namespace //////////////////////////////////////////////////////////////////////////////// sf_bool load_client_config( - const char* in_configFilePath, - client_config* out_clientConfig) + const char* configFilePath, + client_config* clientConfig) { // Disable easy logging for 32-bit windows debug build due to linking issues // with _osfile causing hanging/assertions until dynamic linking is available #if (!defined(_WIN32) && !defined(_DEBUG)) || defined(_WIN64) EasyLoggingConfigParser configParser; - return configParser.loadClientConfig(in_configFilePath, *out_clientConfig); + return configParser.loadClientConfig(configFilePath, *clientConfig); #else return true; #endif @@ -73,29 +73,29 @@ EasyLoggingConfigParser::~EasyLoggingConfigParser() //////////////////////////////////////////////////////////////////////////////// sf_bool EasyLoggingConfigParser::loadClientConfig( - const std::string& in_configFilePath, - client_config& out_clientConfig) + const std::string& configFilePath, + client_config& clientConfig) { - std::string derivedConfigPath = resolveClientConfigPath(in_configFilePath); + std::string derivedConfigPath = resolveClientConfigPath(configFilePath); if (!derivedConfigPath.empty()) { - return parseConfigFile(derivedConfigPath, out_clientConfig); + return parseConfigFile(derivedConfigPath, clientConfig); } return false; } //////////////////////////////////////////////////////////////////////////////// std::string EasyLoggingConfigParser::resolveClientConfigPath( - const std::string& in_configFilePath) + const std::string& configFilePath) { char envbuf[MAX_PATH + 1]; // 1. Try config file if it was passed in - if (!in_configFilePath.empty()) + if (!configFilePath.empty()) { - CXX_LOG_INFO("Using client configuration path from a connection string: %s", in_configFilePath.c_str()); - return in_configFilePath; + CXX_LOG_INFO("Using client configuration path from a connection string: %s", configFilePath.c_str()); + return configFilePath; } // 2. Try environment variable SF_CLIENT_CONFIG_ENV_NAME @@ -161,21 +161,21 @@ std::string EasyLoggingConfigParser::resolveHomeDirConfigPath() //////////////////////////////////////////////////////////////////////////////// sf_bool EasyLoggingConfigParser::parseConfigFile( - const std::string& in_filePath, - client_config& out_clientConfig) + const std::string& filePath, + client_config& clientConfig) { value jsonConfig; std::string err; std::ifstream configFile; - configFile.open(in_filePath, std::fstream::in | std::ios::binary); + configFile.open(filePath, std::fstream::in | std::ios::binary); if (!configFile) { CXX_LOG_INFO("Could not open a file. The file may not exist: %s", - in_filePath.c_str()); + filePath.c_str()); return false; } #if !defined(WIN32) && !defined(_WIN64) - if (!checkIfValidPermissions(in_filePath)) + if (!checkIfValidPermissions(filePath)) { return false; } @@ -184,7 +184,7 @@ sf_bool EasyLoggingConfigParser::parseConfigFile( if (!err.empty()) { - CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", in_filePath.c_str(), err.c_str()); + CXX_LOG_ERROR("Error in parsing JSON: %s, err: %s", filePath.c_str(), err.c_str()); return false; } @@ -198,44 +198,44 @@ sf_bool EasyLoggingConfigParser::parseConfigFile( { const char* logLevel = commonProps.get("log_level").get().c_str(); size_t logLevelSize = strlen(logLevel) + 1; - out_clientConfig.logLevel = (char*)SF_CALLOC(1, logLevelSize); - sf_strcpy(out_clientConfig.logLevel, logLevelSize, logLevel); + clientConfig.logLevel = (char*)SF_CALLOC(1, logLevelSize); + sf_strcpy(clientConfig.logLevel, logLevelSize, logLevel); } if (commonProps.contains("log_path") && commonProps.get("log_path").is()) { const char* logPath = commonProps.get("log_path").get().c_str(); size_t logPathSize = strlen(logPath) + 1; - out_clientConfig.logPath = (char*)SF_CALLOC(1, logPathSize); - sf_strcpy(out_clientConfig.logPath, logPathSize, logPath); + clientConfig.logPath = (char*)SF_CALLOC(1, logPathSize); + sf_strcpy(clientConfig.logPath, logPathSize, logPath); } return true; } } - CXX_LOG_ERROR("Malformed client config file: %s", in_filePath.c_str()); + CXX_LOG_ERROR("Malformed client config file: %s", filePath.c_str()); return false; } //////////////////////////////////////////////////////////////////////////////// -sf_bool EasyLoggingConfigParser::checkIfValidPermissions(const std::string& in_filePath) +sf_bool EasyLoggingConfigParser::checkIfValidPermissions(const std::string& filePath) { - boost::filesystem::file_status fileStatus = boost::filesystem::status(in_filePath); + boost::filesystem::file_status fileStatus = boost::filesystem::status(filePath); boost::filesystem::perms permissions = fileStatus.permissions(); if (permissions & boost::filesystem::group_write || permissions & boost::filesystem::others_write) { CXX_LOG_ERROR("Error due to other users having permission to modify the config file: %s", - in_filePath.c_str()); + filePath.c_str()); return false; } return true; } //////////////////////////////////////////////////////////////////////////////// -void EasyLoggingConfigParser::checkUnknownEntries(value& in_config) +void EasyLoggingConfigParser::checkUnknownEntries(value& config) { - if (in_config.is()) + if (config.is()) { - const value::object& configObj = in_config.get(); + const value::object& configObj = config.get(); for (value::object::const_iterator i = configObj.begin(); i != configObj.end(); ++i) { std::string key = i->first; diff --git a/include/snowflake/ClientConfigParser.hpp b/include/snowflake/ClientConfigParser.hpp index b27fc56373..8b3d30c8de 100644 --- a/include/snowflake/ClientConfigParser.hpp +++ b/include/snowflake/ClientConfigParser.hpp @@ -34,25 +34,25 @@ namespace Client * name(sf_client_config.json) under user home directory 5. Searches for default config file * name(sf_client_config.json) under tmp directory * - * @param in_configFilePath The config file path passed in by the user. - * @param out_clientConfig The SFClientConfig object to be filled. + * @param configFilePath The config file path passed in by the user. + * @param clientConfig The SFClientConfig object to be filled. * * @return whether the client config was loaded properly */ sf_bool loadClientConfig( - const std::string& in_configFilePath, - client_config& out_clientConfig); + const std::string& configFilePath, + client_config& clientConfig); // Private ================================================================= private: /** * @brief Resolve the client config path. * - * @param in_configFilePath The config file path passed in by the user. + * @param configFilePath The config file path passed in by the user. * * @return The client config path */ - std::string resolveClientConfigPath(const std::string& in_configFilePath); + std::string resolveClientConfigPath(const std::string& configFilePath); /** * @brief Resolve home directory config path. @@ -64,30 +64,30 @@ namespace Client /** * @brief Parse JSON string. * - * @param in_filePath The filePath of the config file to parse. - * @param out_clientConfig The SFClientConfig object to be filled. + * @param filePath The filePath of the config file to parse. + * @param clientConfig The SFClientConfig object to be filled. * * @return whether parsing the client config file was successful. */ sf_bool parseConfigFile( - const std::string& in_filePath, - client_config& out_clientConfig); + const std::string& filePath, + client_config& clientConfig); /** * @ brief Check if other have permission to modify file * - * @param in_filePath The file path of the config file to check permissions. + * @param filePath The file path of the config file to check permissions. * * @return whether the file has valid permissions */ - sf_bool checkIfValidPermissions(const std::string& in_filePath); + sf_bool checkIfValidPermissions(const std::string& filePath); /** * @ brief Check if there are unknown entries in config file * - * @param in_jsonString The json object to check in json config file. + * @param jsonString The json object to check in json config file. */ - void checkUnknownEntries(picojson::value& in_config); + void checkUnknownEntries(picojson::value& config); /** * @ brief Get the path to the binary file diff --git a/include/snowflake/client_config_parser.h b/include/snowflake/client_config_parser.h index 861272b6c7..c3c851f8b7 100644 --- a/include/snowflake/client_config_parser.h +++ b/include/snowflake/client_config_parser.h @@ -30,14 +30,14 @@ extern "C" { * name(sf_client_config.json) under user home directory 5. Searches for default config file * name(sf_client_config.json) under tmp directory * - * @param in_configFilePath The config file path passed in by the user. - * @param out_clientConfig The client_config object to be filled. + * @param configFilePath The config file path passed in by the user. + * @param clientConfig The client_config object to be filled. * * @return true if successful */ sf_bool load_client_config( - const char* in_configFilePath, - client_config* out_clientConfig); + const char* configFilePath, + client_config* clientConfig); /** * Free client config memory diff --git a/tests/test_unit_logger.c b/tests/test_unit_logger.c index db31e7b400..b7bc223ad7 100644 --- a/tests/test_unit_logger.c +++ b/tests/test_unit_logger.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved. + * Copyright (c) 2018-2025 Snowflake Computing, Inc. All rights reserved. */