No protobuf to azure storage poc

cpp/arcticdb/CMakeLists.txt

@@ -455,6 +455,7 @@ set(arcticdb_srcs
         storage/config_resolvers.cpp
         storage/failure_simulation.cpp
         storage/library_manager.cpp
+        storage/azure/azure_settings.hpp
         storage/azure/azure_storage.cpp
         storage/azure/azure_real_client.cpp
         storage/azure/azure_mock_client.cpp

cpp/arcticdb/storage/azure/azure_client_wrapper.hpp

@@ -47,7 +47,6 @@ inline std::string AzureErrorCode_to_string(AzureErrorCode error) {
     // It can be derived as either a real connection to Azure or a mock used for unit tests.
 class AzureClientWrapper {
 public:
-    using Config = arcticdb::proto::azure_storage::Config;
     virtual void write_blob(
             const std::string& blob_name,
             Segment&& segment,

cpp/arcticdb/storage/azure/azure_real_client.cpp

@@ -27,10 +27,10 @@ Azure::Core::Context get_context(unsigned int request_timeout){
 }
 
 
-RealAzureClient::RealAzureClient(const Config &conf) :
+RealAzureClient::RealAzureClient(const AzureSettings &conf) :
 container_client(BlobContainerClient::CreateFromConnectionString(conf.endpoint(), conf.container_name(), get_client_options(conf))) { }
 
-Azure::Storage::Blobs::BlobClientOptions RealAzureClient::get_client_options(const Config &conf) {
+Azure::Storage::Blobs::BlobClientOptions RealAzureClient::get_client_options(const AzureSettings &conf) {
     BlobClientOptions client_options;
     if (!conf.ca_cert_path().empty() || !conf.ca_cert_dir().empty()) {//WARNING: Setting ca_cert_path or ca_cert_dir will force Azure sdk uses libcurl as backend support, instead of winhttp
         Azure::Core::Http::CurlTransportOptions curl_transport_options;

cpp/arcticdb/storage/azure/azure_real_client.hpp

@@ -13,16 +13,17 @@
 
 #include <arcticdb/storage/storage_utils.hpp>
 #include <arcticdb/storage/azure/azure_client_wrapper.hpp>
+#include <arcticdb/storage/azure/azure_settings.hpp>
 
 namespace arcticdb::storage::azure {
 class RealAzureClient : public AzureClientWrapper {
 private:
     Azure::Storage::Blobs::BlobContainerClient container_client;
 
-    static Azure::Storage::Blobs::BlobClientOptions get_client_options(const Config &conf);
+    static Azure::Storage::Blobs::BlobClientOptions get_client_options(const AzureSettings &conf);
 public:
 
-    explicit RealAzureClient(const Config &conf);
+    explicit RealAzureClient(const AzureSettings &conf);
 
     void write_blob(
             const std::string& blob_name,

cpp/arcticdb/storage/azure/azure_settings.hpp

@@ -0,0 +1,104 @@
+#pragma once
+
+#include <string>
+#include <arcticdb/entity/protobufs.hpp>
+
+class AzureSettings {
+    std::string container_name_;
+    std::string endpoint_;
+    std::string ca_cert_path_;
+    std::string ca_cert_dir_;
+    uint32_t max_connections_;
+    uint32_t request_timeout_;
+    std::string prefix_;
+    bool use_mock_storage_for_testing_;
+
+public:
+    AzureSettings() = default;
+    AzureSettings(const arcticc::pb2::azure_storage_pb2::Config& config) {
+        container_name_ = config.container_name();
+        endpoint_ = config.endpoint();
+        ca_cert_path_ = config.ca_cert_path();
+        ca_cert_dir_ = config.ca_cert_dir();
+        max_connections_ = config.max_connections();
+        request_timeout_ = config.request_timeout();
+        prefix_ = config.prefix();
+        use_mock_storage_for_testing_ = config.use_mock_storage_for_testing();
+    }
+    // export to protobuf
+    arcticc::pb2::azure_storage_pb2::Config to_protobuf() const {
+        arcticc::pb2::azure_storage_pb2::Config config;
+        config.set_container_name(container_name_);
+        config.set_endpoint(endpoint_);
+        config.set_ca_cert_path(ca_cert_path_);
+        config.set_ca_cert_dir(ca_cert_dir_);
+        config.set_max_connections(max_connections_);
+        config.set_request_timeout(request_timeout_);
+        config.set_prefix(prefix_);
+        config.set_use_mock_storage_for_testing(use_mock_storage_for_testing_);
+        return config;
+    }
+    std::string container_name() const {
+        return container_name_;
+    }
+
+    void set_container_name(std::string_view container_name) {
+        container_name_ = container_name;
+    }
+
+    std::string endpoint() const {
+        return endpoint_;
+    }
+
+    void set_endpoint(std::string_view endpoint) {
+        endpoint_ = endpoint;
+    }
+
+    std::string ca_cert_path() const {
+        return ca_cert_path_;
+    }
+
+    void set_ca_cert_path(std::string_view ca_cert_path){
+        ca_cert_path_ = ca_cert_path;
+    }
+
+    std::string ca_cert_dir() const {
+        return ca_cert_dir_;
+    }
+
+    void set_ca_cert_dir(std::string_view ca_cert_dir){
+        ca_cert_dir_ = ca_cert_dir;
+    }
+
+    uint32_t max_connections() const {
+        return max_connections_;
+    }
+
+    void set_max_connections(uint32_t max_connections){
+        max_connections_ = max_connections;
+    }
+
+    uint32_t request_timeout() const {
+        return request_timeout_;
+    }
+
+    void set_request_timeout(uint32_t request_timeout){
+        request_timeout_ = request_timeout;
+    }
+
+    std::string prefix() const {
+        return prefix_;
+    }
+
+    void set_prefix(std::string_view prefix){
+        prefix_ = prefix;
+    }
+
+    bool use_mock_storage_for_testing() const {
+        return use_mock_storage_for_testing_;
+    }
+
+    void set_use_mock_storage_for_testing(bool use_mock_storage_for_testing){
+        use_mock_storage_for_testing_ = use_mock_storage_for_testing;
+    }
+};

cpp/arcticdb/storage/azure/azure_storage.cpp

@@ -341,7 +341,7 @@ using namespace Azure::Storage;
 using namespace Azure::Storage::Blobs;
 
 
-AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const Config &conf) :
+AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const AzureSettings &conf) :
     Storage(library_path, mode),
     root_folder_(object_store_utils::get_root_folder(library_path)),
     container_name_(conf.container_name()),

cpp/arcticdb/storage/azure/azure_storage.hpp

@@ -11,11 +11,10 @@
 #include <arcticdb/storage/storage_factory.hpp>
 #include <arcticdb/storage/object_store_utils.hpp>
 #include <arcticdb/storage/azure/azure_client_wrapper.hpp>
+#include <arcticdb/storage/azure/azure_settings.hpp>
 #include <arcticdb/log/log.hpp>
-#include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/util/composite.hpp>
 #include <arcticdb/util/configs_map.hpp>
-#include <arcticdb/util/pb_util.hpp>
 #include <azure/core.hpp>
 #include <azure/storage/blobs.hpp>
 #include <cstdlib>
@@ -27,10 +26,7 @@ namespace arcticdb::storage::azure {
 
 class AzureStorage final : public Storage {
   public:
-    // friend class AzureTestClientAccessor<AzureStorage>;
-    using Config = arcticdb::proto::azure_storage::Config;
-
-    AzureStorage(const LibraryPath &lib, OpenMode mode, const Config &conf);
+    AzureStorage(const LibraryPath &lib, OpenMode mode, const AzureSettings &conf);
 
     std::string name() const final;
 
@@ -66,30 +62,4 @@ class AzureStorage final : public Storage {
     Azure::Storage::Blobs::UploadBlockBlobFromOptions upload_option_;
     Azure::Storage::Blobs::DownloadBlobToOptions download_option_;
 };
-
-inline arcticdb::proto::storage::VariantStorage pack_config(const std::string &container_name) {
-    arcticdb::proto::storage::VariantStorage output;
-    arcticdb::proto::azure_storage::Config cfg;
-    cfg.set_container_name(container_name);
-    util::pack_to_any(cfg, *output.mutable_config());
-    return output;
-}
-
-inline arcticdb::proto::storage::VariantStorage pack_config(
-        const std::string &container_name,
-        const std::string &endpoint
-        ) {
-    arcticdb::proto::storage::VariantStorage output;
-    arcticdb::proto::azure_storage::Config cfg;
-    cfg.set_container_name(container_name);
-    cfg.set_endpoint(endpoint);
-    util::pack_to_any(cfg, *output.mutable_config());
-    return output;
-}
-
-template<typename ConfigType>
-std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> get_azure_credentials(const ConfigType& conf) {
-    return std::make_shared<Azure::Storage::StorageSharedKeyCredential>(conf.credential_name(), conf.credential_key());
-}
-
 } //namespace arcticdb::azure

cpp/arcticdb/storage/python_bindings.cpp

@@ -20,6 +20,8 @@
 #include <arcticdb/storage/constants.hpp>
 #include <arcticdb/storage/s3/s3_storage.hpp>
 
+#include <arcticdb/storage/azure/azure_settings.hpp>
+
 namespace py = pybind11;
 
 namespace arcticdb::storage::apy {
@@ -153,6 +155,17 @@ void register_bindings(py::module& storage) {
         .def("set_azure_override", &StorageOverride::set_azure_override)
         .def("set_lmdb_override", &StorageOverride::set_lmdb_override);
 
+    py::class_<AzureSettings>(storage, "AzureSettings")
+        .def(py::init<>())
+        .def_property("container_name", &AzureSettings::container_name, &AzureSettings::set_container_name)
+        .def_property("endpoint", &AzureSettings::endpoint, &AzureSettings::set_endpoint)
+        .def_property("ca_cert_path", &AzureSettings::ca_cert_path, &AzureSettings::set_ca_cert_path)
+        .def_property("ca_cert_dir", &AzureSettings::ca_cert_dir, &AzureSettings::set_ca_cert_dir)
+        .def_property("max_connections", &AzureSettings::max_connections, &AzureSettings::set_max_connections)
+        .def_property("request_timeout", &AzureSettings::request_timeout, &AzureSettings::set_request_timeout)
+        .def_property("prefix", &AzureSettings::prefix, &AzureSettings::set_prefix)
+        .def_property("use_mock_storage_for_testing", &AzureSettings::use_mock_storage_for_testing, &AzureSettings::set_use_mock_storage_for_testing);
+
     py::class_<LibraryManager, std::shared_ptr<LibraryManager>>(storage, "LibraryManager")
         .def(py::init<std::shared_ptr<storage::Library>>())
         .def("write_library_config", [](const LibraryManager& library_manager, py::object& lib_cfg,

cpp/arcticdb/storage/storage_factory.cpp

@@ -45,10 +45,10 @@ std::shared_ptr<Storage> create_storage(
         nfs_backed::NfsBackedStorage::Config nfs_backed_config;
         storage_descriptor.config().UnpackTo(&nfs_backed_config);
         storage = std::make_shared<nfs_backed::NfsBackedStorage>(library_path, mode, nfs_backed_config);
-    } else if (type_name == azure::AzureStorage::Config::descriptor()->full_name()) {
-        azure::AzureStorage::Config azure_config;
+    } else if (type_name == arcticdb::proto::azure_storage::Config::descriptor()->full_name()) {
+        arcticdb::proto::azure_storage::Config azure_config;
         storage_descriptor.config().UnpackTo(&azure_config);
-        storage = std::make_shared<azure::AzureStorage >(library_path, mode, azure_config);
+        storage = std::make_shared<azure::AzureStorage>(library_path, mode, azure_config);
     } else if (type_name == file::MappedFileStorage::Config::descriptor()->full_name()) {
         file::MappedFileStorage::Config mapped_config;
         storage_descriptor.config().UnpackTo(&mapped_config);