Merge pull request #711 from PowerGridModel/feature/columnar-data-c-api

Feature / Columnar data C API

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp

@@ -302,20 +302,34 @@ template <dataset_type_tag dataset_type_> class Dataset {
         }
     }
 
-    void add_attribute_buffer(std::string_view component, std::string_view attribute, Data* data) {
-        Idx const idx = find_component(component, true);
-        Buffer& buffer = buffers_[idx];
-        if (!is_columnar(buffer)) {
-            throw DatasetError{"Cannot add attribute buffers to row-based dataset!\n"};
-        }
-        if (std::ranges::find_if(buffer.attributes, [&attribute](auto const& buffer_attribute) {
-                return buffer_attribute.meta_attribute->name == attribute;
-            }) != buffer.attributes.end()) {
-            throw DatasetError{"Cannot have duplicated attribute buffers!\n"};
-        }
-        AttributeBuffer<Data> attribute_buffer{
-            .data = data, .meta_attribute = &dataset_info_.component_info[idx].component->get_attribute(attribute)};
-        buffer.attributes.emplace_back(attribute_buffer);
+    void add_attribute_buffer(std::string_view component, std::string_view attribute, Data* data)
+        requires(!is_indptr_mutable_v<dataset_type>)
+    {
+        add_attribute_buffer_impl(component, attribute, data);
+    }
+
+    /*
+    we decided to go with the same behavior between `add_attribute_buffer` and `set_attribute_buffer` (but different
+    entrypoints). The behavior of `set_attribute_buffer` therefore differs from the one of `set_buffer`. The reasoning
+    is as follows:
+
+    For components:
+    - the deserializer tells the user via the dataset info that a certain component is present in the serialized
+    data.
+    - It is possible to efficiently determine whether that is the case.
+    - The user can then only call `set_buffer` for those components that are already present
+    For attributes:
+    - the deserializer would need to go over the entire dataset to look for components with the map serialization
+        representation to determine whether an attribute is present.
+    - this is expensive.
+    - the deserializer therefore cannot let the user know beforehand which attributes are present.
+    - `set_attribute_buffer` therefore should only be called if it has not been set yet.
+    - this is the same behavior as `add_attribute_buffer`.
+    */
+    void set_attribute_buffer(std::string_view component, std::string_view attribute, Data* data)
+        requires is_indptr_mutable_v<dataset_type>
+    {
+        add_attribute_buffer_impl(component, attribute, data);
     }
 
     // get buffer by component type
@@ -442,6 +456,22 @@ template <dataset_type_tag dataset_type_> class Dataset {
         buffers_.push_back(Buffer{});
     }
 
+    void add_attribute_buffer_impl(std::string_view component, std::string_view attribute, Data* data) {
+        Idx const idx = find_component(component, true);
+        Buffer& buffer = buffers_[idx];
+        if (!is_columnar(buffer)) {
+            throw DatasetError{"Cannot add attribute buffers to row-based dataset!\n"};
+        }
+        if (std::ranges::find_if(buffer.attributes, [&attribute](auto const& buffer_attribute) {
+                return buffer_attribute.meta_attribute->name == attribute;
+            }) != buffer.attributes.end()) {
+            throw DatasetError{"Cannot have duplicated attribute buffers!\n"};
+        }
+        AttributeBuffer<Data> const attribute_buffer{
+            .data = data, .meta_attribute = &dataset_info_.component_info[idx].component->get_attribute(attribute)};
+        buffer.attributes.emplace_back(attribute_buffer);
+    }
+
     template <class RangeType>
     RangeType get_span_impl(RangeType const& total_range, Idx scenario, Buffer const& buffer,
                             ComponentInfo const& info) const {

power_grid_model_c/power_grid_model/include/power_grid_model/main_model_impl.hpp

@@ -720,9 +720,7 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
             return true;
         }
 
-        auto const is_component_update_independent = [&update_data]<typename CT>() -> bool {
-            // get span of all the update data
-            auto const all_spans = update_data.get_buffer_span_all_scenarios<meta_data::update_getter_s, CT>();
+        auto const process_buffer_span = []<typename CT>(auto const& all_spans) -> bool {
             // Remember the first batch size, then loop over the remaining batches and check if they are of the same
             // length
             auto const elements_per_scenario = static_cast<Idx>(all_spans.front().size());
@@ -747,6 +745,16 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
             });
         };
 
+        auto const is_component_update_independent = [&update_data, &process_buffer_span]<typename CT>() -> bool {
+            // get span of all the update data
+            if (update_data.is_columnar(CT::name)) {
+                return process_buffer_span.template operator()<CT>(
+                    update_data.get_columnar_buffer_span_all_scenarios<meta_data::update_getter_s, CT>());
+            }
+            return process_buffer_span.template operator()<CT>(
+                update_data.get_buffer_span_all_scenarios<meta_data::update_getter_s, CT>());
+        };
+
         // check all components
         auto const update_independent = run_functor_with_all_types_return_array(is_component_update_independent);
         return std::ranges::all_of(update_independent, [](bool const is_independent) { return is_independent; });
@@ -828,7 +836,7 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
     void output_result(MathOutput<std::vector<SolverOutputType>> const& math_output, MutableDataset const& result_data,
                        Idx pos = 0) const {
         auto const output_func = [this, &math_output, &result_data, pos]<typename CT>() {
-            auto process_output = [this, &math_output](auto const& span) {
+            auto process_output_span = [this, &math_output](auto const& span) {
                 if (std::empty(span)) {
                     return;
                 }
@@ -838,11 +846,11 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
             if (result_data.is_columnar(CT::name)) {
                 auto const span =
                     result_data.get_columnar_buffer_span<typename output_type_getter<SolverOutputType>::type, CT>(pos);
-                process_output(span);
+                process_output_span(span);
             } else {
                 auto const span =
                     result_data.get_buffer_span<typename output_type_getter<SolverOutputType>::type, CT>(pos);
-                process_output(span);
+                process_output_span(span);
             }
         };
 

power_grid_model_c/power_grid_model/include/power_grid_model/sparse_ordering.hpp

@@ -120,11 +120,8 @@ inline std::vector<std::pair<IdxVector, IdxVector>> check_indistguishable(Idx co
 }
 
 inline bool in_graph(std::pair<Idx, Idx> const& e, std::map<Idx, IdxVector> const& d) {
-    if (auto edges_it = d.find(e.first);
-        edges_it != d.cend() && std::ranges::find(edges_it->second, e.second) != edges_it->second.cend()) {
-        return true;
-    }
-    return false;
+    auto edges_it = d.find(e.first);
+    return edges_it != d.cend() && std::ranges::find(edges_it->second, e.second) != edges_it->second.cend();
 }
 
 inline IdxVector remove_vertices_update_degrees(Idx const u, std::map<Idx, IdxVector>& d, DegreeLookup& dgd,

power_grid_model_c/power_grid_model_c/include/power_grid_model_c/dataset.h

@@ -140,13 +140,25 @@ PGM_API void PGM_destroy_dataset_const(PGM_ConstDataset* dataset);
  *     If the component is not uniform, indptr must point to an array of size (batch_size + 1).
  *         The values in the array must be not decreasing.
  *         And we must have indptr[0] = 0, indptr[batch_size] = total_elements.
- * @param data A void pointer to the buffer data.
+ * @param data A void pointer to the row based buffer data or a nullptr for columnar data.
  * @return
  */
 PGM_API void PGM_dataset_const_add_buffer(PGM_Handle* handle, PGM_ConstDataset* dataset, char const* component,
                                           PGM_Idx elements_per_scenario, PGM_Idx total_elements, PGM_Idx const* indptr,
                                           void const* data);
 
+/**
+ * @brief Add a attribute buffer to an instance of PGM_ConstDataset/component.
+ * @param handle
+ * @param dataset The pointer to the PGM_ConstDataset.
+ * @param component The name of the component.
+ * @param attribute The name of the attribute.
+ * @param data A void pointer to the buffer data.
+ * @return
+ */
+PGM_API void PGM_dataset_const_add_attribute_buffer(PGM_Handle* handle, PGM_ConstDataset* dataset,
+                                                    char const* component, char const* attribute, void const* data);
+
 /**
  * @brief Get the dataset info of the instance PGM_ConstDataset.
  * @param handle
@@ -173,12 +185,24 @@ PGM_API PGM_DatasetInfo const* PGM_dataset_writable_get_info(PGM_Handle* handle,
  * @param indptr A pointer to an array of indptr of a non-uniform component.
  *     If the component is uniform, indptr must be NULL.
  *     If the component is not uniform, indptr must point to an array of size (batch_size + 1).
- * @param data A void pointer to the buffer data.
+ * @param data A void pointer to the row based buffer data or a nullptr for columnar data.
  * @return
  */
 PGM_API void PGM_dataset_writable_set_buffer(PGM_Handle* handle, PGM_WritableDataset* dataset, char const* component,
                                              PGM_Idx* indptr, void* data);
 
+/**
+ * @brief Set buffer into the instance PGM_WritableDataset.
+ * @param handle
+ * @param dataset A pointer to the PGM_WritableDataset.
+ * @param component The name of the component.
+ * @param attribute The name of the attribute.
+ * @param data A void pointer to the buffer data.
+ * @return
+ */
+PGM_API void PGM_dataset_writable_set_attribute_buffer(PGM_Handle* handle, PGM_WritableDataset* dataset,
+                                                       char const* component, char const* attribute, void* data);
+
 /**
  * @brief Create an instance of PGM_MutableDataset.
  * @param handle
@@ -213,13 +237,25 @@ PGM_API void PGM_destroy_dataset_mutable(PGM_MutableDataset* dataset);
  *     If the component is not uniform, indptr must point to an array of size (batch_size + 1).
  *         The values in the array must be not decreasing.
  *         And we must have indptr[0] = 0, indptr[batch_size] = total_elements.
- * @param data A void pointer to the buffer data.
+ * @param data A void pointer to the row based buffer data or a nullptr for columnar data.
  * @return
  */
 PGM_API void PGM_dataset_mutable_add_buffer(PGM_Handle* handle, PGM_MutableDataset* dataset, char const* component,
                                             PGM_Idx elements_per_scenario, PGM_Idx total_elements,
                                             PGM_Idx const* indptr, void* data);
 
+/**
+ * @brief Add a attribute buffer to an instance of PGM_MutableDataset/component.
+ * @param handle
+ * @param dataset The pointer to the PGM_MutableDataset.
+ * @param component The name of the component.
+ * @param attribute The name of the attribute.
+ * @param data A void pointer to the buffer data.
+ * @return
+ */
+PGM_API void PGM_dataset_mutable_add_attribute_buffer(PGM_Handle* handle, PGM_MutableDataset* dataset,
+                                                      char const* component, char const* attribute, void* data);
+
 /**
  * @brief Get the dataset info of the instance PGM_MutableDataset.
  * @param handle

power_grid_model_c/power_grid_model_c/src/dataset.cpp

@@ -79,6 +79,13 @@ void PGM_dataset_const_add_buffer(PGM_Handle* handle, PGM_ConstDataset* dataset,
         PGM_regular_error);
 }
 
+void PGM_dataset_const_add_attribute_buffer(PGM_Handle* handle, PGM_ConstDataset* dataset, char const* component,
+                                            char const* attribute, void const* data) {
+    call_with_catch(
+        handle, [dataset, component, attribute, data]() { dataset->add_attribute_buffer(component, attribute, data); },
+        PGM_regular_error);
+}
+
 PGM_DatasetInfo const* PGM_dataset_const_get_info(PGM_Handle* /*unused*/, PGM_ConstDataset const* dataset) {
     return &dataset->get_description();
 }
@@ -96,6 +103,13 @@ void PGM_dataset_writable_set_buffer(PGM_Handle* handle, PGM_WritableDataset* da
         PGM_regular_error);
 }
 
+void PGM_dataset_writable_set_attribute_buffer(PGM_Handle* handle, PGM_WritableDataset* dataset, char const* component,
+                                               char const* attribute, void* data) {
+    call_with_catch(
+        handle, [dataset, component, attribute, data]() { dataset->set_attribute_buffer(component, attribute, data); },
+        PGM_regular_error);
+}
+
 // mutable dataset
 
 PGM_MutableDataset* PGM_create_dataset_mutable(PGM_Handle* handle, char const* dataset, PGM_Idx is_batch,
@@ -121,6 +135,13 @@ void PGM_dataset_mutable_add_buffer(PGM_Handle* handle, PGM_MutableDataset* data
         PGM_regular_error);
 }
 
+void PGM_dataset_mutable_add_attribute_buffer(PGM_Handle* handle, PGM_MutableDataset* dataset, char const* component,
+                                              char const* attribute, void* data) {
+    call_with_catch(
+        handle, [dataset, component, attribute, data]() { dataset->add_attribute_buffer(component, attribute, data); },
+        PGM_regular_error);
+}
+
 PGM_DatasetInfo const* PGM_dataset_mutable_get_info(PGM_Handle* /*unused*/, PGM_MutableDataset const* dataset) {
     return &dataset->get_description();
 }

power_grid_model_c/power_grid_model_cpp/include/power_grid_model_cpp/buffer.hpp

@@ -33,14 +33,14 @@ class Buffer {
         buffer.handle_.call_with(PGM_buffer_set_value, attribute, buffer.buffer_.get(), src_ptr, buffer_offset, size,
                                  src_stride);
     }
-    void set_value(MetaAttribute const* attribute, RawDataConstPtr src_ptr, Idx src_stride) const {
+    void set_value(MetaAttribute const* attribute, RawDataConstPtr src_ptr, Idx src_stride) {
         set_value(attribute, *this, src_ptr, 0, size_, src_stride);
     }
-    void set_value(MetaAttribute const* attribute, RawDataConstPtr src_ptr, Idx buffer_offset, Idx src_stride) const {
+    void set_value(MetaAttribute const* attribute, RawDataConstPtr src_ptr, Idx buffer_offset, Idx src_stride) {
         set_value(attribute, *this, src_ptr, buffer_offset, 1, src_stride);
     }
     void set_value(MetaAttribute const* attribute, RawDataConstPtr src_ptr, Idx buffer_offset, Idx size,
-                   Idx src_stride) const {
+                   Idx src_stride) {
         set_value(attribute, *this, src_ptr, buffer_offset, size, src_stride);
     }