Replace boost memory-mapped files with own implementation in zstd Dec…

…ompressor.
y-scope · May 8, 2024 · 2bd4eaa · 2bd4eaa
1 parent ca9431b
commit 2bd4eaa
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 32 deletions.
diff --git a/components/core/src/clp/streaming_compression/zstd/Decompressor.cpp b/components/core/src/clp/streaming_compression/zstd/Decompressor.cpp
@@ -1,8 +1,7 @@
 #include "Decompressor.hpp"
 
 #include <algorithm>
-
-#include <boost/filesystem.hpp>
+#include <sys/mman.h>
 
 #include "../../Defs.h"
 #include "../../spdlog_with_specializations.hpp"
@@ -182,10 +181,8 @@ void Decompressor::open(FileReader& file_reader, size_t file_read_buffer_capacit
 void Decompressor::close() {
     switch (m_input_type) {
         case InputType::MemoryMappedCompressedFile:
-            if (m_memory_mapped_compressed_file.is_open()) {
-                // An existing file is memory mapped by the decompressor
-                m_memory_mapped_compressed_file.close();
-            }
+            munmap(m_mem_mapped_compressed_file_buffer, m_compressed_file_size);
+            ::close(m_compressed_file_fd);
             break;
         case InputType::File:
             m_file_read_buffer.reset();
@@ -209,40 +206,54 @@ ErrorCode Decompressor::open(std::string const& compressed_file_path) {
     }
     m_input_type = InputType::MemoryMappedCompressedFile;
 
-    // Create memory mapping for compressed_file_path, use boost read only
-    // memory mapped file
-    boost::system::error_code boost_error_code;
-    size_t compressed_file_size
-            = boost::filesystem::file_size(compressed_file_path, boost_error_code);
-    if (boost_error_code) {
+    // Create memory mapping for compressed_file_path
+    m_compressed_file_fd = ::open(compressed_file_path.c_str(), O_RDONLY);
+    if (-1 == m_compressed_file_fd) {
         SPDLOG_ERROR(
-                "streaming_compression::zstd::Decompressor: Unable to obtain file size for "
-                "'{}' - {}.",
+            "streaming_compression::zstd::Decompressor: Unable to open the compressed file with "
+            "path: {}, error: {}-{}",
+            compressed_file_path.c_str(),
+            errno,
+            strerror(errno)
+        );
+        return ErrorCode_errno;
+    }
+
+    m_compressed_file_size = lseek(m_compressed_file_fd, 0, SEEK_END);
+    if (-1 == m_compressed_file_size) {
+        SPDLOG_ERROR(
+                "streaming_compression::zstd::Decompressor: Unable to obtain file size with "
+                "path: {}, error: {}-{}",
                 compressed_file_path.c_str(),
-                boost_error_code.message().c_str()
+                errno,
+                strerror(errno)
         );
-        return ErrorCode_Failure;
+        return ErrorCode_errno;
     }
 
-    boost::iostreams::mapped_file_params memory_map_params;
-    memory_map_params.path = compressed_file_path;
-    memory_map_params.flags = boost::iostreams::mapped_file::readonly;
-    memory_map_params.length = compressed_file_size;
-    // Try to map it to the same memory location as previous memory mapped
-    // file
-    memory_map_params.hint = m_memory_mapped_compressed_file.data();
-    m_memory_mapped_compressed_file.open(memory_map_params);
-    if (!m_memory_mapped_compressed_file.is_open()) {
+    m_mem_mapped_compressed_file_buffer = static_cast<char *>(
+        mmap(
+            nullptr,
+            m_compressed_file_size,
+            PROT_READ,
+            MAP_PRIVATE,
+            m_compressed_file_fd,
+            0
+        )
+    );
+    if (MAP_FAILED == m_mem_mapped_compressed_file_buffer) {
         SPDLOG_ERROR(
-                "streaming_compression::zstd::Decompressor: Unable to memory map the "
-                "compressed file with path: {}",
-                compressed_file_path.c_str()
+                "streaming_compression::zstd::Decompressor: Unable to memory map the compressed "
+                "file with path: {}, error: {}-{}",
+                compressed_file_path.c_str(),
+                errno,
+                strerror(errno)
         );
-        return ErrorCode_Failure;
+        return ErrorCode_errno;
     }
 
     // Configure input stream
-    m_compressed_stream_block = {m_memory_mapped_compressed_file.data(), compressed_file_size, 0};
+    m_compressed_stream_block = {m_mem_mapped_compressed_file_buffer, m_compressed_file_size, 0};
 
     reset_stream();
 

diff --git a/components/core/src/clp/streaming_compression/zstd/Decompressor.hpp b/components/core/src/clp/streaming_compression/zstd/Decompressor.hpp
@@ -4,7 +4,6 @@
 #include <memory>
 #include <string>
 
-#include <boost/iostreams/device/mapped_file.hpp>
 #include <zstd.h>
 
 #include "../../FileReader.hpp"
@@ -125,7 +124,9 @@ class Decompressor : public ::clp::streaming_compression::Decompressor {
     // Compressed stream variables
     ZSTD_DStream* m_decompression_stream;
 
-    boost::iostreams::mapped_file_source m_memory_mapped_compressed_file;
+    int m_compressed_file_fd;
+    size_t m_compressed_file_size;
+    char* m_mem_mapped_compressed_file_buffer;
     FileReader* m_file_reader;
     size_t m_file_reader_initial_pos;
     std::unique_ptr<char[]> m_file_read_buffer;

diff --git a/components/core/tests/test-StreamingCompression.cpp b/components/core/tests/test-StreamingCompression.cpp
@@ -1,6 +1,7 @@
 #include <string>
 
 #include <boost/filesystem.hpp>
+#include <boost/iostreams/device/mapped_file.hpp>
 #include <Catch2/single_include/catch2/catch.hpp>
 #include <zstd.h>