adds decompressing streambuffer for gzip bzip2 xz, and zstd

.gitignore

@@ -10,3 +10,4 @@
 /.direnv
 /.cache
 /install
+/*.zst

CMakeLists.txt

@@ -91,7 +91,17 @@ if (ENABLE_CLANG_TIDY)
 endif()
 
 find_package(Boost REQUIRED QUIET COMPONENTS date_time) # header only libraries must not be added here
+find_package(fmt REQUIRED)
+find_package(spdlog REQUIRED)
+find_package(yaml-cpp REQUIRED)
+find_package(elfio QUIET)
+find_package(Threads REQUIRED)
 find_package(lz4 QUIET)
+find_package(ZLIB QUIET)
+find_package(BZip2 QUIET)
+find_package(zstd QUIET)
+find_package(LibLZMA QUIET)
+
 if(NOT ${lz4_FOUND})
     find_package(PkgConfig QUIET)
     if(PkgConfig_FOUND)
@@ -101,13 +111,7 @@ if(NOT ${lz4_FOUND})
         endif()
     endif()
 endif()
-find_package(fmt REQUIRED)
-find_package(spdlog REQUIRED)
-find_package(yaml-cpp REQUIRED)
 find_package(Catch2 QUIET)
-find_package(Threads REQUIRED)
-find_package(ZLIB REQUIRED)
-find_package(elfio QUIET)
 
 if(MSVC)
     add_compile_options(/vmg /MP /W3 /wd4244 /wd4267 /wd4996  -DNOMINMAX /EHsc)

CMakePresets.json

@@ -23,6 +23,15 @@
                 "CONAN_BUILD_PROFILE": "conan_host_profile"
             }
         },
+        {
+            "name": "Config",
+            "inherits": "Release",
+            "displayName": "Config build",
+            "cacheVariables": {
+				"CMAKE_BUILD_TYPE": "Debug",
+               "ENABLE_CLANG_FORMAT": "ON"
+            }
+        },
         {
             "name": "Debug",
             "inherits": "Base",

conanfile.py

@@ -48,6 +48,9 @@ def requirements(self):
         self.requires("jsoncpp/1.9.6")
         self.requires("zlib/1.3.1")
         self.requires("catch2/3.11.0")
+        self.requires("bzip2/1.0.8")
+        self.requires("zstd/1.5.7")
+        self.requires("xz_utils/5.8.1")
 
     def build_requirements(self):
         if self.settings.compiler.cppstd:

src/common/CMakeLists.txt

@@ -7,6 +7,18 @@ endif()
 if(TARGET elfio::elfio)
     list(APPEND SRC util/elf.cpp)
 endif()
+if(TARGET ZLIB::ZLIB)
+    list(APPEND SRC util/gzip_streambuf.cpp)
+endif()
+if(TARGET BZip2::BZip2)
+    list(APPEND SRC util/bzip2_streambuf.cpp)
+endif()
+if(TARGET zstd::libzstd_static)
+    list(APPEND SRC util/zstd_streambuf.cpp)
+endif()
+if(TARGET LibLZMA::LibLZMA)
+    list(APPEND SRC util/xz_streambuf.cpp)
+endif()
 add_library(${PROJECT_NAME} ${SRC})
 add_library(scc::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
 
@@ -24,6 +36,18 @@ endif()
 if(TARGET elfio::elfio)
     target_link_libraries(${PROJECT_NAME} PRIVATE elfio::elfio)
 endif()
+if(TARGET ZLIB::ZLIB)
+    target_link_libraries(${PROJECT_NAME} PUBLIC ZLIB::ZLIB)
+endif()
+if(TARGET BZip2::BZip2)
+    target_link_libraries(${PROJECT_NAME} PUBLIC BZip2::BZip2)
+endif()
+if(TARGET zstd::libzstd_static)
+    target_link_libraries(${PROJECT_NAME} PUBLIC zstd::libzstd_static)
+endif()
+if(TARGET LibLZMA::LibLZMA)
+    target_link_libraries(${PROJECT_NAME} PUBLIC LibLZMA::LibLZMA)
+endif()
 
 if(CLANG_TIDY_EXE)
     set_target_properties(${PROJECT_NAME} PROPERTIES CXX_CLANG_TIDY "${DO_CLANG_TIDY}" )

src/common/util/bzip2_streambuf.cpp

@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * Copyright 2026 MINRES Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *******************************************************************************/
+
+#include "bzip2_streambuf.h"
+
+util::bzip2_streambuf::bzip2_streambuf(const std::string& path)
+: file_(fopen(path.c_str(), "rb"))
+, buffer_(64 * 1024) {
+    if(!file_)
+        throw std::runtime_error("fopen failed");
+
+    bz_ = BZ2_bzReadOpen(&bzerror_, file_, 0, 0, nullptr, 0);
+    if(bzerror_ != BZ_OK)
+        throw std::runtime_error("BZ2_bzReadOpen failed");
+
+    setg(buffer_.data(), buffer_.data(), buffer_.data());
+}
+
+util::bzip2_streambuf::~bzip2_streambuf() {
+    if(bz_)
+        BZ2_bzReadClose(&bzerror_, bz_);
+
+    if(file_)
+        fclose(file_);
+}
+
+auto util::bzip2_streambuf::underflow() -> int_type {
+    int n = BZ2_bzRead(&bzerror_, bz_, buffer_.data(), buffer_.size());
+    if(bzerror_ != BZ_OK && bzerror_ != BZ_STREAM_END)
+        return traits_type::eof();
+
+    if(n <= 0)
+        return traits_type::eof();
+
+    setg(buffer_.data(), buffer_.data(), buffer_.data() + n);
+    return traits_type::to_int_type(*gptr());
+}

src/common/util/bzip2_streambuf.h

@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Copyright 2026 MINRES Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *******************************************************************************/
+
+#ifndef _UTIL_BZIP2_STREAMBUF_H_
+#define _UTIL_BZIP2_STREAMBUF_H_
+
+#include <bzlib.h>
+#include <cctype>
+#include <climits>
+#include <fstream>
+#include <stdexcept>
+#include <string>
+#include <sys/stat.h>
+#include <vector>
+
+namespace util {
+// ============================================================
+// BZIP2
+// ============================================================
+class bzip2_streambuf : public std::streambuf {
+public:
+    explicit bzip2_streambuf(const std::string& path);
+
+    ~bzip2_streambuf() override;
+
+protected:
+    int_type underflow() override;
+
+private:
+    FILE* file_;
+    BZFILE* bz_;
+    int bzerror_;
+    std::vector<char> buffer_;
+};
+
+class bzip2_stream : public std::istream {
+public:
+    bzip2_stream() = delete;
+    explicit bzip2_stream(const std::string& path)
+    : buf_(path) {
+        rdbuf(&buf_);
+    }
+
+private:
+    bzip2_streambuf buf_;
+};
+
+} // namespace util
+#endif // _UTIL_BZIP2_STREAMBUF_H_

src/common/util/gzip_streambuf.cpp

@@ -0,0 +1,40 @@
+/*******************************************************************************
+ * Copyright 2026 MINRES Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *******************************************************************************/
+
+#include "gzip_streambuf.h"
+
+util::gzip_streambuf::gzip_streambuf(const std::string& path)
+: gz_(gzopen(path.c_str(), "rb"))
+, buffer_(64 * 1024) {
+    if(!gz_)
+        throw std::runtime_error("gzopen failed");
+
+    setg(buffer_.data(), buffer_.data(), buffer_.data());
+}
+
+util::gzip_streambuf::~gzip_streambuf() {
+    if(gz_)
+        gzclose(gz_);
+}
+
+auto util::gzip_streambuf::underflow() -> int_type {
+    int n = gzread(gz_, buffer_.data(), buffer_.size());
+    if(n <= 0)
+        return traits_type::eof();
+
+    setg(buffer_.data(), buffer_.data(), buffer_.data() + n);
+    return traits_type::to_int_type(*gptr());
+}

src/common/util/gzip_streambuf.h

@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Copyright 2026 MINRES Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *******************************************************************************/
+
+#ifndef _UTIL_GZIP_STREAMBUF_H_
+#define _UTIL_GZIP_STREAMBUF_H_
+
+#include <cctype>
+#include <climits>
+#include <fstream>
+#include <stdexcept>
+#include <string>
+#include <sys/stat.h>
+#include <vector>
+#include <zlib.h>
+
+namespace util {
+// ============================================================
+// GZIP
+// ============================================================
+
+class gzip_streambuf : public std::streambuf {
+public:
+    explicit gzip_streambuf(const std::string& path);
+
+    ~gzip_streambuf() override;
+
+protected:
+    int_type underflow() override;
+
+private:
+    gzFile gz_;
+    std::vector<char> buffer_;
+};
+
+class gzip_stream : public std::istream {
+public:
+    gzip_stream() = delete;
+    explicit gzip_stream(const std::string& path)
+    : buf_(path) {
+        rdbuf(&buf_);
+    }
+
+private:
+    gzip_streambuf buf_;
+};
+
+} // namespace util
+#endif // _UTIL_GZIP_STREAMBUF_H_

src/common/util/ities.h

@@ -23,6 +23,7 @@
 #include <bitset>
 #include <cctype>
 #include <climits>
+#include <fstream>
 #include <iterator>
 #include <limits>
 #include <memory>
@@ -526,6 +527,34 @@ inline std::string glob_to_regex(std::string val) {
     return oss.str();
 }
 
+// ============================================================
+// File type detection
+// ============================================================
+
+enum class file_type_e { Plain, Gzip, Bzip2, Xz, Zstd };
+
+inline file_type_e detect_file_type(const std::string& path) {
+    std::ifstream file(path, std::ios::binary);
+    if(!file)
+        throw std::runtime_error("Cannot open file for detection");
+
+    unsigned char magic[6] = {0};
+    file.read(reinterpret_cast<char*>(magic), sizeof(magic));
+
+    if(magic[0] == 0x1F && magic[1] == 0x8B)
+        return file_type_e::Gzip;
+
+    if(magic[0] == 0x42 && magic[1] == 0x5A && magic[2] == 0x68)
+        return file_type_e::Bzip2;
+
+    if(magic[0] == 0xFD && magic[1] == 0x37 && magic[2] == 0x7A && magic[3] == 0x58 && magic[4] == 0x5A && magic[5] == 0x00)
+        return file_type_e::Xz;
+
+    if(magic[0] == 0x28 && magic[1] == 0xB5 && magic[2] == 0x2F && magic[3] == 0xFD)
+        return file_type_e::Zstd;
+    return file_type_e::Plain;
+}
+
 } // namespace util
 /** @} */
 #endif /* _UTIL_ITIES_H_ */