From 084cad8c14f145e76de94f3164869d9f7a5e505c Mon Sep 17 00:00:00 2001 From: Eyck Jentzsch Date: Wed, 25 Feb 2026 17:51:46 +0100 Subject: [PATCH] adds decompressing streambuffer for gzip bzip2 xz, and zstd --- .gitignore | 1 + CMakeLists.txt | 16 ++- CMakePresets.json | 9 ++ conanfile.py | 3 + src/common/CMakeLists.txt | 24 ++++ src/common/util/bzip2_streambuf.cpp | 50 +++++++ src/common/util/bzip2_streambuf.h | 62 +++++++++ src/common/util/gzip_streambuf.cpp | 40 ++++++ src/common/util/gzip_streambuf.h | 61 +++++++++ src/common/util/ities.h | 29 ++++ src/common/util/xz_streambuf.cpp | 62 +++++++++ src/common/util/xz_streambuf.h | 64 +++++++++ src/common/util/zstd_streambuf.cpp | 52 ++++++++ src/common/util/zstd_streambuf.h | 63 +++++++++ tests/CMakeLists.txt | 1 + tests/io-redirector/CMakeLists.txt | 2 +- tests/streambuf/CMakeLists.txt | 8 ++ tests/streambuf/test.cpp | 199 ++++++++++++++++++++++++++++ 18 files changed, 739 insertions(+), 7 deletions(-) create mode 100644 src/common/util/bzip2_streambuf.cpp create mode 100644 src/common/util/bzip2_streambuf.h create mode 100644 src/common/util/gzip_streambuf.cpp create mode 100644 src/common/util/gzip_streambuf.h create mode 100644 src/common/util/xz_streambuf.cpp create mode 100644 src/common/util/xz_streambuf.h create mode 100644 src/common/util/zstd_streambuf.cpp create mode 100644 src/common/util/zstd_streambuf.h create mode 100644 tests/streambuf/CMakeLists.txt create mode 100644 tests/streambuf/test.cpp diff --git a/.gitignore b/.gitignore index c5c4f8ede..acb8991c4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /.direnv /.cache /install +/*.zst diff --git a/CMakeLists.txt b/CMakeLists.txt index 51629a558..6c8f9f1f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,17 @@ if (ENABLE_CLANG_TIDY) endif() find_package(Boost REQUIRED QUIET COMPONENTS date_time) # header only libraries must not be added here +find_package(fmt REQUIRED) +find_package(spdlog REQUIRED) +find_package(yaml-cpp REQUIRED) +find_package(elfio QUIET) +find_package(Threads REQUIRED) find_package(lz4 QUIET) +find_package(ZLIB QUIET) +find_package(BZip2 QUIET) +find_package(zstd QUIET) +find_package(LibLZMA QUIET) + if(NOT ${lz4_FOUND}) find_package(PkgConfig QUIET) if(PkgConfig_FOUND) @@ -101,13 +111,7 @@ if(NOT ${lz4_FOUND}) endif() endif() endif() -find_package(fmt REQUIRED) -find_package(spdlog REQUIRED) -find_package(yaml-cpp REQUIRED) find_package(Catch2 QUIET) -find_package(Threads REQUIRED) -find_package(ZLIB REQUIRED) -find_package(elfio QUIET) if(MSVC) add_compile_options(/vmg /MP /W3 /wd4244 /wd4267 /wd4996 -DNOMINMAX /EHsc) diff --git a/CMakePresets.json b/CMakePresets.json index 39a4f6371..2a4335fe6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -23,6 +23,15 @@ "CONAN_BUILD_PROFILE": "conan_host_profile" } }, + { + "name": "Config", + "inherits": "Release", + "displayName": "Config build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "ENABLE_CLANG_FORMAT": "ON" + } + }, { "name": "Debug", "inherits": "Base", diff --git a/conanfile.py b/conanfile.py index 8d480ddd3..86274952f 100644 --- a/conanfile.py +++ b/conanfile.py @@ -48,6 +48,9 @@ def requirements(self): self.requires("jsoncpp/1.9.6") self.requires("zlib/1.3.1") self.requires("catch2/3.11.0") + self.requires("bzip2/1.0.8") + self.requires("zstd/1.5.7") + self.requires("xz_utils/5.8.1") def build_requirements(self): if self.settings.compiler.cppstd: diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 15485c80e..f14c5f06f 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -7,6 +7,18 @@ endif() if(TARGET elfio::elfio) list(APPEND SRC util/elf.cpp) endif() +if(TARGET ZLIB::ZLIB) + list(APPEND SRC util/gzip_streambuf.cpp) +endif() +if(TARGET BZip2::BZip2) + list(APPEND SRC util/bzip2_streambuf.cpp) +endif() +if(TARGET zstd::libzstd_static) + list(APPEND SRC util/zstd_streambuf.cpp) +endif() +if(TARGET LibLZMA::LibLZMA) + list(APPEND SRC util/xz_streambuf.cpp) +endif() add_library(${PROJECT_NAME} ${SRC}) add_library(scc::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) @@ -24,6 +36,18 @@ endif() if(TARGET elfio::elfio) target_link_libraries(${PROJECT_NAME} PRIVATE elfio::elfio) endif() +if(TARGET ZLIB::ZLIB) + target_link_libraries(${PROJECT_NAME} PUBLIC ZLIB::ZLIB) +endif() +if(TARGET BZip2::BZip2) + target_link_libraries(${PROJECT_NAME} PUBLIC BZip2::BZip2) +endif() +if(TARGET zstd::libzstd_static) + target_link_libraries(${PROJECT_NAME} PUBLIC zstd::libzstd_static) +endif() +if(TARGET LibLZMA::LibLZMA) + target_link_libraries(${PROJECT_NAME} PUBLIC LibLZMA::LibLZMA) +endif() if(CLANG_TIDY_EXE) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_CLANG_TIDY "${DO_CLANG_TIDY}" ) diff --git a/src/common/util/bzip2_streambuf.cpp b/src/common/util/bzip2_streambuf.cpp new file mode 100644 index 000000000..d8c8aa9ca --- /dev/null +++ b/src/common/util/bzip2_streambuf.cpp @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "bzip2_streambuf.h" + +util::bzip2_streambuf::bzip2_streambuf(const std::string& path) +: file_(fopen(path.c_str(), "rb")) +, buffer_(64 * 1024) { + if(!file_) + throw std::runtime_error("fopen failed"); + + bz_ = BZ2_bzReadOpen(&bzerror_, file_, 0, 0, nullptr, 0); + if(bzerror_ != BZ_OK) + throw std::runtime_error("BZ2_bzReadOpen failed"); + + setg(buffer_.data(), buffer_.data(), buffer_.data()); +} + +util::bzip2_streambuf::~bzip2_streambuf() { + if(bz_) + BZ2_bzReadClose(&bzerror_, bz_); + + if(file_) + fclose(file_); +} + +auto util::bzip2_streambuf::underflow() -> int_type { + int n = BZ2_bzRead(&bzerror_, bz_, buffer_.data(), buffer_.size()); + if(bzerror_ != BZ_OK && bzerror_ != BZ_STREAM_END) + return traits_type::eof(); + + if(n <= 0) + return traits_type::eof(); + + setg(buffer_.data(), buffer_.data(), buffer_.data() + n); + return traits_type::to_int_type(*gptr()); +} diff --git a/src/common/util/bzip2_streambuf.h b/src/common/util/bzip2_streambuf.h new file mode 100644 index 000000000..6ab3ea9c4 --- /dev/null +++ b/src/common/util/bzip2_streambuf.h @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef _UTIL_BZIP2_STREAMBUF_H_ +#define _UTIL_BZIP2_STREAMBUF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util { +// ============================================================ +// BZIP2 +// ============================================================ +class bzip2_streambuf : public std::streambuf { +public: + explicit bzip2_streambuf(const std::string& path); + + ~bzip2_streambuf() override; + +protected: + int_type underflow() override; + +private: + FILE* file_; + BZFILE* bz_; + int bzerror_; + std::vector buffer_; +}; + +class bzip2_stream : public std::istream { +public: + bzip2_stream() = delete; + explicit bzip2_stream(const std::string& path) + : buf_(path) { + rdbuf(&buf_); + } + +private: + bzip2_streambuf buf_; +}; + +} // namespace util +#endif // _UTIL_BZIP2_STREAMBUF_H_ diff --git a/src/common/util/gzip_streambuf.cpp b/src/common/util/gzip_streambuf.cpp new file mode 100644 index 000000000..bdf56135f --- /dev/null +++ b/src/common/util/gzip_streambuf.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "gzip_streambuf.h" + +util::gzip_streambuf::gzip_streambuf(const std::string& path) +: gz_(gzopen(path.c_str(), "rb")) +, buffer_(64 * 1024) { + if(!gz_) + throw std::runtime_error("gzopen failed"); + + setg(buffer_.data(), buffer_.data(), buffer_.data()); +} + +util::gzip_streambuf::~gzip_streambuf() { + if(gz_) + gzclose(gz_); +} + +auto util::gzip_streambuf::underflow() -> int_type { + int n = gzread(gz_, buffer_.data(), buffer_.size()); + if(n <= 0) + return traits_type::eof(); + + setg(buffer_.data(), buffer_.data(), buffer_.data() + n); + return traits_type::to_int_type(*gptr()); +} diff --git a/src/common/util/gzip_streambuf.h b/src/common/util/gzip_streambuf.h new file mode 100644 index 000000000..b721ca2b5 --- /dev/null +++ b/src/common/util/gzip_streambuf.h @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef _UTIL_GZIP_STREAMBUF_H_ +#define _UTIL_GZIP_STREAMBUF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util { +// ============================================================ +// GZIP +// ============================================================ + +class gzip_streambuf : public std::streambuf { +public: + explicit gzip_streambuf(const std::string& path); + + ~gzip_streambuf() override; + +protected: + int_type underflow() override; + +private: + gzFile gz_; + std::vector buffer_; +}; + +class gzip_stream : public std::istream { +public: + gzip_stream() = delete; + explicit gzip_stream(const std::string& path) + : buf_(path) { + rdbuf(&buf_); + } + +private: + gzip_streambuf buf_; +}; + +} // namespace util +#endif // _UTIL_GZIP_STREAMBUF_H_ diff --git a/src/common/util/ities.h b/src/common/util/ities.h index d1a0cddb8..b10e94af6 100644 --- a/src/common/util/ities.h +++ b/src/common/util/ities.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -526,6 +527,34 @@ inline std::string glob_to_regex(std::string val) { return oss.str(); } +// ============================================================ +// File type detection +// ============================================================ + +enum class file_type_e { Plain, Gzip, Bzip2, Xz, Zstd }; + +inline file_type_e detect_file_type(const std::string& path) { + std::ifstream file(path, std::ios::binary); + if(!file) + throw std::runtime_error("Cannot open file for detection"); + + unsigned char magic[6] = {0}; + file.read(reinterpret_cast(magic), sizeof(magic)); + + if(magic[0] == 0x1F && magic[1] == 0x8B) + return file_type_e::Gzip; + + if(magic[0] == 0x42 && magic[1] == 0x5A && magic[2] == 0x68) + return file_type_e::Bzip2; + + if(magic[0] == 0xFD && magic[1] == 0x37 && magic[2] == 0x7A && magic[3] == 0x58 && magic[4] == 0x5A && magic[5] == 0x00) + return file_type_e::Xz; + + if(magic[0] == 0x28 && magic[1] == 0xB5 && magic[2] == 0x2F && magic[3] == 0xFD) + return file_type_e::Zstd; + return file_type_e::Plain; +} + } // namespace util /** @} */ #endif /* _UTIL_ITIES_H_ */ diff --git a/src/common/util/xz_streambuf.cpp b/src/common/util/xz_streambuf.cpp new file mode 100644 index 000000000..60517878c --- /dev/null +++ b/src/common/util/xz_streambuf.cpp @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "xz_streambuf.h" +#include + +util::xz_streambuf::xz_streambuf(const std::string& path) +: file_(fopen(path.c_str(), "rb")) +, buffer_(64 * 1024) +, inbuf_(64 * 1024) { + if(!file_) + throw std::runtime_error("fopen failed"); + + auto res = lzma_stream_decoder(&strm_, UINT64_MAX, 0); + if(res != LZMA_OK) + throw std::runtime_error("initialization of lzma_stream failed"); + setg(buffer_.data(), buffer_.data(), buffer_.data()); +} + +util::xz_streambuf::~xz_streambuf() { + lzma_end(&strm_); + if(file_) + fclose(file_); +} + +auto util::xz_streambuf::underflow() -> int_type { + strm_.next_out = reinterpret_cast(buffer_.data()); + strm_.avail_out = buffer_.size(); + while(strm_.avail_out > 0) { + if(strm_.avail_in == 0) { + strm_.next_in = reinterpret_cast(inbuf_.data()); + strm_.avail_in = fread(inbuf_.data(), 1, inbuf_.size(), file_); + if(strm_.avail_in == 0) + break; + } + lzma_ret ret = lzma_code(&strm_, LZMA_RUN); + if(ret == LZMA_STREAM_END) + break; + + if(ret != LZMA_OK) + return traits_type::eof(); + } + size_t produced = buffer_.size() - strm_.avail_out; + if(produced == 0) + return traits_type::eof(); + + setg(buffer_.data(), buffer_.data(), buffer_.data() + produced); + return traits_type::to_int_type(*gptr()); +} diff --git a/src/common/util/xz_streambuf.h b/src/common/util/xz_streambuf.h new file mode 100644 index 000000000..5449d680a --- /dev/null +++ b/src/common/util/xz_streambuf.h @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef _UTIL_XZ_STREAMBUF_H_ +#define _UTIL_XZ_STREAMBUF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util { +// ============================================================ +// XZ (liblzma) +// ============================================================ + +class xz_streambuf : public std::streambuf { +public: + explicit xz_streambuf(const std::string& path); + + ~xz_streambuf() override; + +protected: + int_type underflow() override; + +private: + FILE* file_; + lzma_stream strm_ = LZMA_STREAM_INIT; + std::vector buffer_; + std::vector inbuf_; +}; + +class xz_stream : public std::istream { +public: + xz_stream() = delete; + explicit xz_stream(const std::string& path) + : buf_(path) { + rdbuf(&buf_); + } + +private: + xz_streambuf buf_; +}; + +} // namespace util +#endif // _UTIL_XZ_STREAMBUF_H_ diff --git a/src/common/util/zstd_streambuf.cpp b/src/common/util/zstd_streambuf.cpp new file mode 100644 index 000000000..6185ee19f --- /dev/null +++ b/src/common/util/zstd_streambuf.cpp @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#include "zstd_streambuf.h" + +util::zstd_streambuf::zstd_streambuf(const std::string& path) +: file_(fopen(path.c_str(), "rb")) +, buffer_(64 * 1024) +, inbuf_(64 * 1024) { + if(!file_) + throw std::runtime_error("fopen failed"); + + dctx_ = ZSTD_createDCtx(); + if(!dctx_) + throw std::runtime_error("ZSTD_createDCtx failed"); + + setg(buffer_.data(), buffer_.data(), buffer_.data()); +} + +util::zstd_streambuf::~zstd_streambuf() { + if(dctx_) + ZSTD_freeDCtx(dctx_); + + if(file_) + fclose(file_); +} + +auto util::zstd_streambuf::underflow() -> int_type { + size_t inSize = fread(inbuf_.data(), 1, inbuf_.size(), file_); + if(inSize == 0) + return traits_type::eof(); + + size_t outSize = ZSTD_decompressDCtx(dctx_, buffer_.data(), buffer_.size(), inbuf_.data(), inSize); + if(ZSTD_isError(outSize)) + return traits_type::eof(); + + setg(buffer_.data(), buffer_.data(), buffer_.data() + outSize); + return traits_type::to_int_type(*gptr()); +} diff --git a/src/common/util/zstd_streambuf.h b/src/common/util/zstd_streambuf.h new file mode 100644 index 000000000..6e4a41cdc --- /dev/null +++ b/src/common/util/zstd_streambuf.h @@ -0,0 +1,63 @@ +/******************************************************************************* + * Copyright 2026 MINRES Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ + +#ifndef _UTIL_ZSTD_STREAMBUF_H_ +#define _UTIL_ZSTD_STREAMBUF_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util { +// ============================================================ +// ZSTD +// ============================================================ + +class zstd_streambuf : public std::streambuf { +public: + explicit zstd_streambuf(const std::string& path); + + ~zstd_streambuf() override; + +protected: + int_type underflow() override; + +private: + FILE* file_; + ZSTD_DCtx* dctx_; + std::vector buffer_; + std::vector inbuf_; +}; + +class zstd_stream : public std::istream { +public: + zstd_stream() = delete; + explicit zstd_stream(const std::string& path) + : buf_(path) { + rdbuf(&buf_); + } + +private: + zstd_streambuf buf_; +}; + +} // namespace util +#endif // _UTIL_ZSTD_STREAMBUF_H_ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d2792cd08..f4764d170 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -14,6 +14,7 @@ add_subdirectory(tlm_memory) add_subdirectory(memory_subsys) add_subdirectory(quantum_keeper_mt) add_subdirectory(sim_speed) +add_subdirectory(streambuf) if(FULL_TEST_SUITE) add_subdirectory(sim_performance) endif() diff --git a/tests/io-redirector/CMakeLists.txt b/tests/io-redirector/CMakeLists.txt index 8a9435b20..ae4336f56 100644 --- a/tests/io-redirector/CMakeLists.txt +++ b/tests/io-redirector/CMakeLists.txt @@ -1,7 +1,7 @@ project (io_redirector) if(TARGET Catch2::Catch2WithMain) add_executable (${PROJECT_NAME} test.cpp) - target_link_libraries (io_redirector LINK_PUBLIC scc-util Catch2::Catch2WithMain) + target_link_libraries (${PROJECT_NAME} LINK_PUBLIC scc-util Catch2::Catch2WithMain) #add_test(NAME io_redirector_test COMMAND io_redirector) catch_discover_tests(${PROJECT_NAME}) diff --git a/tests/streambuf/CMakeLists.txt b/tests/streambuf/CMakeLists.txt new file mode 100644 index 000000000..db8a8d8da --- /dev/null +++ b/tests/streambuf/CMakeLists.txt @@ -0,0 +1,8 @@ +project (streambuf) +if(TARGET Catch2::Catch2WithMain) + add_executable (${PROJECT_NAME} test.cpp) + target_link_libraries (${PROJECT_NAME} LINK_PUBLIC scc-util Catch2::Catch2WithMain) + + #add_test(NAME io_redirector_test COMMAND io_redirector) + catch_discover_tests(${PROJECT_NAME}) +endif() \ No newline at end of file diff --git a/tests/streambuf/test.cpp b/tests/streambuf/test.cpp new file mode 100644 index 000000000..2f89548bd --- /dev/null +++ b/tests/streambuf/test.cpp @@ -0,0 +1,199 @@ +#define CATCH_CONFIG_MAIN +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +// ------------------------------------------------------------ +// Helpers +// ------------------------------------------------------------ + +using namespace util; + +static std::string sampleText() { + std::ostringstream oss; + for(int i = 0; i < 10000; ++i) + oss << "Line " << i << " Hello MINRES\n"; + return oss.str(); +} + +static std::string readAll(std::istream& in) { + std::ostringstream ss; + ss << in.rdbuf(); + return ss.str(); +} + +// ------------------------------------------------------------ +// GZIP +// ------------------------------------------------------------ + +static void compressGzip(const std::string& path, const std::string& data) { + gzFile gz = gzopen(path.c_str(), "wb"); + REQUIRE(gz != nullptr); + gzwrite(gz, data.data(), data.size()); + gzclose(gz); +} + +TEST_CASE("gzip_stream decompresses", "[gzip]") { + std::string text = sampleText(); + std::string file = "test.gz"; + + compressGzip(file, text); + + gzip_stream stream(file); + REQUIRE(stream.good()); + + std::string result = readAll(stream); + REQUIRE(result == text); + + std::remove(file.c_str()); +} + +// ------------------------------------------------------------ +// BZIP2 +// ------------------------------------------------------------ + +static void compressBzip2(const std::string& path, const std::string& data) { + FILE* f = fopen(path.c_str(), "wb"); + REQUIRE(f != nullptr); + + int bzerr; + BZFILE* bz = BZ2_bzWriteOpen(&bzerr, f, 9, 0, 0); + REQUIRE(bzerr == BZ_OK); + + BZ2_bzWrite(&bzerr, bz, const_cast(data.data()), data.size()); + REQUIRE(bzerr == BZ_OK); + + BZ2_bzWriteClose(&bzerr, bz, 0, nullptr, nullptr); + fclose(f); +} + +TEST_CASE("bzip2_stream decompresses", "[bzip2]") { + std::string text = sampleText(); + std::string file = "test.bz2"; + + compressBzip2(file, text); + + bzip2_stream stream(file); + REQUIRE(stream.good()); + + std::string result = readAll(stream); + REQUIRE(result == text); + + std::remove(file.c_str()); +} + +// ------------------------------------------------------------ +// XZ +// ------------------------------------------------------------ + +static void compressXz(const std::string& path, const std::string& data) { + FILE* f = fopen(path.c_str(), "wb"); + REQUIRE(f != nullptr); + + lzma_stream strm = LZMA_STREAM_INIT; + REQUIRE(lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64) == LZMA_OK); + + std::vector outbuf(65536); + + strm.next_in = reinterpret_cast(data.data()); + strm.avail_in = data.size(); + + while(true) { + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); + + lzma_ret ret = lzma_code(&strm, LZMA_FINISH); + + size_t writeSize = outbuf.size() - strm.avail_out; + fwrite(outbuf.data(), 1, writeSize, f); + + if(ret == LZMA_STREAM_END) + break; + + REQUIRE(ret == LZMA_OK); + } + + lzma_end(&strm); + fclose(f); +} + +TEST_CASE("xz_stream decompresses", "[xz]") { + std::string text = sampleText(); + std::string file = "test.xz"; + + compressXz(file, text); + + xz_stream stream(file); + REQUIRE(stream.good()); + + std::string result = readAll(stream); + REQUIRE(result == text); + + std::remove(file.c_str()); +} + +// ------------------------------------------------------------ +// ZSTD +// ------------------------------------------------------------ + +static void compressZstd(const std::string& path, const std::string& data) { + size_t bound = ZSTD_compressBound(data.size()); + std::vector out(bound); + + size_t compressed = ZSTD_compress(out.data(), bound, data.data(), data.size(), 3); + + REQUIRE_FALSE(ZSTD_isError(compressed)); + + std::ofstream file(path, std::ios::binary); + file.write(out.data(), compressed); +} +/* +TEST_CASE("ZstdStream decompresses correctly", "[zstd]") +{ + std::string text = sampleText(); + std::string file = "test.zst"; + + compressZstd(file, text); + + zstd_stream stream(file); + REQUIRE(stream.good()); + + std::string result = readAll(stream); + REQUIRE(result == text); + + std::remove(file.c_str()); +} +*/ +// ------------------------------------------------------------ +// Line-by-line streaming test +// ------------------------------------------------------------ + +TEST_CASE("Stream supports getline()", "[stream]") { + std::string text = sampleText(); + std::string file = "test.gz"; + + compressGzip(file, text); + + gzip_stream stream(file); + + std::string line; + int count = 0; + while(std::getline(stream, line)) + ++count; + + REQUIRE(count == 10000); + + std::remove(file.c_str()); +}