Submission to polydbms

.gitignore

@@ -0,0 +1 @@
+build/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "makefile.configureOnOpen": false
+}

CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.21)
+
+project(xdbc VERSION 0.1 DESCRIPTION "xdbc load")
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -g")
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+find_package(spdlog REQUIRED)
+find_package(ZLIB REQUIRED)
+find_package(FastPFOR REQUIRED)
+find_package(FPZIP REQUIRED)
+find_library(FPZIP_LIBRARY NAMES fpzip)
+
+#TODO: fix hardcoded paths
+set(ZSTD_LIBRARY_PATH "/usr/lib/x86_64-linux-gnu/libzstd.so")
+set(SNAPPY_LIBRARY_PATH "/usr/lib/x86_64-linux-gnu/libsnappy.so")
+
+include_directories("/zfp/include")
+link_directories(/zfp/lib)
+
+include(GNUInstallDirs)
+
+include(FetchContent)
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
+)
+# For Windows: Prevent overriding the parent project's compiler/linker settings
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(googletest)
+
+enable_testing()
+
+add_subdirectory(client)
+add_subdirectory(server)

Dockerfile

@@ -0,0 +1,69 @@
+# postgres server 14 on ubuntu 22.04 image
+FROM ubuntu:jammy
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+RUN apt-get upgrade -qy
+
+#-------------------------------------------- Install XDBC and prerequisites -------------------------------------------
+# install arrow/parquet dependencies
+
+RUN apt install -qy ca-certificates lsb-release wget pip
+
+RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+
+RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+
+RUN apt update && apt install -qy cmake git gdb nlohmann-json3-dev clang libboost-all-dev build-essential libspdlog-dev iproute2 netcat libarrow-dev=18.1.0-1 libparquet-dev=18.1.0-1 libthrift-dev pkg-config
+
+# install compression libs
+
+RUN apt install -qy libzstd-dev liblzo2-dev liblz4-dev libsnappy-dev libbrotli-dev
+
+RUN git clone https://github.com/lemire/FastPFor.git && cd FastPFor && git checkout 039134b && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    cmake --build . --config Release && \
+    make install
+
+RUN ln -s /FastPFor /fastpfor
+
+RUN git clone https://github.com/LLNL/fpzip.git && cd fpzip && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    cmake --build . --config Release && \
+    make install
+
+
+# install postgres dependencies
+RUN apt install -qy libpq-dev libpqxx-dev
+
+# install clickhouse depencencies
+RUN apt-get install -y libabsl-dev
+RUN git clone https://github.com/google/cityhash.git
+RUN cd /cityhash && ./configure && make all check CXXFLAGS="-g -O3" && make install
+
+# install clickhouse-lib
+RUN git clone https://github.com/ClickHouse/clickhouse-cpp.git
+RUN cd /clickhouse-cpp && rm -rf build && mkdir build && cd build && cmake .. -DWITH_SYSTEM_ABSEIL=ON && make -j8 && make install
+
+# install webserver for http experiments
+RUN pip install rangehttpserver
+
+#------------------------------------------------------------------------
+
+# Copy the entire project context
+RUN mkdir /xdbc
+COPY . /xdbc/
+
+# build xdbc
+RUN rm -rf /xdbc/build && mkdir -p /xdbc/build && cd /xdbc/build && cmake .. -D CMAKE_BUILD_TYPE=Release && make -j8 && make install
+
+RUN rm -rf /xdbc/build/client/Sinks/build && mkdir -p /xdbc/build/client/Sinks/build && cd /xdbc/build/client/Sinks/build && cmake /xdbc/client/Sinks -D CMAKE_BUILD_TYPE=Release && make -j8
+
+RUN ldconfig
+
+ENTRYPOINT ["tail", "-f", "/dev/null"]

Makefile

@@ -0,0 +1,2 @@
+build_XDBC:
+	docker build -f Dockerfile -t xdbc-unified:latest .

README.md

@@ -1,17 +1,49 @@
-# XDBC
+# XDBC Monorepo :  `git filter-repo`
 
-- [XDBC](https://dl.acm.org/doi/10.1145/3725294) is a holistic, high-performance framework for fast and scalable data transfers across heterogeneous data systems (e.g. DBMS to dataframes) aiming to combine the generality of generic solutions with performance of specialized connectors
-- It decomposes data transfer into a configurable pipeline (read -> deserialize -> compress -> send/receive -> decompress -> serialize -> write) with pipeline-parallel execution and ring-buffer memory manager for low resource overhead.
-- The core of the framework (xdbc-client and xdbc-server) are written in C++ with bindings available for Python and Spark. It includes built-in adapters to connect to PostgreSQL, CSV, Parquet and Pandas.
-- The project includes a lightweight heuristic optimizer implemented in Python that automatically tunes the parallelism, buffer sizes, intermediate formats and compression algorithms to the current environment.
+This monorepo merges `xdbc-client` and `xdbc-server` (branch: `test/reproduce`) using **`git filter-repo --to-subdirectory-filter`**.
 
+## Structure
 
-## Project Structure
+```
+XDBC-filter/
+  client/   ← xdbc-client source
+  server/   ← xdbc-server source
+```
+To access the history of client (or server) use git log as given below
+```bash
+git log -- client/xdbc/xclient.cpp
+```
 
-XDBC consists of multiple repositories covering the cross-system functionality. For the reproducibility experiments the following repositories will be cloned and used :
+## Running the Project
 
-- [`xdbc-client`](https://github.com/polydbms/xdbc-client) Client-side module, for loading data into the target system.
-- [`xdbc-server`](https://github.com/polydbms/xdbc-server) Server-side module, for extracting the data from the source system.
-- [`xdbc-python`](https://github.com/polydbms/xdbc-python) Python bindings for loading data into Pandas (through pybind).
-- [`xdbc-spark`](https://github.com/polydbms/xdbc-spark) Spark bindings, for loading data into a Spark RDD (through a custom DataSource with JNI).
-- [`pg_xdbc_fdw`](https://github.com/polydbms/pg_xdbc_fdw) PostgreSQL Foreign Data Wrapper, for loading data into a table.
+To build the combined image and spin up both the **client** and **server** containers :
+
+```bash
+# 1. Build the unified image
+make
+
+# 2. Start the infrastructure
+docker compose up -d
+```
+
+This will create two containers (`xdbcserver` and `xdbcclient`) using the same `xdbc-unified:latest` image, mapping their shared `/dev/shm` volumes correctly.
+
+Before running, download and extract the required dataset to `/dev/shm`:
+
+```bash
+# Download the ss13husallm dataset (~250 MB compressed, ~1.2 GB extracted)
+wget -O ss13husallm.csv.tar.gz "https://tubcloud.tu-berlin.de/s/M3aeptL8R5ekWSD/download?path=%2F&files=ss13husallm.csv.tar.gz"
+
+# Extract to /dev/shm (shared memory, accessible inside containers)
+tar --overwrite -xzf ss13husallm.csv.tar.gz -C /dev/shm
+```
+
+You can then run commands inside them:
+
+```bash
+# Start the server
+docker exec -it xdbcserver bash -c "./xdbc-server/build/xdbc-server"
+
+# Run a client command
+docker exec -it xdbcclient bash -c "/xdbc-client/Sinks/build/xdbcsinks --server-host=xdbcserver --table ss13husallm -f1 -b 1024 -p 32000 -n1 -w1 -d1 -s1 --skip-serializer=0 --target=csv"
+```

README_development.md

@@ -0,0 +1,125 @@
+# XDBC Development Guide: Testing Framework
+
+This document outlines the testing strategies used in the XDBC monorepo. It explains the differences between our end-to-end (E2E) Docker-based testing and our component-level Unit Testing framework, and how to use both effectively during development.
+
+---
+
+### Framework Used
+We use [Google Test (gtest)](https://google.github.io/googletest/). This is automatically downloaded and configured when you run CMake on the host machine. You do not need to install anything manually aside from a C++ compiler and CMake.
+
+### Test Executables
+
+#### Server Tests (`server/tests/`)
+
+| Executable | Source File | What It Tests |
+|---|---|---|
+| `xdbc-server-tests` | `test_queue.cpp` | customQueue: FIFO ordering, capacity blocking, concurrency (multi-producer/multi-consumer), sentinel termination pattern |
+| `xdbc-compressor-tests` | `test_compressor.cpp` | All 5 compression algorithms (zstd, snappy, lzo, lz4, zlib): compress, round-trip integrity, edge cases, integer column data |
+| `xdbc-deserializer-tests` | `test_deserializers.cpp` | CSV deserializer templates: int, double, char, fixed-size string parsing, full CSV row deserialization |
+| `xdbc-datasource-tests` | `test_datasource.cpp` | DataSource utilities: JSON schema parsing (`createSchemaFromJsonString`), `getSchemaSize`, `createSchemaAttribute` |
+
+#### Client Tests (`client/Sinks/tests/`)
+
+| Executable | Source File | What It Tests |
+|---|---|---|
+| `xdbc-client-tests` | `test_sink.cpp` | CSVSink `SerializeAttribute` templates (int, double, char, fixed-string), full tuple serialization, `RuntimeEnv.calculateTupleSize`, `RuntimeEnv.toString` |
+| `xdbc-decompressor-tests` | `test_decompressor.cpp` | All 5 decompression algorithms, dispatch method routing, server-to-client round-trip integrity for text/integer/double data |
+| `xdbc-pqsink-tests` | `test_pqsink.cpp` | `CreateParquetSchema` for all column types (INT, DOUBLE, STRING, CHAR), mixed schemas, error handling for unsupported types |
+| `xdbc-utils-tests` | `test_utils.cpp` | `Utils::compute_checksum`, `boolVectorToString`, `boolVecAsStr`, `slStr` |
+
+### How to Run Unit Tests Locally
+on your host machine during active development.
+
+```bash
+# 1. Create a build directory
+mkdir build && cd build
+
+# 2. Run CMake (this downloads googletest automatically)
+cmake ..
+
+# 3. Build all test binaries
+make -j$(nproc)
+
+# 4. Run the full test suite via CTest
+ctest --output-on-failure
+```
+
+To build and run a specific test target:
+
+```bash
+# Build only the compressor tests
+make xdbc-compressor-tests
+
+# Run it directly
+./server/tests/xdbc-compressor-tests
+
+# Or run via ctest with a filter
+ctest -R Compressor --output-on-failure
+```
+
+### How to Write a New Unit Test
+
+1. Create a `.cpp` file in the appropriate `tests/` directory and use the `TEST()` macro:
+
+```cpp
+#include <gtest/gtest.h>
+#include "../your_component_header.h"
+
+TEST(YourComponentTest, ExpectedBehaviorName) {
+    int result = MyComponent::Add(2, 2);
+    EXPECT_EQ(result, 4); // Assertion
+}
+```
+
+2. Add your new file to the `CMakeLists.txt` inside that `tests/` directory. Either add it to an existing `add_executable` or create a new test target:
+
+```cmake
+add_executable(
+  your-new-tests
+  test_your_component.cpp
+)
+
+target_link_libraries(
+  your-new-tests
+  GTest::gtest_main
+  pthread
+  # ... any additional libraries your component needs
+)
+
+gtest_discover_tests(your-new-tests)
+```
+
+### Test Design Notes
+
+- **Compression/Decompression tests** avoid pulling in the full `xdbcserver.h` dependency chain by calling the compression libraries directly with the same logic as `Compressor.cpp`. This keeps tests fast and dependency-free.
+- **Round-trip tests** simulate the server-compress then client-decompress pipeline to verify data integrity across the network boundary.
+- **Concurrency tests** in `customQueue` use multiple threads and verify blocking behavior, FIFO ordering, and the sentinel-based termination pattern used throughout the XDBC pipeline.
+- **Serializer tests** validate that the templated `SerializeAttribute` functions produce correct CSV output for each data type.
+
+---
+
+## 2. End-to-End Testing (Docker)
+
+**What is it?**
+E2E testing spins up the entire XDBC Server, Postgres DB, and XDBC Client simultaneously to perform massive, real-world data transfers (like transferring the `ss13husallm` dataset).
+
+**Why use it?**
+- **Accuracy:** Guarantees that all micro-components connect and work reliably under realistic network and database loads.
+
+### How to Run E2E Tests via Docker
+
+Instead of testing isolation, you test the full pipeline.
+
+```bash
+# 1. Build the unified image
+make
+
+# 2. Start the infrastructure
+docker compose up -d
+
+# 3. Start the server daemon inside its container
+docker exec -it xdbcserver bash -c "./xdbc-server/build/xdbc-server"
+
+# 4. Run a transfer from the client container
+docker exec -it xdbcclient bash -c "/xdbc-client/Sinks/build/xdbcsinks --server-host=xdbcserver --table ss13husallm -f1 -b 1024 -p 32000 -n1 -w1 -d1 -s1 --skip-serializer=0 --target=csv"
+```

client/.dockerignore

@@ -0,0 +1,5 @@
+**/cmake-build-debug
+**/cmake-build-release
+**/build
+**/.git
+**/.idea

client/.gitignore

@@ -0,0 +1,14 @@
+build/*
+cmake-build-*
+.idea/*
+experiments/local_measurements/*
+__pycache__/
+**/plots/.ipynb_checkpoints/
+**/plots/myvenv
+**/experiments/plots/*.pdf
+**/experiments/plots/paper_plots/*.pdf
+**/experiments/plots/current_analysis/*.csv*
+**/plots/*.csv
+experiments/experiment_scheduler/measurements/*
+experiments/feature_extraction_xdbc/xdbc_experiments_master.csv
+experiments/feature_extraction_xdbc/.idea/*

client/CMakeLists.txt

@@ -0,0 +1,22 @@
+# Client-only local configurations, dependencies have been moved to root
+
+message(STATUS "Configuring xdbc-client")
+
+add_library(xdbc-client-lib SHARED
+        xdbc/xclient.cpp
+        xdbc/Decompression/Decompressor.cpp xdbc/Decompression/Decompressor.h)
+
+set_target_properties(xdbc-client-lib PROPERTIES
+        #VERSION ${PROJECT_VERSION}
+        #SOVERSION 1
+        PUBLIC_HEADER "xdbc/xclient.h;xdbc/SinkInterface.h")
+
+target_include_directories(xdbc-client-lib PUBLIC .)
+target_link_libraries(xdbc-client-lib PRIVATE ${ZSTD_LIBRARY_PATH} ${SNAPPY_LIBRARY_PATH} Threads::Threads lzo2 lz4 spdlog::spdlog_header_only ZLIB::ZLIB FastPFOR::FastPFOR ${FPZIP_LIBRARY})
+install(TARGETS xdbc-client-lib
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+install(FILES xdbc/customQueue.h xdbc/utils.h xdbc/metrics_calculator.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+add_subdirectory(Sinks)