Add async raw S3 packet writes

AGENTS.md

@@ -20,6 +20,7 @@ CMake options (full table in `docs/getting-started.md`):
 - `DAQIRI_BUILD_EXAMPLES` — builds the benchmark executables (default `ON`).
 - `DAQIRI_ENABLE_OTEL_METRICS` — enables OpenTelemetry metrics instrumentation (default `OFF`).
 - `DAQIRI_REORDER_GPU_PROFILE` — enable CUDA event timing in the DPDK reorder kernels (off by default).
+- `DAQIRI_ENABLE_S3` — enable AWS SDK-backed asynchronous raw packet writes to S3 (off by default).
 
 CUDA architectures default to `80;90` (A100, H100), with `121` (GB10) added when configuring with CUDA Toolkit 13.0 or newer. Override `CMAKE_CUDA_ARCHITECTURES` when targeting other GPUs.
 

CMakeLists.txt

@@ -126,6 +126,9 @@ set(DAQIRI_PC_LIBS_PRIVATE "-lcudart -lcuda")
 if(CUDAToolkit_LIBRARY_DIR)
   string(PREPEND DAQIRI_PC_LIBS_PRIVATE "-L${CUDAToolkit_LIBRARY_DIR} ")
 endif()
+if(DAQIRI_ENABLE_S3)
+  string(APPEND DAQIRI_PC_LIBS_PRIVATE " -laws-cpp-sdk-s3 -laws-cpp-sdk-core")
+endif()
 
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/cmake/daqiri.pc.in

Dockerfile

@@ -18,8 +18,10 @@
 ARG DAQIRI_BASE_TARGET=dpdk
 ARG DAQIRI_MGR="dpdk socket"
 ARG DAQIRI_BUILD_PYTHON=OFF
+ARG DAQIRI_ENABLE_S3=OFF
 ARG BUILD_SHARED_LIBS=ON
 ARG DAQIRI_ENABLE_OTEL_METRICS=OFF
+ARG AWS_SDK_CPP_VERSION=1.11.822
 ARG DAQIRI_OS_BASE_IMAGE=nvcr.io/nvidia/cuda:13.1.0-devel-ubuntu24.04
 
 # ============================================================
@@ -63,6 +65,7 @@ ARG TARGETARCH
 ARG CACHEBUST=1
 ARG DEBIAN_FRONTEND=noninteractive
 ARG DOCA_VERSION=3.2.1
+ARG AWS_SDK_CPP_VERSION
 
 WORKDIR /opt
 
@@ -100,8 +103,30 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         ibverbs-utils \
         python3-dev \
         pybind11-dev \
+        libcurl4-openssl-dev \
+        libssl-dev \
+        uuid-dev \
+        zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
+# Build only the AWS SDK for C++ S3 component. DAQIRI links this SDK only when
+# configured with -DDAQIRI_ENABLE_S3=ON, but installing it here keeps the
+# recommended container build path self-contained.
+RUN git clone --depth 1 --branch "${AWS_SDK_CPP_VERSION}" \
+        https://github.com/aws/aws-sdk-cpp.git /tmp/aws-sdk-cpp \
+    && cmake -S /tmp/aws-sdk-cpp -B /tmp/aws-sdk-cpp/build \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=/usr/local \
+        -DBUILD_ONLY=s3 \
+        -DBUILD_SHARED_LIBS=ON \
+        -DENABLE_TESTING=OFF \
+        -DAUTORUN_UNIT_TESTS=OFF \
+        -DCUSTOM_MEMORY_MANAGEMENT=OFF \
+    && cmake --build /tmp/aws-sdk-cpp/build -j "$(nproc)" \
+    && cmake --install /tmp/aws-sdk-cpp/build \
+    && ldconfig \
+    && rm -rf /tmp/aws-sdk-cpp
+
 # PIP installs
 # - pytest: test harness
 # - pyyaml: to parse yaml configs in tests
@@ -277,6 +302,7 @@ FROM ${DAQIRI_BASE_TARGET} AS daqiri-build
 
 ARG DAQIRI_MGR
 ARG DAQIRI_BUILD_PYTHON
+ARG DAQIRI_ENABLE_S3
 ARG BUILD_SHARED_LIBS
 ARG DAQIRI_ENABLE_OTEL_METRICS
 
@@ -291,6 +317,7 @@ RUN cmake -S . -B build \
       -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} \
       -DDAQIRI_BUILD_PYTHON=${DAQIRI_BUILD_PYTHON} \
       -DDAQIRI_ENABLE_OTEL_METRICS=${DAQIRI_ENABLE_OTEL_METRICS} \
+      -DDAQIRI_ENABLE_S3=${DAQIRI_ENABLE_S3} \
       -DDAQIRI_MGR="${DAQIRI_MGR}" \
     && cmake --build build -j "$(nproc)" \
     && cmake --install build

README.md

@@ -23,6 +23,8 @@ DAQIRI provides direct NIC hardware access in userspace, bypassing the Linux ker
   - *Batched GPU*: Entire packets to GPU memory (maximum bandwidth, GPU-side parsing required).
 - **Burst file writes** — Write received bursts as raw packet files or appendable PCAP
   captures. Host-backed buffers use POSIX writes; CUDA device-backed buffers can use cuFile/GDS.
+- **S3 raw object writes** — Optionally upload raw burst packets to Amazon S3 or an
+  S3-compatible object store through the AWS SDK for C++.
 - **Flow Steering** — Configure the NIC's hardware flow engine to route packets by UDP
   source/destination port.
 - **RDMA** — RDMA verbs (READ, WRITE, SEND) over RoCE on Ethernet NICs or InfiniBand.
@@ -67,10 +69,16 @@ and `libcufile` in the build environment. At runtime, regular GDS writes through
 NVIDIA's `nvidia-fs` path require the `nvidia-fs` kernel module to be loaded and the
 target storage stack to be reported as supported by `gdscheck.py -p`.
 
+Enable raw packet uploads to S3 with `-DDAQIRI_ENABLE_S3=ON`. The recommended
+container build installs AWS SDK for C++ with S3 support; bare-metal builds need
+`aws-cpp-sdk-core` and `aws-cpp-sdk-s3` discoverable by CMake. S3 credentials are
+resolved through the AWS SDK provider chain.
+
 Container build:
 
 ```bash
 BASE_TARGET=dpdk DAQIRI_MGR="dpdk socket rdma" scripts/build-container.sh
+DAQIRI_ENABLE_S3=ON DAQIRI_BUILD_PYTHON=ON BASE_TARGET=dpdk scripts/build-container.sh
 ```
 
 OpenTelemetry metrics are opt-in. Build with `-DDAQIRI_ENABLE_OTEL_METRICS=ON`

cmake/daqiriConfig.cmake.in

@@ -6,5 +6,9 @@ set(DAQIRI_ENABLE_OTEL_METRICS @DAQIRI_ENABLE_OTEL_METRICS@)
 if(DAQIRI_ENABLE_OTEL_METRICS)
   find_dependency(opentelemetry-cpp CONFIG COMPONENTS api)
 endif()
+set(DAQIRI_ENABLE_S3 @DAQIRI_ENABLE_S3@)
+if(DAQIRI_ENABLE_S3)
+  find_dependency(AWSSDK COMPONENTS s3)
+endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/daqiriTargets.cmake")

docs/api-reference/cpp.md

@@ -329,6 +329,69 @@ writes raw or pcap output with the synchronous API, the asynchronous API, or bot
 to send real Ethernet/IPv4/UDP frames out of a NIC and receive them back through a
 hardware RX port.
 
+### Writing Raw Packets to S3
+
+Build with `DAQIRI_ENABLE_S3=ON` to upload raw packet objects through AWS SDK
+for C++. This path uses normal S3 `PutObject` requests, so it can target Amazon
+S3 or an S3-compatible service. It is not a cuObject/RDMA path.
+
+Before creating the writer, choose a bucket and region, configure AWS
+credentials through the SDK provider chain, grant `s3:PutObject` on the target
+prefix, and make sure the host can reach the S3 endpoint. For S3-compatible
+stores, set `endpoint_override` and `path_style` if that service requires them.
+
+```cpp
+daqiri::S3WriterConfig cfg;
+cfg.bucket = "daqiri-captures";
+cfg.region = "us-west-2";
+cfg.max_inflight_uploads = 8;
+
+daqiri::S3Writer *writer = nullptr;
+auto st = daqiri::daqiri_s3_writer_create(cfg, &writer);
+
+daqiri::S3WriteHandle *handle = nullptr;
+if (st == daqiri::Status::SUCCESS) {
+    st = daqiri::daqiri_write_raw_to_s3_objects_async(
+        writer,
+        burst,
+        "runs/run42/packet",
+        60,
+        &handle);
+}
+
+daqiri::S3WriteStatus s3_status{};
+if (st == daqiri::Status::SUCCESS) {
+    st = daqiri::daqiri_s3_write_wait(handle, &s3_status);
+    daqiri::daqiri_s3_write_destroy(handle);
+}
+daqiri::daqiri_s3_writer_destroy(writer);
+```
+
+Object keys mirror raw file naming: `object_prefix_<packet_index>`. DAQIRI
+copies each packet's post-offset logical bytes into owned host staging memory
+before submission, so the burst may be released after
+`daqiri_write_raw_to_s3_objects_async()` succeeds. Header-data split and other
+multi-segment packets are concatenated into one object. The first S3 version
+uses one single-part `PutObject` per packet; objects larger than 5 GiB return
+`NOT_SUPPORTED`, and multipart/burst aggregation is future work.
+
+The Python bindings expose the same C++ writer when both
+`DAQIRI_BUILD_PYTHON=ON` and `DAQIRI_ENABLE_S3=ON` are used:
+
+```python
+cfg = daqiri.S3WriterConfig()
+cfg.bucket = "daqiri-captures"
+cfg.region = "us-west-2"
+
+writer = daqiri.S3Writer(cfg)
+status = writer.write_raw_objects(
+    burst,
+    "runs/run42/packet",
+    packet_data_offset=60,
+)
+writer.destroy()
+```
+
 ## Utility Functions
 
 ```cpp
@@ -468,6 +531,12 @@ workflow sections above show the common call order and ownership rules.
 | `daqiri_file_write_poll(handle, &status)` | Poll an asynchronous file-write handle. |
 | `daqiri_file_write_wait(handle, &status)` | Wait for asynchronous file writes to complete. |
 | `daqiri_file_write_destroy(handle)` | Release asynchronous file-write resources. |
+| `daqiri_s3_writer_create(config, &writer)` | Create an AWS SDK-backed S3 raw object writer. |
+| `daqiri_write_raw_to_s3_objects_async(writer, burst, object_prefix, packet_data_offset, &handle)` | Submit asynchronous raw packet uploads to S3. |
+| `daqiri_s3_write_poll(handle, &status)` | Poll asynchronous S3 uploads. |
+| `daqiri_s3_write_wait(handle, &status)` | Wait for asynchronous S3 uploads to complete. |
+| `daqiri_s3_write_destroy(handle)` | Release asynchronous S3 upload resources. |
+| `daqiri_s3_writer_destroy(writer)` | Release an S3 writer. |
 
 ### Ports, Traffic, Socket, and RDMA
 

docs/getting-started.md

@@ -18,6 +18,7 @@ DAQIRI's baseline requirements depend on which [stream type](concepts.md#stream-
 | **DPDK** | Included in the DAQIRI container (patched for dma-buf, so `nvidia-peermem` is **not required** inside the container); see [bare-metal dependencies](#bare-metal-dependencies) below for the host build. |
 | **RoCE** | `libibverbs` and `librdmacm` (for `stream_type: "socket"`, `protocol: "roce"`). |
 | **GDS** | Optional `cufile.h` and `libcufile` for file writes from CUDA device memory. Runtime device-memory writes require a working cuFile installation; for regular `nvidia-fs` mode, the `nvidia-fs` kernel module must be loaded and the destination storage stack must be supported. |
+| **S3** | Optional AWS SDK for C++ with the `s3` component for raw packet uploads to Amazon S3 or S3-compatible object stores. The DAQIRI container builds this SDK from source. |
 
 Supported platforms include [NVIDIA Data Center](https://www.nvidia.com/en-us/data-center/) systems, edge systems like [NVIDIA IGX](https://www.nvidia.com/en-us/edge-computing/products/igx/) and [NVIDIA DGX Spark](https://www.nvidia.com/en-us/products/workstations/dgx-spark/), and `x86_64` systems with the above components.
 
@@ -194,6 +195,7 @@ Both methods use the same public C++ include:
 | `DAQIRI_BUILD_EXAMPLES` | `ON` | Build benchmark executables. |
 | `DAQIRI_ENABLE_GDS` | `OFF` | Enable cuFile-backed burst file writes from CUDA device memory. Host-memory writes use POSIX APIs without GDS. |
 | `DAQIRI_ENABLE_OTEL_METRICS` | `OFF` | Enable OpenTelemetry C++ metrics instrumentation. When enabled, OpenTelemetry C++ API package metadata must be available to CMake. |
+| `DAQIRI_ENABLE_S3` | `OFF` | Enable AWS SDK-backed asynchronous raw packet writes to S3. |
 | `BUILD_SHARED_LIBS` | — | Build as shared library. |
 
 CUDA architectures default to `80;90` (A100, H100), with `121` (GB10) added
@@ -219,6 +221,14 @@ transmitted packets, received bytes, transmitted bytes, and dropped packets. DAQ
 does not configure an SDK reader or exporter; applications that want exported data
 must configure the OpenTelemetry C++ SDK before or during DAQIRI initialization.
 
+When using `DAQIRI_ENABLE_S3=ON`, the container build installs AWS SDK for C++
+with S3 support. Bare-metal builds must provide `aws-cpp-sdk-core` and
+`aws-cpp-sdk-s3` so CMake can resolve `find_package(AWSSDK COMPONENTS s3)`.
+Configure credentials through the AWS SDK provider chain, such as environment
+variables, a shared AWS profile, container credentials, or an EC2 instance role.
+DAQIRI writes one object per packet with a single `PutObject`; multipart uploads
+and PCAP output are not part of the S3 path.
+
 ## Next Steps
 
 Once DAQIRI is built, follow the tutorials to configure your system and run your first benchmark:

examples/README.md

@@ -29,6 +29,13 @@ cmake -S . -B build -DDAQIRI_BUILD_EXAMPLES=ON -DDAQIRI_ENABLE_GDS=ON -DDAQIRI_M
 cmake --build build -j
 ```
 
+Build with S3 raw object upload support:
+
+```bash
+cmake -S . -B build -DDAQIRI_BUILD_EXAMPLES=ON -DDAQIRI_ENABLE_S3=ON -DDAQIRI_MGR="dpdk socket"
+cmake --build build -j
+```
+
 For CUDA device-memory output, the runtime must have a working cuFile/GDS stack. In
 regular `nvidia-fs` mode, verify that the kernel module is loaded and the destination
 storage is supported before running the example:

include/daqiri/common.h

@@ -49,6 +49,25 @@ struct FileWriteStatus {
   uint64_t bytes_written = 0;
 };
 
+struct S3Writer;
+struct S3WriteHandle;
+
+struct S3WriterConfig {
+  std::string bucket;
+  std::string region;
+  std::string endpoint_override;
+  bool path_style = false;
+  bool aws_sdk_already_initialized = false;
+  uint32_t max_inflight_uploads = 8;
+  uint64_t max_staged_bytes = 1ULL << 30;
+};
+
+struct S3WriteStatus {
+  uint32_t completed_objects = 0;
+  uint32_t failed_objects = 0;
+  uint64_t bytes_uploaded = 0;
+};
+
 static constexpr uint32_t DEFAULT_TX_META_BUFFERS = 1UL << 8;
 static constexpr uint32_t DEFAULT_RX_META_BUFFERS = 1UL << 8;
 namespace detail {
@@ -457,6 +476,83 @@ Status daqiri_file_write_wait(FileWriteHandle *handle, FileWriteStatus *status);
  */
 Status daqiri_file_write_destroy(FileWriteHandle *handle);
 
+/**
+ * @brief Create an S3 raw object writer.
+ *
+ * Credentials are resolved by the AWS SDK provider chain. endpoint_override and
+ * path_style are only needed for S3-compatible services that require them.
+ * Returns NOT_SUPPORTED when DAQIRI was built without DAQIRI_ENABLE_S3=ON.
+ *
+ * @param config S3 writer configuration
+ * @param writer Output writer handle
+ * @return Status indicating success or failure
+ */
+Status daqiri_s3_writer_create(const S3WriterConfig &config,
+                               S3Writer **writer);
+
+/**
+ * @brief Asynchronously write each packet in a burst to a separate S3 object.
+ *
+ * Object keys are object_prefix_<packet_index>. The write path stages each
+ * packet's logical bytes into DAQIRI-owned host memory before returning, so the
+ * caller may free the burst after successful submission. No multipart upload is
+ * performed; objects larger than the S3 single-PUT limit are not supported.
+ *
+ * @param writer Writer returned by daqiri_s3_writer_create()
+ * @param burst Burst structure containing packets
+ * @param object_prefix Prefix used for each S3 object key
+ * @param packet_data_offset Bytes to skip from the start of each logical packet
+ * @param handle Output handle for polling, waiting, and cleanup
+ * @return Status indicating whether submission succeeded
+ */
+Status daqiri_write_raw_to_s3_objects_async(S3Writer *writer,
+                                            BurstParams *burst,
+                                            const std::string &object_prefix,
+                                            uint64_t packet_data_offset,
+                                            S3WriteHandle **handle);
+
+/**
+ * @brief Poll an asynchronous S3 write.
+ *
+ * @param handle Handle returned by daqiri_write_raw_to_s3_objects_async()
+ * @param status Optional output status summary
+ * @return SUCCESS when all uploads are complete, NOT_READY while pending, or an
+ * error status
+ */
+Status daqiri_s3_write_poll(S3WriteHandle *handle, S3WriteStatus *status);
+
+/**
+ * @brief Wait for asynchronous S3 writes to complete.
+ *
+ * @param handle Handle returned by daqiri_write_raw_to_s3_objects_async()
+ * @param status Optional output status summary
+ * @return SUCCESS when all uploads are complete or an error status
+ */
+Status daqiri_s3_write_wait(S3WriteHandle *handle, S3WriteStatus *status);
+
+/**
+ * @brief Destroy an asynchronous S3 write handle.
+ *
+ * If uploads are still pending, this call waits for completion before
+ * releasing staging buffers and request resources.
+ *
+ * @param handle Handle returned by daqiri_write_raw_to_s3_objects_async()
+ * @return SUCCESS when resources are released or an error status
+ */
+Status daqiri_s3_write_destroy(S3WriteHandle *handle);
+
+/**
+ * @brief Destroy an S3 raw object writer.
+ *
+ * The caller must not use the writer after this call. S3WriteHandle instances
+ * keep their own client references, so destroying a writer does not invalidate
+ * already submitted asynchronous writes.
+ *
+ * @param writer Writer returned by daqiri_s3_writer_create()
+ * @return SUCCESS when resources are released
+ */
+Status daqiri_s3_writer_destroy(S3Writer *writer);
+
 /**
  * @brief Frees all segments of a single packet
  *