mmap populate + diagnostics

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Filter changes
       uses: dorny/paths-filter@v4
@@ -63,7 +63,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Install Doxygen
       uses: ssciwr/doxygen-install@v2
@@ -89,7 +89,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Install LCov
       run: |
@@ -100,7 +100,13 @@ jobs:
 
     - name: Configure cmake
       shell: bash
-      run: cmake -B build -DCMAKE_CXX_FLAGS="--coverage" -Dndtbl_BUILD_TESTING=ON -Dndtbl_BUILD_DOCS=OFF
+      run: |
+        cmake -B build \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DCMAKE_CXX_FLAGS="--coverage -fprofile-update=atomic -O0 -g" \
+          -DCMAKE_EXE_LINKER_FLAGS="--coverage" \
+          -Dndtbl_BUILD_TESTING=ON \
+          -Dndtbl_BUILD_DOCS=OFF
 
     - name: Build
       shell: bash
@@ -118,7 +124,7 @@ jobs:
         lcov --directory . --capture --output-file coverage.info --ignore-errors mismatch,unused --exclude '*/catch2/*'
 
     - name: Upload C++ coverage to Codecov
-      uses: codecov/codecov-action@v6
+      uses: codecov/codecov-action@v7
       with:
         disable_search: true
         fail_ci_if_error: true
@@ -141,7 +147,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
@@ -166,7 +172,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Set up Python
       uses: actions/setup-python@v6
@@ -189,7 +195,7 @@ jobs:
         grep -o 'filename="[^"]*"' coverage.xml | head -n 1000
 
     - name: Upload Python coverage to Codecov
-      uses: codecov/codecov-action@v6
+      uses: codecov/codecov-action@v7
       with:
         disable_search: true
         fail_ci_if_error: true

.github/workflows/release-pypi.yml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -81,7 +81,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Download built distributions
         uses: actions/download-artifact@v8
@@ -117,7 +117,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Download built distributions
         uses: actions/download-artifact@v8

CMakeLists.txt

@@ -16,11 +16,35 @@ option(ndtbl_BUILD_TESTING "Enable building of tests" OFF)
 
 # Option for enabling POSIX mmap support for payload reads
 option(ndtbl_ENABLE_MMAP "Enable POSIX mmap-backed ndtbl payload reads" OFF)
+option(ndtbl_ENABLE_MMAP_POPULATE
+  "Enable Linux MAP_POPULATE for mmap-backed ndtbl payload reads"
+  OFF
+)
+option(ndtbl_ENABLE_MMAP_DIAGNOSTICS
+  "Enable mmap-backed payload diagnostics"
+  OFF
+)
 
 if(ndtbl_ENABLE_MMAP AND NOT UNIX)
   message(FATAL_ERROR "ndtbl_ENABLE_MMAP requires a POSIX platform")
 endif()
 
+if(ndtbl_ENABLE_MMAP_POPULATE AND NOT ndtbl_ENABLE_MMAP)
+  message(FATAL_ERROR "ndtbl_ENABLE_MMAP_POPULATE requires ndtbl_ENABLE_MMAP")
+endif()
+
+if(ndtbl_ENABLE_MMAP_POPULATE AND NOT LINUX)
+  message(FATAL_ERROR "ndtbl_ENABLE_MMAP_POPULATE requires Linux")
+endif()
+
+if(ndtbl_ENABLE_MMAP_DIAGNOSTICS AND NOT ndtbl_ENABLE_MMAP)
+  message(FATAL_ERROR "ndtbl_ENABLE_MMAP_DIAGNOSTICS requires ndtbl_ENABLE_MMAP")
+endif()
+
+if(ndtbl_ENABLE_MMAP_DIAGNOSTICS AND NOT LINUX)
+  message(FATAL_ERROR "ndtbl_ENABLE_MMAP_DIAGNOSTICS requires a LINUX platform")
+endif()
+
 # Add an interface target for the header-only library
 add_library(ndtbl INTERFACE)
 
@@ -35,6 +59,7 @@ target_sources(ndtbl
       include/ndtbl/grid.hpp
       include/ndtbl/field_group.hpp
       include/ndtbl/runtime_field_group.hpp
+      include/ndtbl/diagnostics.hpp
       include/ndtbl/metadata.hpp
       include/ndtbl/payload.hpp
       include/ndtbl/io.hpp
@@ -48,6 +73,8 @@ target_sources(ndtbl
 target_compile_definitions(ndtbl
   INTERFACE
     NDTBL_ENABLE_MMAP=$<BOOL:${ndtbl_ENABLE_MMAP}>
+    NDTBL_ENABLE_MMAP_POPULATE=$<BOOL:${ndtbl_ENABLE_MMAP_POPULATE}>
+    NDTBL_ENABLE_MMAP_DIAGNOSTICS=$<BOOL:${ndtbl_ENABLE_MMAP_DIAGNOSTICS}>
 )
 
 # Request a minimum C++ standard for this target

README.md

@@ -66,12 +66,31 @@ Relevant CMake options:
 - `ndtbl_BUILD_BENCHMARKS`: build developer lookup benchmarks, default `OFF`
 - `ndtbl_BUILD_DOCS`: build the documentation, default `ON` for top-level builds
 - `ndtbl_ENABLE_MMAP`: enable POSIX-only `mmap`-backed payload reads, default `OFF`
+- `ndtbl_ENABLE_MMAP_POPULATE`: add Linux-only `MAP_POPULATE` to `mmap`-backed
+  payload reads, default `OFF`; requires `ndtbl_ENABLE_MMAP=ON`
+- `ndtbl_ENABLE_MMAP_DIAGNOSTICS`: enable mmap payload residency diagnostics
+  through `mincore`, default `OFF`; requires `ndtbl_ENABLE_MMAP=ON`
 
 When `ndtbl_ENABLE_MMAP=OFF` (the default), `read_field_group()` and
 `read_runtime_field_group()` read payload data into owned heap storage. When
 `ndtbl_ENABLE_MMAP=ON`, supported POSIX builds use read-only memory mapping
 instead, which can reduce heap usage for large tables and enables shared memory
-access in multi-process environments.
+access in multi-process environments. On Linux, `ndtbl_ENABLE_MMAP_POPULATE=ON`
+adds `MAP_POPULATE` to the mapping flags so the kernel faults the mapped payload
+in during `mmap()` instead of on first access.
+
+When `ndtbl_ENABLE_MMAP_DIAGNOSTICS=ON`, mmap-loaded field groups can report OS
+page residency for their payload:
+
+```cpp
+const auto info = group.payload_residency();
+if (info.available) {
+  // inspect info.resident_pages, info.total_pages, info.resident_fraction
+}
+```
+
+The diagnostic is page-granular: it reports what `mincore` exposes for the
+mapped payload pages, not exact application RSS or exact bytes touched.
 
 If you want to install the C++ headers and CMake package metadata:
 
@@ -86,6 +105,20 @@ cmake -B build -Dndtbl_ENABLE_MMAP=ON
 cmake --build build
 ```
 
+To additionally pre-populate mapped payload pages on Linux:
+
+```bash
+cmake -B build -Dndtbl_ENABLE_MMAP=ON -Dndtbl_ENABLE_MMAP_POPULATE=ON
+cmake --build build
+```
+
+To enable mmap payload residency diagnostics on Linux:
+
+```bash
+cmake -B build -Dndtbl_ENABLE_MMAP=ON -Dndtbl_ENABLE_MMAP_DIAGNOSTICS=ON
+cmake --build build
+```
+
 ## ⚙️ C++ Tool Workflow
 
 Inspect existing `.ndtbl` files:

include/ndtbl/detail/mapped_payload.hpp

@@ -4,13 +4,25 @@
 #define NDTBL_ENABLE_MMAP 0
 #endif
 
+#ifndef NDTBL_ENABLE_MMAP_POPULATE
+#define NDTBL_ENABLE_MMAP_POPULATE 0
+#endif
+
+#ifndef NDTBL_ENABLE_MMAP_DIAGNOSTICS
+#define NDTBL_ENABLE_MMAP_DIAGNOSTICS 0
+#endif
+
+#include "ndtbl/diagnostics.hpp"
+
 #include <cstddef>
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <stdexcept>
 #include <string>
+#include <vector>
 
-#if NDTBL_ENABLE_MMAP
+#if NDTBL_ENABLE_MMAP || NDTBL_ENABLE_MMAP_DIAGNOSTICS
 #include <cerrno>
 #include <cstring>
 
@@ -20,9 +32,107 @@
 #include <unistd.h>
 #endif
 
+#if NDTBL_ENABLE_MMAP_DIAGNOSTICS
+#if !NDTBL_ENABLE_MMAP
+#error "NDTBL_ENABLE_MMAP_DIAGNOSTICS requires NDTBL_ENABLE_MMAP"
+#endif
+#endif
+
+#if NDTBL_ENABLE_MMAP_POPULATE
+#if !NDTBL_ENABLE_MMAP
+#error "NDTBL_ENABLE_MMAP_POPULATE requires NDTBL_ENABLE_MMAP"
+#endif
+#if !defined(__linux__)
+#error "NDTBL_ENABLE_MMAP_POPULATE requires Linux"
+#endif
+#if !defined(MAP_POPULATE)
+#error "NDTBL_ENABLE_MMAP_POPULATE requires MAP_POPULATE"
+#endif
+#endif
+
 namespace ndtbl {
 namespace detail {
 
+inline residency_info
+unavailable_residency()
+{
+  return residency_info{ false, 0, 0, 0, 0, 0.0 };
+}
+
+#if NDTBL_ENABLE_MMAP || NDTBL_ENABLE_MMAP_DIAGNOSTICS
+
+inline std::string
+system_error_message(const std::string& prefix)
+{
+  return prefix + ": " + std::strerror(errno);
+}
+
+#endif
+
+#if NDTBL_ENABLE_MMAP_DIAGNOSTICS
+
+inline residency_info
+query_residency(const void* address, std::size_t length)
+{
+  if (length == 0) {
+    return residency_info{ true, 0, 0, 0, 0, 0.0 };
+  }
+  if (address == nullptr) {
+    throw std::invalid_argument("cannot query residency for a null payload");
+  }
+
+  const long page_size_long = sysconf(_SC_PAGESIZE);
+  if (page_size_long <= 0) {
+    throw std::runtime_error("failed to query system page size for mincore");
+  }
+
+  const std::size_t page_size = static_cast<std::size_t>(page_size_long);
+  const std::uintptr_t addr = reinterpret_cast<std::uintptr_t>(address);
+  const std::uintptr_t aligned_addr = addr - (addr % page_size);
+  const std::size_t delta = static_cast<std::size_t>(addr - aligned_addr);
+  if (length > std::numeric_limits<std::size_t>::max() - delta) {
+    throw std::overflow_error("payload residency range is too large");
+  }
+  const std::size_t aligned_length = delta + length;
+  const std::size_t total_pages =
+    aligned_length / page_size + (aligned_length % page_size == 0 ? 0 : 1);
+  if (total_pages > std::numeric_limits<std::size_t>::max() / page_size) {
+    throw std::overflow_error("payload residency page range is too large");
+  }
+
+  std::vector<unsigned char> vec(total_pages);
+  if (mincore(reinterpret_cast<void*>(aligned_addr),
+              total_pages * page_size,
+              vec.data()) != 0) {
+    throw std::runtime_error(system_error_message("mincore failed"));
+  }
+
+  std::size_t resident_pages = 0;
+  for (std::size_t index = 0; index < vec.size(); ++index) {
+    if ((vec[index] & 1U) != 0U) {
+      ++resident_pages;
+    }
+  }
+
+  return residency_info{ true,
+                         page_size,
+                         total_pages,
+                         resident_pages,
+                         resident_pages * page_size,
+                         static_cast<double>(resident_pages) /
+                           static_cast<double>(total_pages) };
+}
+
+#else
+
+inline residency_info
+query_residency(const void*, std::size_t)
+{
+  return unavailable_residency();
+}
+
+#endif
+
 #if NDTBL_ENABLE_MMAP
 
 class mapped_payload_owner
@@ -49,12 +159,6 @@ class mapped_payload_owner
   std::size_t mapping_length_;
 };
 
-inline std::string
-system_error_message(const std::string& prefix)
-{
-  return prefix + ": " + std::strerror(errno);
-}
-
 inline std::shared_ptr<const std::uint8_t>
 map_payload_bytes(const std::string& path,
                   std::size_t payload_offset,
@@ -99,8 +203,13 @@ map_payload_bytes(const std::string& path,
   const std::size_t delta = payload_offset - aligned_offset;
   const std::size_t mapping_length = delta + payload_size;
 
+  int mapping_flags = MAP_PRIVATE;
+#if NDTBL_ENABLE_MMAP_POPULATE
+  mapping_flags |= MAP_POPULATE;
+#endif
+
   void* const mapping =
-    mmap(nullptr, mapping_length, PROT_READ, MAP_PRIVATE, fd, aligned_offset);
+    mmap(nullptr, mapping_length, PROT_READ, mapping_flags, fd, aligned_offset);
   const int saved_errno = errno;
   close(fd);
   if (mapping == MAP_FAILED) {

include/ndtbl/diagnostics.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <cstddef>
+
+namespace ndtbl {
+
+/**
+ * @brief Page residency information for an ndtbl payload.
+ *
+ * The byte count is page-granular because OS residency is reported per page.
+ */
+struct residency_info
+{
+  bool available;
+  std::size_t page_size;
+  std::size_t total_pages;
+  std::size_t resident_pages;
+  std::size_t resident_bytes;
+  double resident_fraction;
+};
+
+} // namespace ndtbl