From f57eb33e27fb12bc22216e6f55e057351fa40e3c Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 17 Jul 2026 15:37:28 +0200 Subject: [PATCH 1/8] =?UTF-8?q?feat(nativemem):=20categorized=20native-mem?= =?UTF-8?q?ory=20accounting=20=E2=80=94=20first=20cut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a NativeMem facility that tracks the profiler's own native memory usage per category, with a moving-window average and a running peak. - NativeMemCategory enum whose per-category live gauges partition the total (each backing allocation belongs to exactly one category, so there is no double counting). - Live accounting is always-on and independent of the COUNTERS build flag: record() is a single relaxed atomic add, async-signal-safe and usable from signal handlers. - sample() folds the live gauges into a moving-window average and a high-water max; it is ticked once per JFR chunk finish. - Totals mirror into the existing NATIVE_MEM_{LIVE,AVG,MAX}_BYTES counters (JFR + JNI counter path); per-category values are emitted as native_mem_{live,avg,max}_bytes. counter events, reusing the existing counter event format (no new event type). Instrumented sites (tagged CALLTRACE, their sole use today): the LinearAllocator chunk alloc/free (the call-trace arena) and the per-shard calltrace buffers. Accounting lives with the semantic owner rather than OS::safeAlloc, which stays category-agnostic. First-cut limits, documented in code: only CALLTRACE sites are instrumented so far (other categories read 0 until tagged); the max is sampled rather than spike-accurate; transient negative live is clamped to 0. The existing reserved/used/waste counters (CALLTRACE_STORAGE_BYTES, DICTIONARY_ARENA_WASTE_BYTES) remain an independent nested dimension and are intentionally not summed into the per-category total. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/counters.h | 3 + ddprof-lib/src/main/cpp/flightRecorder.cpp | 47 +++++++++ ddprof-lib/src/main/cpp/flightRecorder.h | 3 + ddprof-lib/src/main/cpp/linearAllocator.cpp | 6 ++ ddprof-lib/src/main/cpp/nativeMem.cpp | 103 ++++++++++++++++++++ ddprof-lib/src/main/cpp/nativeMem.h | 99 +++++++++++++++++++ ddprof-lib/src/main/cpp/profiler.cpp | 6 ++ ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 89 +++++++++++++++++ 8 files changed, 356 insertions(+) create mode 100644 ddprof-lib/src/main/cpp/nativeMem.cpp create mode 100644 ddprof-lib/src/main/cpp/nativeMem.h create mode 100644 ddprof-lib/src/test/cpp/nativeMem_ut.cpp diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index ffee1c0c4b..0cfe1a76dd 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -53,6 +53,9 @@ X(CALLTRACE_STORAGE_TRACES, "calltrace_storage_traces") \ X(LINEAR_ALLOCATOR_BYTES, "linear_allocator_bytes") \ X(LINEAR_ALLOCATOR_CHUNKS, "linear_allocator_chunks") \ + X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes") \ + X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes") \ + X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes") \ X(THREAD_IDS_COUNT, "thread_ids_count") \ X(THREAD_NAMES_COUNT, "thread_names_count") \ X(THREAD_FILTER_PAGES, "thread_filter_pages") \ diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index a622baf638..5fa16f7a89 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -12,6 +12,7 @@ #include "context.h" #include "context_api.h" #include "counters.h" +#include "nativeMem.h" #include "dictionary.h" #include "flightRecorder.inline.h" #include "incbin.h" @@ -810,7 +811,9 @@ off_t Recording::finishChunk(bool end_recording, bool do_cleanup) { // dictionary) will reflect the previous serialization. That is, some level of // familiarity with the code base will be required to use this diagnostic // information for now. + updateNativeMemStats(); writeCounters(_buf); + writeNativeMem(_buf); // Keep a simple stats for where we failed to unwind // For the sakes of simplicity we are not keeping the count of failed unwinds which would also be @@ -1776,6 +1779,50 @@ void Recording::writeLogLevels(Buffer *buf) { } } +void Recording::updateNativeMemStats() { + // Fold the per-category live gauges into their moving-window averages and + // running peaks. This is a sampled max: peaks that rise and fall entirely + // between two chunk finishes are not observed. A spike-accurate max would + // track the high-water mark at allocation time. + NativeMem::sample(); + + // Mirror the totals into the flat counter table so they flow out through the + // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug + // counters). Per-category values are emitted separately by writeNativeMem(). + Counters::set(NATIVE_MEM_LIVE_BYTES, NativeMem::liveTotal()); + Counters::set(NATIVE_MEM_AVG_BYTES, NativeMem::avgTotal()); + Counters::set(NATIVE_MEM_MAX_BYTES, NativeMem::maxTotal()); +} + +void Recording::writeNativeMem(Buffer *buf) { + // Emit live/avg/max per category as counter events, reusing the counter event + // format with a "." name so they land alongside the totals + // without needing a dedicated event type or a slot in the counter table. + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + NativeMemCategory cat = (NativeMemCategory)c; + const char *name = NativeMem::categoryName(cat); + const struct { + const char *prefix; + long long value; + } metrics[] = { + {"native_mem_live_bytes.", NativeMem::live(cat)}, + {"native_mem_avg_bytes.", NativeMem::avg(cat)}, + {"native_mem_max_bytes.", NativeMem::max(cat)}, + }; + for (const auto &m : metrics) { + char label[64]; + snprintf(label, sizeof(label), "%s%s", m.prefix, name); + int start = buf->skip(1); + buf->putVar64(T_DATADOG_COUNTER); + buf->putVar64(_start_ticks); + buf->putUtf8(label); + buf->putVar64(m.value); + writeEventSizePrefix(buf, start); + flushIfNeeded(buf); + } + } +} + void Recording::writeCounters(Buffer *buf) { long long *counters = Counters::getCounters(); if (counters) { diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index fd5bffda59..ee6929b4f1 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -304,6 +304,9 @@ class Recording { void writeCounters(Buffer *buf); + void updateNativeMemStats(); + void writeNativeMem(Buffer *buf); + void writeUnwindFailures(Buffer *buf); void writeContextSnapshot(Buffer *buf, Context &context); diff --git a/ddprof-lib/src/main/cpp/linearAllocator.cpp b/ddprof-lib/src/main/cpp/linearAllocator.cpp index 4aea288fee..cc12da403c 100644 --- a/ddprof-lib/src/main/cpp/linearAllocator.cpp +++ b/ddprof-lib/src/main/cpp/linearAllocator.cpp @@ -17,6 +17,7 @@ #include "linearAllocator.h" #include "counters.h" +#include "nativeMem.h" #include "os.h" #include "common.h" #include @@ -167,6 +168,9 @@ void LinearAllocator::freeChunks(ChunkList& chunks) { OS::safeFree(current, chunks.chunk_size); Counters::decrement(LINEAR_ALLOCATOR_BYTES, chunks.chunk_size); Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); + // The LinearAllocator's only user is call-trace storage, so all of its + // chunk memory is attributed to the CALLTRACE category. + NativeMem::record(NM_CALLTRACE, -(long long)chunks.chunk_size); current = prev; } @@ -260,6 +264,7 @@ Chunk *LinearAllocator::allocateChunk(Chunk *current) { Counters::increment(LINEAR_ALLOCATOR_BYTES, _chunk_size); Counters::increment(LINEAR_ALLOCATOR_CHUNKS); + NativeMem::record(NM_CALLTRACE, (long long)_chunk_size); } return chunk; } @@ -275,6 +280,7 @@ void LinearAllocator::freeChunk(Chunk *current) { OS::safeFree(current, _chunk_size); Counters::decrement(LINEAR_ALLOCATOR_BYTES, _chunk_size); Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); + NativeMem::record(NM_CALLTRACE, -(long long)_chunk_size); } void LinearAllocator::reserveChunk(Chunk *current) { diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp new file mode 100644 index 0000000000..4373e1f778 --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "nativeMem.h" + +volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {}; +long long NativeMem::_total_window[NativeMem::WINDOW] = {}; +int NativeMem::_window_pos = 0; +int NativeMem::_window_count = 0; +long long NativeMem::_avg[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; +long long NativeMem::_total_avg = 0; +long long NativeMem::_total_max = 0; + +long long NativeMem::liveTotal() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + total += load(_live[c]); + } + return total; +} + +void NativeMem::sample() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long v = load(_live[c]); + // Clamp transient negatives from not-yet-instrumented free sites so they do + // not skew the window; accounting is currently best-effort per category. + if (v < 0) { + v = 0; + } + _window[c][_window_pos] = v; + if (v > _max[c]) { + _max[c] = v; + } + total += v; + } + + _total_window[_window_pos] = total; + if (total > _total_max) { + _total_max = total; + } + + _window_pos = (_window_pos + 1) % WINDOW; + if (_window_count < WINDOW) { + _window_count++; + } + + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long sum = 0; + for (int i = 0; i < _window_count; i++) { + sum += _window[c][i]; + } + _avg[c] = sum / _window_count; + } + + long long total_sum = 0; + for (int i = 0; i < _window_count; i++) { + total_sum += _total_window[i]; + } + _total_avg = total_sum / _window_count; +} + +void NativeMem::reset() { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + store(_live[c], (long long)0); + _avg[c] = 0; + _max[c] = 0; + for (int i = 0; i < WINDOW; i++) { + _window[c][i] = 0; + } + } + for (int i = 0; i < WINDOW; i++) { + _total_window[i] = 0; + } + _window_pos = 0; + _window_count = 0; + _total_avg = 0; + _total_max = 0; +} + +const char *NativeMem::categoryName(NativeMemCategory category) { +#define X_NM_NAME(a, b) b, + static const char *const names[] = {DD_NATIVE_MEM_CATEGORY_TABLE(X_NM_NAME)}; +#undef X_NM_NAME + if (category < 0 || category >= NM_NUM_CATEGORIES) { + return "unknown"; + } + return names[category]; +} diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h new file mode 100644 index 0000000000..16807b4d68 --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _NATIVEMEM_H +#define _NATIVEMEM_H + +#include "arch.h" + +// Physical native-memory categories used by the profiler's own allocations. +// Every tracked backing allocation belongs to exactly one category, so the +// per-category live byte gauges partition the total: sum(category) == total, +// with no double counting. +// +// The "reserved vs used vs wasted" breakdowns exposed by the existing counters +// (CALLTRACE_STORAGE_BYTES is the used slice of the CALLTRACE arena; +// DICTIONARY_ARENA_WASTE_BYTES is the wasted slice of the DICTIONARY arena) are +// a separate, nested dimension. They are intentionally NOT summed in here. +#define DD_NATIVE_MEM_CATEGORY_TABLE(X) \ + X(CALLTRACE, "calltrace") \ + X(DICTIONARY, "dictionary") \ + X(CONTEXT, "context") \ + X(THREAD_FILTER, "thread_filter") \ + X(CODECACHE, "codecache") \ + X(LINE_TABLES, "line_tables") \ + X(PERF, "perf") \ + X(THREAD_LOCAL, "thread_local") \ + X(JFR_BUFFERS, "jfr_buffers") \ + X(MISC, "misc") + +#define X_NM_ENUM(a, b) NM_##a, +typedef enum NativeMemCategory : int { + DD_NATIVE_MEM_CATEGORY_TABLE(X_NM_ENUM) NM_NUM_CATEGORIES +} NativeMemCategory; +#undef X_NM_ENUM + +// Tracks the profiler's live native memory per category, plus a moving-window +// average and running peak. Live accounting is always-on and independent of the +// COUNTERS build flag: record() is a single relaxed atomic add, which is +// async-signal-safe and therefore usable from signal handlers. +// +// The moving average and peak are refreshed by sample(), which is expected to +// be called from a single thread (the JFR chunk-finish path). +class NativeMem { +private: + // Number of sample() ticks retained in the moving-average window. + static const int WINDOW = 64; + + static volatile long long _live[NM_NUM_CATEGORIES]; + + // sample()-owned state; not touched from allocation sites. + static long long _window[NM_NUM_CATEGORIES][WINDOW]; + static long long _total_window[WINDOW]; + static int _window_pos; + static int _window_count; + static long long _avg[NM_NUM_CATEGORIES]; + static long long _max[NM_NUM_CATEGORIES]; + static long long _total_avg; + static long long _total_max; + +public: + // Account for a backing allocation (delta > 0) or free (delta < 0). + static void record(NativeMemCategory category, long long delta) { + atomicIncRelaxed(_live[category], delta); + } + + static long long live(NativeMemCategory category) { + return load(_live[category]); + } + static long long liveTotal(); + + // Fold the current live gauges into the moving-average window and the running + // peak. Call once per sampling tick from a single thread. + static void sample(); + + static long long avg(NativeMemCategory category) { return _avg[category]; } + static long long max(NativeMemCategory category) { return _max[category]; } + static long long avgTotal() { return _total_avg; } + static long long maxTotal() { return _total_max; } + + // Clear all live gauges and window/peak state. Not thread-safe against + // concurrent record()/sample(); intended for process init and tests. + static void reset(); + + static const char *categoryName(NativeMemCategory category); +}; + +#endif // _NATIVEMEM_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 2820ee8c6f..95e10dbbaf 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -13,6 +13,7 @@ #include "guards.h" #include "common.h" #include "counters.h" +#include "nativeMem.h" #include "ctimer.h" #include "signalInflight.h" #include "dwarf.h" @@ -1370,6 +1371,7 @@ Error Profiler::start(Arguments &args, bool reset) { // (Re-)allocate calltrace buffers if (_max_stack_depth != args._jstackdepth) { + size_t prev_nelem = _max_stack_depth + RESERVED_FRAMES; _max_stack_depth = args._jstackdepth; size_t nelem = _max_stack_depth + RESERVED_FRAMES; @@ -1383,6 +1385,7 @@ Error Profiler::start(Arguments &args, bool reset) { return Error("Not enough memory to allocate stack trace buffers (try " "smaller jstackdepth)"); } + NativeMem::record(NM_CALLTRACE, (long long)(nelem * sizeof(CallTraceBuffer))); // Swap under the per-shard lock: all readers (recordJVMTISample, // recordExternalSample) acquire this lock via tryLock before reading // _calltrace_buffer, so no reader can observe a freed pointer mid-replacement. @@ -1390,6 +1393,9 @@ Error Profiler::start(Arguments &args, bool reset) { CallTraceBuffer *prev = _calltrace_buffer[i]; _calltrace_buffer[i] = fresh; _locks[i].unlock(); + if (prev != NULL) { + NativeMem::record(NM_CALLTRACE, -(long long)(prev_nelem * sizeof(CallTraceBuffer))); + } free(prev); } } diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp new file mode 100644 index 0000000000..d17e8cb0c9 --- /dev/null +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "nativeMem.h" + +class NativeMemTest : public ::testing::Test { +protected: + void SetUp() override { NativeMem::reset(); } + void TearDown() override { NativeMem::reset(); } +}; + +// record() adds to and subtracts from the per-category live gauge, and the +// total is the sum across categories. +TEST_F(NativeMemTest, RecordTracksLivePerCategoryAndTotal) { + NativeMem::record(NM_CALLTRACE, 1000); + NativeMem::record(NM_DICTIONARY, 250); + EXPECT_EQ(1000, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(250, NativeMem::live(NM_DICTIONARY)); + EXPECT_EQ(0, NativeMem::live(NM_CONTEXT)); + EXPECT_EQ(1250, NativeMem::liveTotal()); + + NativeMem::record(NM_CALLTRACE, -400); + EXPECT_EQ(600, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(850, NativeMem::liveTotal()); +} + +// sample() folds the current live gauge into the moving average; a single +// sample yields avg == live and max == live. +TEST_F(NativeMemTest, SingleSampleAverageEqualsLive) { + NativeMem::record(NM_CALLTRACE, 800); + NativeMem::sample(); + EXPECT_EQ(800, NativeMem::avg(NM_CALLTRACE)); + EXPECT_EQ(800, NativeMem::max(NM_CALLTRACE)); + EXPECT_EQ(800, NativeMem::avgTotal()); + EXPECT_EQ(800, NativeMem::maxTotal()); +} + +// The moving average is the mean over the sampled ticks. +TEST_F(NativeMemTest, MovingAverageOverSamples) { + NativeMem::record(NM_CALLTRACE, 100); + NativeMem::sample(); // window: [100] + NativeMem::record(NM_CALLTRACE, 200); // live now 300 + NativeMem::sample(); // window: [100, 300] -> avg 200 + EXPECT_EQ(200, NativeMem::avg(NM_CALLTRACE)); +} + +// The peak is a high-water mark: it retains the largest sampled value even +// after live memory has been freed back down. +TEST_F(NativeMemTest, MaxIsHighWaterMark) { + NativeMem::record(NM_CALLTRACE, 5000); + NativeMem::sample(); + NativeMem::record(NM_CALLTRACE, -4900); // live drops to 100 + NativeMem::sample(); + EXPECT_EQ(100, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(5000, NativeMem::max(NM_CALLTRACE)); +} + +// Transient negative live (an over-counted free at a not-yet-paired site) is +// clamped to zero when sampled so it does not skew the average or total. +TEST_F(NativeMemTest, NegativeLiveClampedInSample) { + NativeMem::record(NM_MISC, -1234); + NativeMem::sample(); + EXPECT_EQ(0, NativeMem::avg(NM_MISC)); + EXPECT_EQ(0, NativeMem::maxTotal()); +} + +// Every category exposes a distinct, non-empty name. +TEST_F(NativeMemTest, CategoryNamesPresent) { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + const char *name = NativeMem::categoryName((NativeMemCategory)c); + ASSERT_NE(nullptr, name); + EXPECT_GT((int)strlen(name), 0); + } +} From ae5afb6d53aea05910deee65ad1f419ce9470fe8 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 17 Jul 2026 17:20:16 +0200 Subject: [PATCH 2/8] feat(nativemem): precise per-category max with bounded total Track each category's peak at allocation time instead of sampling it at chunk finish, so a spike that rises and falls between two ticks is still captured. record() updates the per-category high-water mark on positive deltas via a relaxed CAS: the common (no new peak) path is a single load plus a compare, and the CAS fires only when a genuinely higher peak is set, which is rare since the peak is monotonic. Frees skip the check. The total peak avoids a shared global counter (a contention hotspot on a single cache line hit by every allocation) and is instead reported as a bracket: - NATIVE_MEM_MAX_BYTES = upper bound = sum of the precise per-category peaks (exact when the peaks coincide, otherwise an overestimate). - native_mem_max_observed_total_bytes = lower bound = the largest instantaneous total seen at a sampling tick. sample() no longer touches the per-category peaks; it only refreshes the moving averages and the observed total. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/flightRecorder.cpp | 42 ++++++++++++++-------- ddprof-lib/src/main/cpp/nativeMem.cpp | 25 ++++++++----- ddprof-lib/src/main/cpp/nativeMem.h | 35 +++++++++++++----- ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 20 ++++++++--- 4 files changed, 86 insertions(+), 36 deletions(-) diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 5fa16f7a89..b5bee39c79 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -1780,24 +1780,37 @@ void Recording::writeLogLevels(Buffer *buf) { } void Recording::updateNativeMemStats() { - // Fold the per-category live gauges into their moving-window averages and - // running peaks. This is a sampled max: peaks that rise and fall entirely - // between two chunk finishes are not observed. A spike-accurate max would - // track the high-water mark at allocation time. + // Refresh the moving-window averages and the observed total peak. Per-category + // peaks are maintained precisely at allocation time, so they are not sampled + // here; the total peak is bracketed instead (see writeNativeMem). NativeMem::sample(); // Mirror the totals into the flat counter table so they flow out through the // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug - // counters). Per-category values are emitted separately by writeNativeMem(). + // counters). NATIVE_MEM_MAX_BYTES carries the upper bound on the total peak + // (sum of precise per-category peaks); the observed lower bound and the + // per-category values are emitted by writeNativeMem(). Counters::set(NATIVE_MEM_LIVE_BYTES, NativeMem::liveTotal()); Counters::set(NATIVE_MEM_AVG_BYTES, NativeMem::avgTotal()); Counters::set(NATIVE_MEM_MAX_BYTES, NativeMem::maxTotal()); } void Recording::writeNativeMem(Buffer *buf) { - // Emit live/avg/max per category as counter events, reusing the counter event - // format with a "." name so they land alongside the totals - // without needing a dedicated event type or a slot in the counter table. + // Emit native-memory stats as counter events, reusing the counter event format + // so they land alongside the totals without needing a dedicated event type or + // a slot in the counter table. + auto emit = [&](const char *label, long long value) { + int start = buf->skip(1); + buf->putVar64(T_DATADOG_COUNTER); + buf->putVar64(_start_ticks); + buf->putUtf8(label); + buf->putVar64(value); + writeEventSizePrefix(buf, start); + flushIfNeeded(buf); + }; + + // Per-category live/avg/max, named ".". The max here is the + // precise per-category peak tracked at allocation time. for (int c = 0; c < NM_NUM_CATEGORIES; c++) { NativeMemCategory cat = (NativeMemCategory)c; const char *name = NativeMem::categoryName(cat); @@ -1812,15 +1825,14 @@ void Recording::writeNativeMem(Buffer *buf) { for (const auto &m : metrics) { char label[64]; snprintf(label, sizeof(label), "%s%s", m.prefix, name); - int start = buf->skip(1); - buf->putVar64(T_DATADOG_COUNTER); - buf->putVar64(_start_ticks); - buf->putUtf8(label); - buf->putVar64(m.value); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); + emit(label, m.value); } } + + // The total peak is bracketed: NATIVE_MEM_MAX_BYTES already carries the upper + // bound (sum of precise per-category peaks); here we also emit the observed + // lower bound (largest instantaneous total seen at a sampling tick). + emit("native_mem_max_observed_total_bytes", NativeMem::maxTotalObserved()); } void Recording::writeCounters(Buffer *buf) { diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp index 4373e1f778..3b06cbabf6 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.cpp +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -16,14 +16,14 @@ #include "nativeMem.h" volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {}; +volatile long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {}; long long NativeMem::_total_window[NativeMem::WINDOW] = {}; int NativeMem::_window_pos = 0; int NativeMem::_window_count = 0; long long NativeMem::_avg[NM_NUM_CATEGORIES] = {}; -long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; long long NativeMem::_total_avg = 0; -long long NativeMem::_total_max = 0; +long long NativeMem::_total_max_observed = 0; long long NativeMem::liveTotal() { long long total = 0; @@ -33,6 +33,14 @@ long long NativeMem::liveTotal() { return total; } +long long NativeMem::maxTotal() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + total += load(_max[c]); + } + return total; +} + void NativeMem::sample() { long long total = 0; for (int c = 0; c < NM_NUM_CATEGORIES; c++) { @@ -43,15 +51,14 @@ void NativeMem::sample() { v = 0; } _window[c][_window_pos] = v; - if (v > _max[c]) { - _max[c] = v; - } total += v; } + // The per-category peaks are maintained precisely at allocation time by + // record(); here we only track the observed total (lower bound on the peak). _total_window[_window_pos] = total; - if (total > _total_max) { - _total_max = total; + if (total > _total_max_observed) { + _total_max_observed = total; } _window_pos = (_window_pos + 1) % WINDOW; @@ -77,8 +84,8 @@ void NativeMem::sample() { void NativeMem::reset() { for (int c = 0; c < NM_NUM_CATEGORIES; c++) { store(_live[c], (long long)0); + store(_max[c], (long long)0); _avg[c] = 0; - _max[c] = 0; for (int i = 0; i < WINDOW; i++) { _window[c][i] = 0; } @@ -89,7 +96,7 @@ void NativeMem::reset() { _window_pos = 0; _window_count = 0; _total_avg = 0; - _total_max = 0; + _total_max_observed = 0; } const char *NativeMem::categoryName(NativeMemCategory category) { diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 16807b4d68..f2c64f03e6 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -58,21 +58,35 @@ class NativeMem { static const int WINDOW = 64; static volatile long long _live[NM_NUM_CATEGORIES]; + // Precise per-category high-water mark, maintained at allocation time by + // record() so peaks that rise and fall between sample() ticks are still seen. + static volatile long long _max[NM_NUM_CATEGORIES]; - // sample()-owned state; not touched from allocation sites. + // sample()-owned state; touched only from the single-threaded sampling path. static long long _window[NM_NUM_CATEGORIES][WINDOW]; static long long _total_window[WINDOW]; static int _window_pos; static int _window_count; static long long _avg[NM_NUM_CATEGORIES]; - static long long _max[NM_NUM_CATEGORIES]; static long long _total_avg; - static long long _total_max; + static long long _total_max_observed; public: // Account for a backing allocation (delta > 0) or free (delta < 0). static void record(NativeMemCategory category, long long delta) { - atomicIncRelaxed(_live[category], delta); + long long prev = atomicIncRelaxed(_live[category], delta); + if (delta > 0) { + // Only allocations can raise the peak. In the common case (no new peak) + // this is a single relaxed load plus a compare; the CAS fires only when a + // genuinely higher peak is set, which is rare since _max is monotonic. + long long now = prev + delta; + long long m = load(_max[category]); + while (now > m && + !__atomic_compare_exchange_n(&_max[category], &m, now, false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + // On failure m is reloaded with the current value; loop and retry. + } + } } static long long live(NativeMemCategory category) { @@ -80,14 +94,19 @@ class NativeMem { } static long long liveTotal(); - // Fold the current live gauges into the moving-average window and the running - // peak. Call once per sampling tick from a single thread. + // Fold the current live gauges into the moving-average window and the observed + // total peak. Call once per sampling tick from a single thread. static void sample(); static long long avg(NativeMemCategory category) { return _avg[category]; } - static long long max(NativeMemCategory category) { return _max[category]; } + static long long max(NativeMemCategory category) { return load(_max[category]); } static long long avgTotal() { return _total_avg; } - static long long maxTotal() { return _total_max; } + // Upper bound on the total peak: the sum of the precise per-category peaks. + // Exact only if the category peaks coincided; otherwise an overestimate. + static long long maxTotal(); + // Lower bound on the total peak: the largest instantaneous total seen at a + // sampling tick. Misses peaks that rise and fall entirely between ticks. + static long long maxTotalObserved() { return _total_max_observed; } // Clear all live gauges and window/peak state. Not thread-safe against // concurrent record()/sample(); intended for process init and tests. diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp index d17e8cb0c9..1b5d21d566 100644 --- a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -59,17 +59,29 @@ TEST_F(NativeMemTest, MovingAverageOverSamples) { EXPECT_EQ(200, NativeMem::avg(NM_CALLTRACE)); } -// The peak is a high-water mark: it retains the largest sampled value even -// after live memory has been freed back down. +// The peak is a high-water mark: it retains the largest value even after live +// memory has been freed back down. TEST_F(NativeMemTest, MaxIsHighWaterMark) { NativeMem::record(NM_CALLTRACE, 5000); - NativeMem::sample(); NativeMem::record(NM_CALLTRACE, -4900); // live drops to 100 - NativeMem::sample(); EXPECT_EQ(100, NativeMem::live(NM_CALLTRACE)); EXPECT_EQ(5000, NativeMem::max(NM_CALLTRACE)); } +// The per-category peak is precise: it is captured at allocation time, so a +// spike that rises and falls entirely between two sample() ticks is still seen +// by max(), while the observed total (sampled) misses it. maxTotal() (upper +// bound = sum of per-category peaks) brackets the true peak from above. +TEST_F(NativeMemTest, PerCategoryMaxIsPreciseBetweenSamples) { + NativeMem::record(NM_CALLTRACE, 9000); + NativeMem::record(NM_CALLTRACE, -8000); // peak of 9000 undone before any sample + NativeMem::sample(); // only ever observes live == 1000 + EXPECT_EQ(1000, NativeMem::live(NM_CALLTRACE)); + EXPECT_EQ(9000, NativeMem::max(NM_CALLTRACE)); // precise: caught the spike + EXPECT_EQ(1000, NativeMem::maxTotalObserved()); // sampled: missed the spike + EXPECT_EQ(9000, NativeMem::maxTotal()); // upper bound +} + // Transient negative live (an over-counted free at a not-yet-paired site) is // clamped to zero when sampled so it does not skew the average or total. TEST_F(NativeMemTest, NegativeLiveClampedInSample) { From 0531a39c4d2b88cff8a93e44506b890e387e67ef Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 20 Jul 2026 14:59:12 +0200 Subject: [PATCH 3/8] feat(nativemem): tag the big native-memory consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend per-category accounting beyond CALLTRACE to the other large, cleanly-paired backing allocations: - THREAD_LOCAL: `new`/`delete ProfiledThread` (sizeof, per thread). - JFR_BUFFERS: `new`/`delete Recording` (embeds the RecordingBuffer array and cpu-monitor buffer). - LINE_TABLES: the malloc'd JVMTI line-number table copy, tracked beside the existing LINE_NUMBER_TABLES counter and freed in ~SharedLineNumberTable (byte size recovered from the stored entry count). - PERF: the perf ring mmap (2 * page_size), paired with its munmap. - THREAD_FILTER: ChunkStorage chunks (bounded, tagged only on successful CAS install) and the FreeListNode array. - CODECACHE: a recomputed gauge, mirrored via the new NativeMem::setLive() at the existing CodeCache size-set site. setLive() overwrites live and still advances the peak. Not tagged in this commit: - CONTEXT has no separate allocation — the OTel context record is embedded in ProfiledThread, so it is already counted under THREAD_LOCAL; tagging it again would double-count. The category stays 0 by design. - DICTIONARY is deferred to its own commit: it has two implementations (the older per-key-malloc Dictionary used for symbols/packages, with set()-based counters and bulk key-frees that lose per-item size, and the arena-based StringDictionary where keys live inside chunks). A partial number that tagged only one would mislead, so it needs dedicated per-implementation handling. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/flightRecorder.cpp | 10 ++++++++++ ddprof-lib/src/main/cpp/nativeMem.h | 12 ++++++++++++ ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 6 ++++++ ddprof-lib/src/main/cpp/profiler.cpp | 3 +++ ddprof-lib/src/main/cpp/threadFilter.cpp | 11 +++++++++++ ddprof-lib/src/main/cpp/threadLocalData.cpp | 3 +++ ddprof-lib/src/main/cpp/threadLocalData.h | 7 ++++++- ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 11 +++++++++++ 8 files changed, 62 insertions(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index b5bee39c79..23be7b2275 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -74,6 +74,10 @@ SharedLineNumberTable::~SharedLineNumberTable() { if (_ptr != nullptr) { free(_ptr); Counters::decrement(LINE_NUMBER_TABLES); + // _size is the JVMTI entry count passed at construction (see + // fillJavaMethodInfo), so the byte size matches the allocation. + NativeMem::record(NM_LINE_TABLES, -(long long)((size_t)_size * + sizeof(jvmtiLineNumberEntry))); } } @@ -421,6 +425,8 @@ void Lookup::fillJavaMethodInfo(MethodInfo *mi, jmethodID method, line_number_table_size, owned_table); // Increment counter for tracking live line number tables Counters::increment(LINE_NUMBER_TABLES); + NativeMem::record(NM_LINE_TABLES, (long long)((size_t)line_number_table_size * + sizeof(jvmtiLineNumberEntry))); } } @@ -2123,6 +2129,9 @@ Error FlightRecorder::newRecording(bool reset) { } _rec = new Recording(fd, _args); + // The Recording embeds the JFR RecordingBuffer array and the cpu-monitor + // buffer, so its allocation size is the profiler's JFR buffer footprint. + NativeMem::record(NM_JFR_BUFFERS, (long long)sizeof(Recording)); return Error::OK; } @@ -2133,6 +2142,7 @@ void FlightRecorder::stop() { if (rec != nullptr) { // NULL first, deallocate later _rec = nullptr; + NativeMem::record(NM_JFR_BUFFERS, -(long long)sizeof(Recording)); delete rec; } } diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index f2c64f03e6..b5d9a94977 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -89,6 +89,18 @@ class NativeMem { } } + // Set a category's live value directly, for gauge-style subsystems whose size + // is recomputed as an absolute (rather than tracked via alloc/free deltas). + // Also advances the peak. Not for use from a signal handler. + static void setLive(NativeMemCategory category, long long value) { + store(_live[category], value); + long long m = load(_max[category]); + while (value > m && + !__atomic_compare_exchange_n(&_max[category], &m, value, false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + } + } + static long long live(NativeMemCategory category) { return load(_live[category]); } diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 6eae577f5f..1a1354290e 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -26,6 +26,7 @@ #include "jvmThread.h" #include "libraries.h" #include "log.h" +#include "nativeMem.h" #include "os.h" #include "perfEvents.h" #include "profiler.h" @@ -673,6 +674,10 @@ int PerfEvents::registerThread(int tid) { } } + if (page != NULL) { + NativeMem::record(NM_PERF, (long long)(2 * OS::page_size)); + } + _events[tid].reset(); _events[tid]._fd = fd; _events[tid]._page = (struct perf_event_mmap_page *)page; @@ -707,6 +712,7 @@ void PerfEvents::unregisterThread(int tid) { munmap(event->_page, 2 * OS::page_size); event->_page = NULL; event->unlock(); + NativeMem::record(NM_PERF, -(long long)(2 * OS::page_size)); } } diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 95e10dbbaf..e2e46e8826 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1745,6 +1745,9 @@ Error Profiler::dump(const char *path, const int length) { Counters::set(CODECACHE_NATIVE_SIZE_BYTES, native_libs.memoryUsage()); Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, native_libs.memoryUsage()); + // CodeCache size is a recomputed gauge (not alloc/free deltas), so mirror it + // as an absolute. It shares the accuracy caveats of the counter above. + NativeMem::setLive(NM_CODECACHE, native_libs.memoryUsage()); Error err = Error::OK; // rotateDictsAndRun rotates the dictionaries, takes lockAll() around the diff --git a/ddprof-lib/src/main/cpp/threadFilter.cpp b/ddprof-lib/src/main/cpp/threadFilter.cpp index 531ce75a1a..981e8483b6 100644 --- a/ddprof-lib/src/main/cpp/threadFilter.cpp +++ b/ddprof-lib/src/main/cpp/threadFilter.cpp @@ -21,6 +21,7 @@ #include "threadFilter.h" #include "arch.h" +#include "nativeMem.h" #include "os.h" #include "threadLocalData.h" #include @@ -38,6 +39,8 @@ ThreadFilter::ThreadFilter() : _enabled(false) { _chunks[i].store(nullptr, std::memory_order_relaxed); } _free_list = std::make_unique(kFreeListSize); + NativeMem::record(NM_THREAD_FILTER, + (long long)(kFreeListSize * sizeof(FreeListNode))); // Initialize the first chunk initializeChunk(0); @@ -64,8 +67,15 @@ ThreadFilter::~ThreadFilter() { // Detach and delete chunks for (int i = 0; i < kMaxChunks; ++i) { ChunkStorage* chunk = _chunks[i].exchange(nullptr, std::memory_order_acquire); + if (chunk != nullptr) { + NativeMem::record(NM_THREAD_FILTER, -(long long)sizeof(ChunkStorage)); + } delete chunk; } + // The _free_list unique_ptr frees its FreeListNode[] as this destructor + // returns; account for it here. + NativeMem::record(NM_THREAD_FILTER, + -(long long)(kFreeListSize * sizeof(FreeListNode))); } void ThreadFilter::initializeChunk(int chunk_idx) { @@ -86,6 +96,7 @@ void ThreadFilter::initializeChunk(int chunk_idx) { ChunkStorage* expected = nullptr; if (_chunks[chunk_idx].compare_exchange_strong(expected, new_chunk, std::memory_order_release)) { // Successfully installed + NativeMem::record(NM_THREAD_FILTER, (long long)sizeof(ChunkStorage)); } else { // Another thread beat us to it - clean up our allocation delete new_chunk; diff --git a/ddprof-lib/src/main/cpp/threadLocalData.cpp b/ddprof-lib/src/main/cpp/threadLocalData.cpp index 96c6c77d8e..f93a863afe 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.cpp +++ b/ddprof-lib/src/main/cpp/threadLocalData.cpp @@ -49,6 +49,9 @@ ProfiledThread* ProfiledThread::initCurrentThreadSignalSafe() { void ProfiledThread::freeValue(void* value) { SignalBlocker blocker; ProfiledThread* pt = reinterpret_cast(value); + // Sole deletion site for a ProfiledThread (invoked by the ThreadLocal + // destructor callback), so the THREAD_LOCAL decrement belongs here. + NativeMem::record(NM_THREAD_LOCAL, -(long long)sizeof(ProfiledThread)); delete pt; } diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h index 96364b6610..901af7b7a1 100644 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ b/ddprof-lib/src/main/cpp/threadLocalData.h @@ -7,6 +7,7 @@ #define THREAD_LOCAL_DATA_H #include "context.h" +#include "nativeMem.h" #include "otel_context.h" #include "os.h" #include "threadLocal.h" @@ -117,7 +118,11 @@ class ProfiledThread : public ThreadLocalData { virtual ~ProfiledThread() { } public: - static ProfiledThread *forTid(int tid) { return new ProfiledThread(tid); } + static ProfiledThread *forTid(int tid) { + ProfiledThread *pt = new ProfiledThread(tid); + NativeMem::record(NM_THREAD_LOCAL, (long long)sizeof(ProfiledThread)); + return pt; + } static bool isThreadKeyValid() { return _current_thread.isKeyValid(); } diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp index 1b5d21d566..114719d239 100644 --- a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -91,6 +91,17 @@ TEST_F(NativeMemTest, NegativeLiveClampedInSample) { EXPECT_EQ(0, NativeMem::maxTotal()); } +// setLive() overwrites the live value (gauge semantics) but still advances the +// peak, which is retained when the gauge shrinks. +TEST_F(NativeMemTest, SetLiveOverwritesAndAdvancesMax) { + NativeMem::setLive(NM_CODECACHE, 4096); + EXPECT_EQ(4096, NativeMem::live(NM_CODECACHE)); + EXPECT_EQ(4096, NativeMem::max(NM_CODECACHE)); + NativeMem::setLive(NM_CODECACHE, 1024); // gauge shrinks + EXPECT_EQ(1024, NativeMem::live(NM_CODECACHE)); + EXPECT_EQ(4096, NativeMem::max(NM_CODECACHE)); // peak retained +} + // Every category exposes a distinct, non-empty name. TEST_F(NativeMemTest, CategoryNamesPresent) { for (int c = 0; c < NM_NUM_CATEGORIES; c++) { From 40b1fa1f2b6cc46f7c21f0643f8ee9cf173bc4a5 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 20 Jul 2026 15:27:03 +0200 Subject: [PATCH 4/8] feat(nativemem): tag DICTIONARY, remove CONTEXT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add DICTIONARY to the per-category accounting, and remove the CONTEXT category. CONTEXT had no separate backing allocation — the OTel context record is embedded in ProfiledThread and already counted under THREAD_LOCAL — so it would have been a permanently-zero line. Removed rather than left dangling. DICTIONARY is tagged across both implementations, counting physical backing allocations once each (single category; per-role breakdown, if ever wanted, belongs in a nested dimension, not extra top-level categories): - Dictionary (older, per-key malloc; used for symbols/packages): root and overflow DictTables (constant size) and key strings. Key frees are bulk and lose per-item sizes, but keys are null-terminated so the malloc'd size (strlen + 1) is recovered at free — no running total needed. - StringDictionary / StringArena (arena-based): arena chunks and root / overflow SBTables. Keys are bump-allocated inside chunks, so they are NOT counted separately (that would double-count). Chunk and SBTable accounting is unconditional, independent of the diagnostic counters' _counter_offset gate, so anonymous dictionaries are covered too. Tests: lifecycle invariants for both implementations assert the accounting grows on insert, returns exactly to the construction baseline after clear(), and to zero after destruction — directly proving inc/dec pairing (including the strlen-at-free path and arena chunk growth). Full gtestDebug suite green (356 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/dictionary.cpp | 7 +++ ddprof-lib/src/main/cpp/dictionary.h | 2 + ddprof-lib/src/main/cpp/nativeMem.h | 1 - ddprof-lib/src/main/cpp/stringDictionary.h | 46 +++++++++++++++---- ddprof-lib/src/test/cpp/dictionary_ut.cpp | 27 +++++++++++ ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 2 +- .../src/test/cpp/stringDictionary_ut.cpp | 29 ++++++++++++ 7 files changed, 104 insertions(+), 10 deletions(-) diff --git a/ddprof-lib/src/main/cpp/dictionary.cpp b/ddprof-lib/src/main/cpp/dictionary.cpp index 29db2eef5b..be9fa0cddc 100644 --- a/ddprof-lib/src/main/cpp/dictionary.cpp +++ b/ddprof-lib/src/main/cpp/dictionary.cpp @@ -37,6 +37,7 @@ static inline bool keyEquals(const char *candidate, const char *key, Dictionary::~Dictionary() { clear(_table, _id); free(_table); + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(DictTable)); Counters::set(DICTIONARY_BYTES, 0, _id); Counters::set(DICTIONARY_PAGES, 0, _id); } @@ -58,6 +59,9 @@ void Dictionary::clear(DictTable *table, int id) { DictRow *row = &table->rows[i]; for (int j = 0; j < CELLS; j++) { if (row->keys[j]) { + // Keys are null-terminated, so the malloc'd size (length + 1) is + // recoverable at free time without tracking it per key. + NativeMem::record(NM_DICTIONARY, -(long long)(strlen(row->keys[j]) + 1)); free(row->keys[j]); // content is zeroed en-mass in the clear() function } } @@ -65,6 +69,7 @@ void Dictionary::clear(DictTable *table, int id) { clear(row->next, id); DictTable *tmp = row->next; row->next = NULL; + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(DictTable)); free(tmp); } } @@ -110,6 +115,7 @@ unsigned int Dictionary::lookup(const char *key, size_t length, bool for_insert, if (__sync_bool_compare_and_swap(&row->keys[c], NULL, new_key)) { Counters::increment(DICTIONARY_KEYS, 1, _id); Counters::increment(DICTIONARY_KEYS_BYTES, length + 1, _id); + NativeMem::record(NM_DICTIONARY, (long long)(length + 1)); atomicInc(_size); return table->index(h % ROWS, c); } @@ -130,6 +136,7 @@ unsigned int Dictionary::lookup(const char *key, size_t length, bool for_insert, } else { Counters::increment(DICTIONARY_PAGES, 1, _id); Counters::increment(DICTIONARY_BYTES, sizeof(DictTable), _id); + NativeMem::record(NM_DICTIONARY, (long long)sizeof(DictTable)); } } else { return sentinel; diff --git a/ddprof-lib/src/main/cpp/dictionary.h b/ddprof-lib/src/main/cpp/dictionary.h index e5df18f828..949814eb1c 100644 --- a/ddprof-lib/src/main/cpp/dictionary.h +++ b/ddprof-lib/src/main/cpp/dictionary.h @@ -18,6 +18,7 @@ #define _DICTIONARY_H #include "counters.h" +#include "nativeMem.h" #include #include #include @@ -67,6 +68,7 @@ class Dictionary { _table = (DictTable *)calloc(1, sizeof(DictTable)); Counters::set(DICTIONARY_PAGES, 1, id); Counters::set(DICTIONARY_BYTES, sizeof(DictTable), id); + NativeMem::record(NM_DICTIONARY, (long long)sizeof(DictTable)); _table->base_index = _base_index = 1; _size = 0; } diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index b5d9a94977..8e3d7ea61b 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -30,7 +30,6 @@ #define DD_NATIVE_MEM_CATEGORY_TABLE(X) \ X(CALLTRACE, "calltrace") \ X(DICTIONARY, "dictionary") \ - X(CONTEXT, "context") \ X(THREAD_FILTER, "thread_filter") \ X(CODECACHE, "codecache") \ X(LINE_TABLES, "line_tables") \ diff --git a/ddprof-lib/src/main/cpp/stringDictionary.h b/ddprof-lib/src/main/cpp/stringDictionary.h index a87b7ec0f8..dd9989d05e 100644 --- a/ddprof-lib/src/main/cpp/stringDictionary.h +++ b/ddprof-lib/src/main/cpp/stringDictionary.h @@ -8,6 +8,7 @@ #include "counters.h" #include "log.h" +#include "nativeMem.h" #include "refCountGuard.h" #include "tripleBuffer.h" #include "arch.h" @@ -80,7 +81,14 @@ class StringArena { bool _oom_logged{false};// latched per generation; cleared by reset() static Chunk* make_chunk() { - return static_cast(calloc(1, sizeof(Chunk))); + Chunk* c = static_cast(calloc(1, sizeof(Chunk))); + // Track every backing chunk regardless of _counter_offset (which only + // gates the diagnostic DICTIONARY_BYTES counter). Keys are bump-allocated + // inside these chunks, so they must not be counted separately. + if (c != nullptr) { + NativeMem::record(NM_DICTIONARY, (long long)sizeof(Chunk)); + } + return c; } void countChunkAlloc() { @@ -133,7 +141,12 @@ class StringArena { ~StringArena() { Chunk* c = _first; - while (c) { Chunk* n = c->next; free(c); c = n; } + while (c) { + Chunk* n = c->next; + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); + free(c); + c = n; + } } StringArena(const StringArena&) = delete; @@ -167,7 +180,13 @@ class StringArena { void reset() { Chunk* c = _first ? _first->next : nullptr; int freed = 0; - while (c) { Chunk* n = c->next; free(c); c = n; ++freed; } + while (c) { + Chunk* n = c->next; + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); + free(c); + c = n; + ++freed; + } if (_first) { _first->next = nullptr; __atomic_store_n(&_first->pos, (size_t)0, __ATOMIC_RELAXED); @@ -229,7 +248,11 @@ class StringDictionaryBuffer { while (!stk.empty()) { Frame& f = stk.back(); if (f.row >= ROWS) { - if (f.t != table) { free(f.t); freed++; } + if (f.t != table) { + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(SBTable)); + free(f.t); + freed++; + } stk.pop_back(); continue; } @@ -264,11 +287,15 @@ class StringDictionaryBuffer { public: StringDictionaryBuffer() { _table = static_cast(calloc(1, sizeof(SBTable))); + if (_table != nullptr) { + NativeMem::record(NM_DICTIONARY, (long long)sizeof(SBTable)); + } } ~StringDictionaryBuffer() { if (_table != nullptr) { - freeOverflowNodes(_table); + freeOverflowNodes(_table); // records its own decrements + NativeMem::record(NM_DICTIONARY, -(long long)sizeof(SBTable)); free(_table); _table = nullptr; } @@ -356,9 +383,12 @@ class StringDictionaryBuffer { if (nt == nullptr) return 0; if (!__sync_bool_compare_and_swap(&row->next, nullptr, nt)) { free(nt); - } else if (_counter_offset != 0) { - Counters::increment(DICTIONARY_PAGES, 1, _counter_offset); - Counters::increment(DICTIONARY_BYTES, (long long)sizeof(SBTable), _counter_offset); + } else { + NativeMem::record(NM_DICTIONARY, (long long)sizeof(SBTable)); + if (_counter_offset != 0) { + Counters::increment(DICTIONARY_PAGES, 1, _counter_offset); + Counters::increment(DICTIONARY_BYTES, (long long)sizeof(SBTable), _counter_offset); + } } } table = __atomic_load_n(&row->next, __ATOMIC_ACQUIRE); diff --git a/ddprof-lib/src/test/cpp/dictionary_ut.cpp b/ddprof-lib/src/test/cpp/dictionary_ut.cpp index 9d6cf343e3..c54030ae55 100644 --- a/ddprof-lib/src/test/cpp/dictionary_ut.cpp +++ b/ddprof-lib/src/test/cpp/dictionary_ut.cpp @@ -16,7 +16,10 @@ #include #include "dictionary.h" +#include "nativeMem.h" #include +#include +#include #include // ── Dictionary ───────────────────────────────────────────────────────────── @@ -28,6 +31,30 @@ TEST(DictionaryTest, LookupReturnsSameIdForSameKey) { EXPECT_EQ(id1, d.lookup("hello", 5)); } +// Native-memory accounting must balance: growth is tracked, clear() returns to +// the root-table baseline, and destruction releases everything. Exercises the +// per-key strlen-at-free path and the overflow-table inc/dec pairing. +TEST(DictionaryTest, NativeMemAccountingBalancesAcrossLifecycle) { + long long before = NativeMem::live(NM_DICTIONARY); + { + Dictionary d(0); + long long after_ctor = NativeMem::live(NM_DICTIONARY); + EXPECT_GT(after_ctor, before); // root table counted + + for (int i = 0; i < 5000; i++) { + char key[32]; + snprintf(key, sizeof(key), "symbol_%d", i); + d.lookup(key, strlen(key)); // inserts key strings + overflow tables + } + EXPECT_GT(NativeMem::live(NM_DICTIONARY), after_ctor); // grew + + d.clear(); + // Keys + overflow tables freed; root table retained. + EXPECT_EQ(after_ctor, NativeMem::live(NM_DICTIONARY)); + } + EXPECT_EQ(before, NativeMem::live(NM_DICTIONARY)); // fully released +} + TEST(DictionaryTest, LookupReturnsDifferentIdsForDifferentKeys) { Dictionary d(0); unsigned int a = d.lookup("alpha", 5); diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp index 114719d239..f2fb74e103 100644 --- a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -31,7 +31,7 @@ TEST_F(NativeMemTest, RecordTracksLivePerCategoryAndTotal) { NativeMem::record(NM_DICTIONARY, 250); EXPECT_EQ(1000, NativeMem::live(NM_CALLTRACE)); EXPECT_EQ(250, NativeMem::live(NM_DICTIONARY)); - EXPECT_EQ(0, NativeMem::live(NM_CONTEXT)); + EXPECT_EQ(0, NativeMem::live(NM_THREAD_FILTER)); EXPECT_EQ(1250, NativeMem::liveTotal()); NativeMem::record(NM_CALLTRACE, -400); diff --git a/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp b/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp index 554418f394..20100b8ed5 100644 --- a/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp +++ b/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp @@ -1,6 +1,9 @@ #include #include "stringDictionary.h" +#include "nativeMem.h" #include +#include +#include #include #include #include @@ -8,6 +11,32 @@ // ── StringDictionaryBuffer ───────────────────────────────────────────────── +// Native-memory accounting must balance across the arena lifecycle: the root +// SBTable and initial chunk are counted at construction, growth adds overflow +// SBTables and arena chunks, clear() returns to the construction baseline, and +// destruction releases everything. 50k keys force at least one extra 512 KiB +// arena chunk so chunk alloc/free pairing is exercised too. +TEST(StringDictionaryBufferTest, NativeMemAccountingBalancesAcrossLifecycle) { + long long before = NativeMem::live(NM_DICTIONARY); + { + StringDictionaryBuffer buf; + long long after_ctor = NativeMem::live(NM_DICTIONARY); + EXPECT_GT(after_ctor, before); // root SBTable + initial arena chunk + + for (int i = 0; i < 50000; i++) { + char key[32]; + int len = snprintf(key, sizeof(key), "string_key_%d", i); + buf.insert_with_id(key, (size_t)len, (u32)(i + 1)); + } + EXPECT_GT(NativeMem::live(NM_DICTIONARY), after_ctor); // grew + + buf.clear(); + // Overflow SBTables + extra arena chunks freed; root + first chunk kept. + EXPECT_EQ(after_ctor, NativeMem::live(NM_DICTIONARY)); + } + EXPECT_EQ(before, NativeMem::live(NM_DICTIONARY)); // fully released +} + TEST(StringDictionaryBufferTest, InsertWithIdReturnsSameIdForSameKey) { StringDictionaryBuffer buf; u32 id = buf.insert_with_id("hello", 5, 42); From 508b6bef193d76b5cf1b38c4a3b96701b7bcd77e Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 20 Jul 2026 18:54:55 +0200 Subject: [PATCH 5/8] docs(nativemem): scope the async-signal-safety note to CALLTRACE The class comment implied every record() call needs to be async-signal-safe. In fact most categories allocate via malloc/new off the signal path, where the property is irrelevant. Only the CALLTRACE arena allocates from within the sampling signal handler (via OS::safeAlloc's raw mmap syscall), so that is where record() staying a relaxed atomic add actually matters. Reword accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/nativeMem.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 8e3d7ea61b..854cc4c9d5 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -46,8 +46,13 @@ typedef enum NativeMemCategory : int { // Tracks the profiler's live native memory per category, plus a moving-window // average and running peak. Live accounting is always-on and independent of the -// COUNTERS build flag: record() is a single relaxed atomic add, which is -// async-signal-safe and therefore usable from signal handlers. +// COUNTERS build flag: record() is a single relaxed atomic add. +// +// Most call sites are off the signal path (they allocate via malloc/new, which +// are not async-signal-safe anyway). The exception is the CALLTRACE arena: it +// allocates from within the sampling signal handler (via OS::safeAlloc's raw +// mmap syscall), so record() must stay async-signal-safe there -- which a +// relaxed atomic add is. // // The moving average and peak are refreshed by sample(), which is expected to // be called from a single thread (the JFR chunk-finish path). From 3c82245cff2102b58ef2e0ca5e28df5faf67d3b7 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 21 Jul 2026 11:23:31 +0200 Subject: [PATCH 6/8] refactor(nativemem): rename CODECACHE category to NATIVE_SYMBOLS The "CodeCache" name is async-profiler's, and collides with the JVM's JIT code cache. This category measures something unrelated: the profiler's own per-native-library symbol tables (used to symbolicate native frames), not JVM-managed code. Rename the NativeMem category and its JFR label to native_symbols so the metric is unambiguous. The existing CODECACHE_NATIVE_SIZE_BYTES counter keeps its name for continuity; only the new NativeMem category is renamed. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/nativeMem.h | 2 +- ddprof-lib/src/main/cpp/profiler.cpp | 7 ++++--- ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 12 ++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 854cc4c9d5..9d97997afa 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -31,7 +31,7 @@ X(CALLTRACE, "calltrace") \ X(DICTIONARY, "dictionary") \ X(THREAD_FILTER, "thread_filter") \ - X(CODECACHE, "codecache") \ + X(NATIVE_SYMBOLS, "native_symbols") \ X(LINE_TABLES, "line_tables") \ X(PERF, "perf") \ X(THREAD_LOCAL, "thread_local") \ diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index e2e46e8826..9c7be0ceda 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1745,9 +1745,10 @@ Error Profiler::dump(const char *path, const int length) { Counters::set(CODECACHE_NATIVE_SIZE_BYTES, native_libs.memoryUsage()); Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, native_libs.memoryUsage()); - // CodeCache size is a recomputed gauge (not alloc/free deltas), so mirror it - // as an absolute. It shares the accuracy caveats of the counter above. - NativeMem::setLive(NM_CODECACHE, native_libs.memoryUsage()); + // CodeCache here is the profiler's native-symbol tables (not the JVM code + // cache). Its size is a recomputed gauge (not alloc/free deltas), so mirror + // it as an absolute. It shares the accuracy caveats of the counter above. + NativeMem::setLive(NM_NATIVE_SYMBOLS, native_libs.memoryUsage()); Error err = Error::OK; // rotateDictsAndRun rotates the dictionaries, takes lockAll() around the diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp index f2fb74e103..ce467bc997 100644 --- a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -94,12 +94,12 @@ TEST_F(NativeMemTest, NegativeLiveClampedInSample) { // setLive() overwrites the live value (gauge semantics) but still advances the // peak, which is retained when the gauge shrinks. TEST_F(NativeMemTest, SetLiveOverwritesAndAdvancesMax) { - NativeMem::setLive(NM_CODECACHE, 4096); - EXPECT_EQ(4096, NativeMem::live(NM_CODECACHE)); - EXPECT_EQ(4096, NativeMem::max(NM_CODECACHE)); - NativeMem::setLive(NM_CODECACHE, 1024); // gauge shrinks - EXPECT_EQ(1024, NativeMem::live(NM_CODECACHE)); - EXPECT_EQ(4096, NativeMem::max(NM_CODECACHE)); // peak retained + NativeMem::setLive(NM_NATIVE_SYMBOLS, 4096); + EXPECT_EQ(4096, NativeMem::live(NM_NATIVE_SYMBOLS)); + EXPECT_EQ(4096, NativeMem::max(NM_NATIVE_SYMBOLS)); + NativeMem::setLive(NM_NATIVE_SYMBOLS, 1024); // gauge shrinks + EXPECT_EQ(1024, NativeMem::live(NM_NATIVE_SYMBOLS)); + EXPECT_EQ(4096, NativeMem::max(NM_NATIVE_SYMBOLS)); // peak retained } // Every category exposes a distinct, non-empty name. From 2e48ae5d9d29fef9f0f62b050758a20afcfb4cda Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 21 Jul 2026 15:00:52 +0200 Subject: [PATCH 7/8] docs(nativemem): correct record() description (add + high-water CAS) The class comment still described record() as "a single relaxed atomic add" from before precise per-category max was added. record() now also does a conditional lock-free high-water update on allocation. Correct the wording; the async-signal-safety guarantee still holds (lock-free atomics). Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/nativeMem.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 9d97997afa..dfa31f9a1e 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -46,13 +46,14 @@ typedef enum NativeMemCategory : int { // Tracks the profiler's live native memory per category, plus a moving-window // average and running peak. Live accounting is always-on and independent of the -// COUNTERS build flag: record() is a single relaxed atomic add. +// COUNTERS build flag: record() is a relaxed atomic add, plus (for allocations) +// a conditional lock-free high-water update of the per-category peak. // // Most call sites are off the signal path (they allocate via malloc/new, which // are not async-signal-safe anyway). The exception is the CALLTRACE arena: it // allocates from within the sampling signal handler (via OS::safeAlloc's raw -// mmap syscall), so record() must stay async-signal-safe there -- which a -// relaxed atomic add is. +// mmap syscall), so record() must stay async-signal-safe there -- which +// lock-free relaxed atomics are. // // The moving average and peak are refreshed by sample(), which is expected to // be called from a single thread (the JFR chunk-finish path). From 19db9d3172a3b825b2a3c2d9cde16d419f2a16ab Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 21 Jul 2026 15:13:58 +0200 Subject: [PATCH 8/8] harden(nativemem): assert the non-negative and key-length invariants Two invariants the accounting relies on are now asserted (stripped under NDEBUG, so no release cost and never in a real signal handler): - Per-category live bytes never go negative: record() asserts the post-update value >= 0. A negative means an unbalanced/oversized free. This lets liveTotal() sum without clamping, since each term is >= 0. - Dictionary keys are NUL-free strings of exactly `length`: allocateKey() asserts strlen == length. The NM_DICTIONARY free path recovers a key's size via strlen at clear(), so an embedded NUL would under-count; the assert trips in debug/gtest instead of silently drifting. The sample() clamp is retained as a release-mode safety net for the asserted-impossible negative case, and its comment updated to say so. The old NegativeLiveClampedInSample test is removed: it deliberately drove a category negative, which now (correctly) trips the record() assert. Co-Authored-By: Claude Opus 4.8 (1M context) --- ddprof-lib/src/main/cpp/dictionary.cpp | 7 +++++++ ddprof-lib/src/main/cpp/nativeMem.cpp | 6 ++++-- ddprof-lib/src/main/cpp/nativeMem.h | 13 ++++++++++--- ddprof-lib/src/test/cpp/nativeMem_ut.cpp | 9 --------- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ddprof-lib/src/main/cpp/dictionary.cpp b/ddprof-lib/src/main/cpp/dictionary.cpp index be9fa0cddc..df01a51def 100644 --- a/ddprof-lib/src/main/cpp/dictionary.cpp +++ b/ddprof-lib/src/main/cpp/dictionary.cpp @@ -18,6 +18,7 @@ #include "arch.h" #include "counters.h" #include "signalSafety.h" +#include #include #include #include @@ -26,6 +27,12 @@ static inline char *allocateKey(const char *key, size_t length) { char *result = (char *)malloc(length + 1); memcpy(result, key, length); result[length] = 0; + // NM_DICTIONARY accounting recovers a freed key's size via strlen at clear() + // time, which requires the key to be a NUL-free string of exactly `length`. + // Pin that assumption here so a future caller passing an embedded NUL trips in + // debug/gtest rather than silently under-counting the free. Stripped under + // NDEBUG. + assert(strlen(result) == length); return result; } diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp index 3b06cbabf6..166f9cc0a1 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.cpp +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -45,8 +45,10 @@ void NativeMem::sample() { long long total = 0; for (int c = 0; c < NM_NUM_CATEGORIES; c++) { long long v = load(_live[c]); - // Clamp transient negatives from not-yet-instrumented free sites so they do - // not skew the window; accounting is currently best-effort per category. + // A category's live bytes are never negative under correct pairing (asserted + // in record()). This clamp is a release-mode safety net: should an accounting + // bug slip past the assert under NDEBUG, it keeps a negative from skewing the + // window average and total rather than propagating garbage. if (v < 0) { v = 0; } diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index dfa31f9a1e..2ae30c3aae 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -17,6 +17,7 @@ #define _NATIVEMEM_H #include "arch.h" +#include // Physical native-memory categories used by the profiler's own allocations. // Every tracked backing allocation belongs to exactly one category, so the @@ -80,14 +81,20 @@ class NativeMem { // Account for a backing allocation (delta > 0) or free (delta < 0). static void record(NativeMemCategory category, long long delta) { long long prev = atomicIncRelaxed(_live[category], delta); + long long updated = prev + delta; + // Invariant: a category's live bytes never go negative — every decrement is + // matched by a prior same-sized increment. A negative result means an + // unbalanced or oversized free (an accounting bug). Stripped under NDEBUG, + // so this costs nothing in release and never runs in a real signal handler + // there; it catches pairing bugs in debug/gtest builds instead. + assert(updated >= 0); if (delta > 0) { // Only allocations can raise the peak. In the common case (no new peak) // this is a single relaxed load plus a compare; the CAS fires only when a // genuinely higher peak is set, which is rare since _max is monotonic. - long long now = prev + delta; long long m = load(_max[category]); - while (now > m && - !__atomic_compare_exchange_n(&_max[category], &m, now, false, + while (updated > m && + !__atomic_compare_exchange_n(&_max[category], &m, updated, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { // On failure m is reloaded with the current value; loop and retry. } diff --git a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp index ce467bc997..389d447819 100644 --- a/ddprof-lib/src/test/cpp/nativeMem_ut.cpp +++ b/ddprof-lib/src/test/cpp/nativeMem_ut.cpp @@ -82,15 +82,6 @@ TEST_F(NativeMemTest, PerCategoryMaxIsPreciseBetweenSamples) { EXPECT_EQ(9000, NativeMem::maxTotal()); // upper bound } -// Transient negative live (an over-counted free at a not-yet-paired site) is -// clamped to zero when sampled so it does not skew the average or total. -TEST_F(NativeMemTest, NegativeLiveClampedInSample) { - NativeMem::record(NM_MISC, -1234); - NativeMem::sample(); - EXPECT_EQ(0, NativeMem::avg(NM_MISC)); - EXPECT_EQ(0, NativeMem::maxTotal()); -} - // setLive() overwrites the live value (gauge semantics) but still advances the // peak, which is retained when the gauge shrinks. TEST_F(NativeMemTest, SetLiveOverwritesAndAdvancesMax) {