feat(profiler): native malloc/socket stack capture with profiler-hook frame skipping

ddprof-lib/src/main/cpp/codeCache.h

@@ -55,7 +55,7 @@ enum Mark {
     MARK_VM_RUNTIME = 1,
     MARK_INTERPRETER = 2,
     MARK_COMPILER_ENTRY = 3,
-    MARK_ASYNC_PROFILER = 4, // async-profiler internals such as native hooks.
+    MARK_JAVA_PROFILER = 4, // java-profiler internals such as native hooks.
     MARK_THREAD_ENTRY = 5,   // Thread entry points (thread_native_entry, JavaThread::, etc.)
 };
 
@@ -218,7 +218,7 @@ class CodeCache {
    * Mark symbols matching the predicate with the given mark value.
    *
    * This is called during profiler initialization to mark JVM internal functions
-   * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_ASYNC_PROFILER).
+   * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_JAVA_PROFILER).
    */
   template <typename NamePredicate>
   inline void mark(NamePredicate predicate, char value) {

ddprof-lib/src/main/cpp/counters.h

@@ -119,6 +119,8 @@
   X(JVMTI_STACKS_INIT_OK, "jvmti_stacks_init_ok")                             \
   X(JVMTI_STACKS_INIT_FAILED, "jvmti_stacks_init_failed")                     \
   X(JVMTI_STACKS_REQUESTED, "jvmti_stacks_requested")                         \
+  X(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND, "native_trace_hook_prefix_not_found") \
+  X(NATIVE_HOOK_MARK_RESOLVE_FAILED, "native_hook_mark_resolve_failed")       \
   X(JVMTI_STACKS_FAILED_WRONG_PHASE, "jvmti_stacks_failed_wrong_phase")       \
   X(JVMTI_STACKS_FAILED_OTHER, "jvmti_stacks_failed_other")                  \
   /* Delegated stacks dropped at slot-lock. Rec-lock drops from all recording  \

ddprof-lib/src/main/cpp/event.h

@@ -34,6 +34,7 @@ enum EventType {
     EXECUTION_SAMPLE,
     WALL_CLOCK_SAMPLE,
     MALLOC_SAMPLE,
+    SOCKET_SAMPLE,
     INSTRUMENTED_METHOD,
     METHOD_TRACE,
     ALLOC_SAMPLE,

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -126,6 +126,8 @@ inline EventType eventTypeFromBCI(jint bci_type) {
             return PARK_SAMPLE;
         case BCI_NATIVE_MALLOC:
             return MALLOC_SAMPLE;
+        case BCI_NATIVE_SOCKET:
+            return SOCKET_SAMPLE;
         default:
             // For unknown or invalid BCI types, default to EXECUTION_SAMPLE
             // This maintains backward compatibility and prevents undefined behavior
@@ -305,8 +307,14 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
     uintptr_t saved_anchor_fp = 0;
     bool anchor_recovery_used = false;
 
+    // Set once the MARK_JAVA_PROFILER hook boundary is found for a
+    // malloc/socket sample — mirrors skip_hook_prefix/skipping in
+    // Profiler::convertNativeTrace so the same "boundary never found"
+    // condition is observable from walkVM.
+    bool hook_boundary_found = false;
+
     // Show extended frame types and stub frames for execution-type events
-    bool details = event_type <= MALLOC_SAMPLE || features.mixed;
+    bool details = event_type <= SOCKET_SAMPLE || features.mixed;
 
     if (details && vm_thread != NULL && VMThread::isJavaThread(vm_thread)) {
         anchor = vm_thread->anchor();
@@ -681,18 +689,16 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         } else {
             // Resolve native frame (may use remote symbolication if enabled)
             Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index);
-            if (resolution.is_marked) {
-                // This is a marked C++ interpreter frame, terminate scan
-                break;
-            }
-            const char* method_name = resolution.method_name;
-            int frame_bci = resolution.bci;
-            char mark;
-            if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) {
-                if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) {
-                    // Skip all internal frames above malloc_hook functions, leave the hook itself
+            if (resolution.is_marked()) {
+                if (resolution.mark == MARK_JAVA_PROFILER &&
+                    isHookPrefixedSample(event_type)) {
+                    // Discard frames captured above the malloc/socket hook boundary,
+                    // excluding the hook's own frame, and resume from the real
+                    // caller above it — mirrors the FP/DWARF skip-prefix logic in
+                    // Profiler::convertNativeTrace.
+                    hook_boundary_found = true;
                     depth = 0;
-                } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
+                } else if (resolution.mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) {
                     // Insert current compile task as a pseudo Java frame
                     VMMethod* method = vm_thread->compiledMethod();
                     if (method != nullptr) {
@@ -701,13 +707,19 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                             fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id, method);
                         }
                     }
-                } else if (mark == MARK_THREAD_ENTRY) {
+                } else if (resolution.mark == MARK_THREAD_ENTRY) {
                     // Thread entry point detected via pre-computed mark - this is the root frame
-                    // No need for expensive symbol resolution, just stop unwinding
                     Counters::increment(THREAD_ENTRY_MARK_DETECTIONS);
                     break;
+                } else {
+                    // Other marks (VM runtime / interpreter) terminate the scan.
+                    break;
                 }
-            } else if (method_name == NULL && details && !anchor_recovery_used
+                goto dwarf_unwind;
+            }
+            const char* method_name = resolution.method_name;
+            int frame_bci = resolution.bci;
+            if (method_name == NULL && details && !anchor_recovery_used
                        && profiler->findLibraryByAddress(pc) == NULL) {
                 // Try anchor recovery — prefer live anchor, fall back to saved data
                 anchor_recovery_used = true;
@@ -945,6 +957,12 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         Counters::increment(WALKVM_DEPTH_ZERO);
     }
 
+    if (isHookPrefixedSample(event_type) && !hook_boundary_found) {
+        // The malloc/socket hook boundary was never found in this walk;
+        // mirrors Profiler::convertNativeTrace's NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND.
+        Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+    }
+
     if (truncated) {
         if (depth > max_depth) {
             *truncated = true;
@@ -1198,7 +1216,7 @@ int HotspotSupport::walkJavaStack(StackWalkRequest& request) {
   int java_frames = 0;
   if (features.mixed) {
     java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
-  } else if (request.event_type == BCI_NATIVE_MALLOC || request.event_type == BCI_NATIVE_SOCKET) {
+  } else if (isHookPrefixedSample(request.event_type)) {
     if (cstack >= CSTACK_VM) {
       java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated);
     } else {

ddprof-lib/src/main/cpp/jfrMetadata.cpp

@@ -303,8 +303,8 @@ void JfrMetadata::initialize(
               << field("name", T_STRING, "Name")
               << field("count", T_LONG, "Count"))
 
-          << (type("profiler.Malloc", T_MALLOC, "malloc")
-                  << category("Java Virtual Machine", "Native Memory")
+          << (type("datadog.NativeMemoryAllocation", T_MALLOC, "Native Memory Allocation")
+                  << category("Datadog", "Profiling")
                   << field("startTime", T_LONG, "Start Time", F_TIME_TICKS)
                   << field("eventThread", T_THREAD, "Event Thread", F_CPOOL)
                   << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL)

ddprof-lib/src/main/cpp/mallocTracer.cpp

@@ -231,7 +231,7 @@ bool MallocHooker::initialize() {
                 || strcmp(s, "posix_memalign_hook") == 0
                 || strcmp(s, "aligned_alloc_hook") == 0;
         },
-        MARK_ASYNC_PROFILER);
+        MARK_JAVA_PROFILER);
 
     splitmix64_seed(TSC::ticks());
     _initialized = true;

ddprof-lib/src/main/cpp/nativeSocketSampler.cpp

@@ -7,8 +7,11 @@
 
 #if defined(__linux__)
 
+#include "codeCache.h"
 #include "common.h"
+#include "counters.h"
 #include "flightRecorder.h"
+#include "libraries.h"
 #include "libraryPatcher.h"
 #include "log.h"
 #include "os.h"
@@ -27,6 +30,30 @@
 static thread_local PoissonSampler _send_sampler;
 static thread_local PoissonSampler _recv_sampler;
 
+// Marks the hook wrapper's own symbol as MARK_JAVA_PROFILER so native call-stack
+// unwinding (Profiler::convertNativeTrace) can recognize the boundary between
+// profiler-internal frames and the real caller, mirroring MallocHooker::initialize().
+// Resolved by address (not by symbol-name predicate) because these are mangled
+// C++ static member functions, unlike malloc_hook's extern "C" free functions.
+// Returns false if the symbol could not be resolved/marked, in which case the
+// hook boundary is never recognized and every socket sample's native stack
+// comes back empty (see NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND).
+static bool markJavaProfilerHook(void* fn_addr) {
+    CodeCache* lib = Libraries::instance()->findLibraryByAddress(fn_addr);
+    if (lib == nullptr) {
+        Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+        return false;
+    }
+    const char* name = nullptr;
+    lib->binarySearch(fn_addr, &name);
+    if (name == nullptr) {
+        Counters::increment(NATIVE_HOOK_MARK_RESOLVE_FAILED);
+        return false;
+    }
+    NativeFunc::set_mark(name, MARK_JAVA_PROFILER);
+    return true;
+}
+
 // Debug-only hook-fire counters, paired with TEST_LOG (common.h). Gated at
 // compile time to keep release hot paths free of cross-thread atomic writes.
 #ifdef DEBUG
@@ -386,6 +413,18 @@ Error NativeSocketSampler::start(Arguments &args) {
     TEST_LOG("NativeSocketSampler::start interval_ticks=%ld tsc_freq=%llu",
              init_interval, (unsigned long long)TSC::frequency());
 #endif
+    bool hooks_marked = markJavaProfilerHook((void*)&NativeSocketSampler::send_hook);
+    hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::recv_hook);
+    hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::write_hook);
+    hooks_marked &= markJavaProfilerHook((void*)&NativeSocketSampler::read_hook);
+    if (!hooks_marked) {
+        // Not fatal: hooks are still installed and sampling still works, but
+        // native stacks for socket samples will come back empty because the
+        // hook boundary frame can't be recognized during unwinding.
+        Log::warn("NativeSocketSampler: failed to mark one or more hook symbols; "
+                  "native call stacks for socket samples may be empty");
+    }
+
     if (!LibraryPatcher::patch_socket_functions()) {
         return Error("failed to install native socket hooks (dlsym returned NULL)");
     }

ddprof-lib/src/main/cpp/profiler.cpp

@@ -302,6 +302,7 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
   if (_cstack == CSTACK_NO ||
       (event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) ||
       (event_type != BCI_CPU && event_type != BCI_WALL &&
+       !isHookPrefixedSample(event_type) &&
        _cstack == CSTACK_DEFAULT)) {
     return 0;
   }
@@ -326,7 +327,9 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames,
                                          java_ctx, truncated);
   }
 
-  return convertNativeTrace(native_frames, callchain, frames, lock_index);
+  bool skip_hook_prefix = isHookPrefixedSample(event_type);
+  return convertNativeTrace(native_frames, callchain, frames, lock_index,
+                            skip_hook_prefix);
 }
 
 /**
@@ -383,24 +386,27 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
     char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
     if (mark != 0) {
-      return {nullptr, BCI_NATIVE_FRAME, true};  // Marked - stop processing
+      return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark);  // Marked - caller dispatches on mark
     }
 
     // Pack remote symbolication data using utility struct
     uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
     uint32_t lib_index = (uint32_t)lib->libIndex();
     unsigned long packed = RemoteFramePacker::pack(pc_offset, mark, lib_index);
 
-    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
   }
 
   // Traditional symbol resolution
   const char *method_name = nullptr;
   if (lib != nullptr) {
     lib->binarySearch((void*)pc, &method_name);
   }
-  if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
-    return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, true);
+  if (method_name != nullptr) {
+    char mark = NativeFunc::read_mark(method_name);
+    if (mark != 0) {
+      return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, mark);
+    }
   }
 
   // No symbol but known library: pack for library-relative identification.
@@ -410,10 +416,10 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
     uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase();
     uint32_t lib_index = (uint32_t)lib->libIndex();
     unsigned long packed = RemoteFramePacker::pack(pc_offset, 0, lib_index);
-    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false);
+    return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE);
   }
 
-  return NativeFrameResolution(method_name, BCI_NATIVE_FRAME, false);
+  return NativeFrameResolution(method_name, BCI_NATIVE_FRAME);
 }
 
 /**
@@ -426,9 +432,16 @@ Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t
  * marked frames (JVM internals) that should terminate the stack walk.
  */
 int Profiler::convertNativeTrace(int native_frames, const void **callchain,
-                                 ASGCT_CallFrame *frames, int lock_index) {
+                                 ASGCT_CallFrame *frames, int lock_index,
+                                 bool skip_hook_prefix) {
   int depth = 0;
   void* prev_identifier = NULL;  // Can be jmethodID or frame pointer for remote
+  // skip_hook_prefix: the walk started inside profiler-internal code (e.g. the
+  // malloc/socket hook call chain), not at an interrupted user PC. Discard frames
+  // until the hook wrapper's own MARK_JAVA_PROFILER-marked frame is reached, then
+  // resume normally from the real caller. Other mark kinds still terminate the
+  // scan immediately, same as the non-skipping case.
+  bool skipping = skip_hook_prefix;
 
   for (int i = 0; i < native_frames; i++) {
     uintptr_t pc = (uintptr_t)callchain[i];
@@ -444,9 +457,20 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
         char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0;
 
         if (mark != 0) {
+          if (skip_hook_prefix && mark == MARK_JAVA_PROFILER) {
+            depth = 0;
+            skipping = false;
+            continue;
+          }
+          if (skipping) {
+            // A non-MARK_JAVA_PROFILER mark terminated the scan before the
+            // hook boundary was ever found; the sample has no native stack.
+            Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+          }
           // Terminate scan at marked frame
           return depth;
         }
+        if (skipping) continue;
 
         // Populate remote frame inline - no allocation needed!
         // Pass the mark we already retrieved to avoid duplicate binarySearch
@@ -464,10 +488,24 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
 
     // Fallback: Traditional symbol resolution
     const char *method_name = findNativeMethod((void*)pc);
-    if (method_name != nullptr && NativeFunc::is_marked(method_name)) {
-      // Terminate scan at marked frame
-      return depth;
+    if (method_name != nullptr) {
+      char mark = NativeFunc::read_mark(method_name);
+      if (mark != 0) {
+        if (skip_hook_prefix && mark == MARK_JAVA_PROFILER) {
+          depth = 0;
+          skipping = false;
+          continue;
+        }
+        if (skipping) {
+          // A non-MARK_JAVA_PROFILER mark terminated the scan before the
+          // hook boundary was ever found; the sample has no native stack.
+          Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+        }
+        // Terminate scan at marked frame
+        return depth;
+      }
     }
+    if (skipping) continue;
 
     // Store standard frame
     jmethodID current_method = (jmethodID)method_name;
@@ -481,6 +519,11 @@ int Profiler::convertNativeTrace(int native_frames, const void **callchain,
     }
   }
 
+  if (skipping) {
+    // The hook-boundary (MARK_JAVA_PROFILER) frame was never found in the
+    // callchain; every frame was discarded and the sample has no native stack.
+    Counters::increment(NATIVE_TRACE_HOOK_PREFIX_NOT_FOUND);
+  }
   return depth;
 }