-
Notifications
You must be signed in to change notification settings - Fork 344
Fix Netty 4.1 server span completion for streaming responses #11941
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: ygree/fix-netty41-http11-pipelining-context-queue
Are you sure you want to change the base?
Changes from all commits
7937757
5be33eb
aaaf77d
372e9a8
0a6269f
344cbb4
c2e0b3f
4f7d924
69ec7b0
cde82d0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| package datadog.trace.instrumentation.netty41.server; | ||
|
|
||
| import static datadog.trace.instrumentation.netty41.AttributeKeys.HTTP2_STREAM_CODEC_ATTRIBUTE_KEY; | ||
| import static datadog.trace.instrumentation.netty41.AttributeKeys.WEBSOCKET_SENDER_HANDLER_CONTEXT; | ||
| import static datadog.trace.instrumentation.netty41.server.NettyHttpServerDecorator.DECORATE; | ||
|
|
||
|
|
@@ -8,25 +9,27 @@ | |
| import datadog.trace.bootstrap.instrumentation.api.AgentSpan; | ||
| import datadog.trace.bootstrap.instrumentation.websocket.HandlerContext; | ||
| import datadog.trace.instrumentation.netty41.ServerRequestContext; | ||
| import io.netty.buffer.ByteBuf; | ||
| import io.netty.buffer.ByteBufHolder; | ||
| import io.netty.channel.ChannelHandler; | ||
| import io.netty.channel.ChannelHandlerContext; | ||
| import io.netty.channel.ChannelOutboundHandlerAdapter; | ||
| import io.netty.channel.ChannelPromise; | ||
| import io.netty.channel.FileRegion; | ||
| import io.netty.handler.codec.http.HttpHeaderNames; | ||
| import io.netty.handler.codec.http.HttpHeaderValues; | ||
| import io.netty.handler.codec.http.HttpResponse; | ||
| import io.netty.handler.codec.http.HttpResponseStatus; | ||
| import io.netty.handler.codec.http.HttpUtil; | ||
| import io.netty.handler.codec.http.LastHttpContent; | ||
| import io.netty.util.concurrent.Future; | ||
|
|
||
| @ChannelHandler.Sharable | ||
| public class HttpServerResponseTracingHandler extends ChannelOutboundHandlerAdapter { | ||
| public static HttpServerResponseTracingHandler INSTANCE = new HttpServerResponseTracingHandler(); | ||
|
|
||
| @Override | ||
| public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise prm) { | ||
| if (!(msg instanceof HttpResponse)) { | ||
| ctx.write(msg, prm); | ||
| return; | ||
| } | ||
|
|
||
| final ServerRequestContext serverContext = ServerRequestContext.nextResponse(ctx.channel()); | ||
| final Context storedContext = serverContext == null ? null : serverContext.tracingContext(); | ||
| final AgentSpan span = AgentSpan.fromContext(storedContext); | ||
|
|
@@ -36,33 +39,165 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann | |
| return; | ||
| } | ||
|
|
||
| try (final ContextScope scope = storedContext.attach()) { | ||
| final HttpResponse response = (HttpResponse) msg; | ||
| if (!(msg instanceof HttpResponse) && !serverContext.isResponseStarted()) { | ||
| ctx.write(msg, prm); | ||
| return; | ||
| } | ||
|
|
||
| try (final ContextScope ignored = storedContext.attach()) { | ||
| final boolean terminalResponse = isTerminalResponse(ctx, span, serverContext, msg); | ||
| final boolean knownLengthResponseComplete = | ||
| !terminalResponse && serverContext.recordResponseBodyBytes(responseBodyBytes(msg)); | ||
| final boolean finishResponseOnWrite = terminalResponse || knownLengthResponseComplete; | ||
| final ChannelPromise writePromise = | ||
| finishResponseOnWrite && prm.isVoid() ? ctx.newPromise() : prm; | ||
| try { | ||
| ctx.write(msg, prm); | ||
| if (finishResponseOnWrite) { | ||
| ServerRequestContext.remove(ctx.channel(), serverContext); | ||
| writePromise.addListener( | ||
| future -> finishSpan(serverContext, storedContext, span, future)); | ||
| } | ||
| ctx.write(msg, writePromise); | ||
| } catch (final Throwable throwable) { | ||
| DECORATE.onError(span, throwable); | ||
| span.setHttpStatusCode(500); | ||
| span.finish(); // Finish the span manually since finishSpanOnClose was false | ||
| ServerRequestContext.remove(ctx.channel(), serverContext); | ||
| if (!finishResponseOnWrite || !writePromise.isDone()) { | ||
| DECORATE.onError(span, throwable); | ||
| span.setHttpStatusCode(500); | ||
| if (!finishResponseOnWrite) { | ||
| ServerRequestContext.remove(ctx.channel(), serverContext); | ||
| } | ||
| finishSpan(serverContext, storedContext, span); | ||
| } | ||
| throw throwable; | ||
| } | ||
| final boolean isWebsocketUpgrade = | ||
| response.status() == HttpResponseStatus.SWITCHING_PROTOCOLS | ||
| && "websocket".equals(response.headers().get(HttpHeaderNames.UPGRADE)); | ||
| } | ||
| } | ||
|
|
||
| private static boolean isTerminalResponse( | ||
| final ChannelHandlerContext ctx, | ||
| final AgentSpan span, | ||
| final ServerRequestContext serverContext, | ||
| final Object msg) { | ||
| if (msg instanceof HttpResponse) { | ||
| final HttpResponse response = (HttpResponse) msg; | ||
|
|
||
| final boolean isWebsocketUpgrade = isWebsocketUpgrade(response); | ||
| if (isInformationalResponse(response) && !isWebsocketUpgrade) { | ||
| return false; | ||
|
ygree marked this conversation as resolved.
|
||
| } | ||
|
|
||
| if (isWebsocketUpgrade) { | ||
| ctx.channel() | ||
| .attr(WEBSOCKET_SENDER_HANDLER_CONTEXT) | ||
| .set(new HandlerContext.Sender(span, ctx.channel().id().asShortText())); | ||
| } | ||
| if (response.status() != HttpResponseStatus.CONTINUE | ||
| && (response.status() != HttpResponseStatus.SWITCHING_PROTOCOLS || isWebsocketUpgrade)) { | ||
| DECORATE.onResponse(span, response); | ||
| DECORATE.beforeFinish(scope.context()); | ||
| span.finish(); // Finish the span manually since finishSpanOnClose was false | ||
| ServerRequestContext.remove(ctx.channel(), serverContext); | ||
| DECORATE.onResponse(span, response); | ||
| serverContext.markResponseStarted(); | ||
| if (isBodylessResponse(serverContext, response) || isWebsocketUpgrade) { | ||
| return true; | ||
| } | ||
| final long contentLength = contentLength(response); | ||
| if (contentLength > 0 && !HttpUtil.isTransferEncodingChunked(response)) { | ||
| serverContext.setResponseContentLength(contentLength); | ||
| } | ||
| // HTTP/1.x responses with neither a Content-Length nor chunked transfer-encoding are | ||
| // delimited by the connection closing. HTTP/2 streams translated through | ||
| // Http2StreamFrameToHttpObjectCodec are delimited by END_STREAM/LastHttpContent instead. | ||
| if (isCloseDelimitedHttp1Response(ctx, response)) { | ||
| serverContext.markResponseCloseDelimited(); | ||
| return false; | ||
| } | ||
| if (msg instanceof LastHttpContent) { | ||
| return true; | ||
| } | ||
| return false; | ||
| } | ||
| return serverContext.isResponseStarted() | ||
| && !serverContext.isResponseCloseDelimited() | ||
| && msg instanceof LastHttpContent; | ||
| } | ||
|
|
||
| private static boolean isInformationalResponse(final HttpResponse response) { | ||
| final int statusCode = response.status().code(); | ||
| return statusCode >= 100 && statusCode < 200; | ||
| } | ||
|
|
||
| private static boolean isWebsocketUpgrade(final HttpResponse response) { | ||
| return response.status().code() == HttpResponseStatus.SWITCHING_PROTOCOLS.code() | ||
| && response | ||
| .headers() | ||
| .containsValue(HttpHeaderNames.UPGRADE, HttpHeaderValues.WEBSOCKET, true); | ||
|
Comment on lines
+124
to
+128
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
If a server constructs a valid websocket upgrade with a distinct Useful? React with 👍 / 👎.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The Netty 4.1 websocket upgrade check is now value-based for status |
||
| } | ||
|
|
||
| private static boolean isBodylessResponse( | ||
| final ServerRequestContext serverContext, final HttpResponse response) { | ||
| final int statusCode = response.status().code(); | ||
| return serverContext.isHeadRequest() | ||
| || statusCode == 204 | ||
| || statusCode == 205 | ||
| || statusCode == 304 | ||
| || (serverContext.isConnectRequest() && statusCode >= 200 && statusCode < 300) | ||
| || (contentLength(response) == 0 && !HttpUtil.isTransferEncodingChunked(response)); | ||
| } | ||
|
|
||
| private static boolean hasKnownBodyLength(final HttpResponse response) { | ||
| return contentLength(response) >= 0 || HttpUtil.isTransferEncodingChunked(response); | ||
| } | ||
|
|
||
| private static boolean isCloseDelimitedHttp1Response( | ||
| final ChannelHandlerContext ctx, final HttpResponse response) { | ||
| return !hasKnownBodyLength(response) | ||
| && !Boolean.TRUE.equals(ctx.channel().attr(HTTP2_STREAM_CODEC_ATTRIBUTE_KEY).get()); | ||
| } | ||
|
|
||
| private static long responseBodyBytes(final Object msg) { | ||
| if (msg instanceof ByteBuf) { | ||
| return ((ByteBuf) msg).readableBytes(); | ||
| } | ||
| if (msg instanceof ByteBufHolder) { | ||
| return ((ByteBufHolder) msg).content().readableBytes(); | ||
| } | ||
| if (msg instanceof FileRegion) { | ||
| return ((FileRegion) msg).count(); | ||
| } | ||
| return 0; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the response {@code Content-Length}, or {@code -1} when it is absent or malformed. A | ||
| * malformed value is left for Netty's encoder to reject rather than failing the write from the | ||
| * tracing handler. | ||
| */ | ||
| private static long contentLength(final HttpResponse response) { | ||
| try { | ||
| return HttpUtil.getContentLength(response, -1L); | ||
| } catch (final NumberFormatException e) { | ||
| return -1L; | ||
| } | ||
| } | ||
|
|
||
| private static void finishSpan( | ||
| final ServerRequestContext serverContext, | ||
| final Context storedContext, | ||
| final AgentSpan span, | ||
| final Future<?> future) { | ||
| if (!future.isSuccess()) { | ||
| DECORATE.onError(span, future.cause()); | ||
| span.setHttpStatusCode(500); | ||
| } | ||
| finishSpan(serverContext, storedContext, span); | ||
| } | ||
|
|
||
| private static void finishSpan( | ||
| final ServerRequestContext serverContext, final Context storedContext, final AgentSpan span) { | ||
| beforeFinish(serverContext, storedContext); | ||
| span.finish(); // Finish the span manually since finishSpanOnClose was false | ||
| } | ||
|
|
||
| private static void beforeFinish( | ||
| final ServerRequestContext serverContext, final Context storedContext) { | ||
| if (!serverContext.isBeforeFinishCalled()) { | ||
| serverContext.markBeforeFinishCalled(); | ||
| DECORATE.beforeFinish(storedContext); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a server writes headers with
Content-Length, streams the body asByteBuf/FileRegion, and then closes the connection instead of sending aLastHttpContentsentinel, the response can be fully sent even thoughresponseCloseDelimitedis false. This close path will mark every such span as an incomplete-response error, which regresses valid non-keepalive/file responses that complete by closing after the known-length body is written.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It now treats known-length responses as complete once the declared body bytes are successfully written, so closing the connection afterward no longer marks the span as an incomplete-response error. Chunked/incomplete known-length closes still error as expected.
Tests passed for
NettyChunkedResponseSpanTest; changes are not committed.