Skip to content

Commit 604409a

Browse files
georgidhristovgeorgidhristov
committed
feat(redaction): add configurable sensitive data redaction
1 parent 97fb43b commit 604409a

9 files changed

Lines changed: 403 additions & 14 deletions

File tree

‎DebugProbe.AspNetCore.Tests/Configuration/DebugProbeOptionsTests.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ public void Defaults_work_correctly()
1717
Assert.Null(options.AllowLocalCompareTargets);
1818
Assert.False(options.AllowUiInProduction);
1919
Assert.Empty(options.IgnorePaths);
20+
Assert.Equal(["Authorization", "Cookie", "Set-Cookie"], options.RedactedHeaders);
21+
Assert.Empty(options.RedactedQueryParameters);
22+
Assert.Empty(options.RedactedJsonFields);
23+
Assert.Equal("[REDACTED]", options.RedactionText);
2024
}
2125

2226
[Fact]
@@ -30,6 +34,10 @@ public void Custom_options_are_registered_and_used()
3034
options.MaxBodyCaptureSizeKb = 4;
3135
options.AllowLocalCompareTargets = true;
3236
options.IgnorePaths = ["/health"];
37+
options.RedactedHeaders = ["X-Api-Key"];
38+
options.RedactedQueryParameters = ["token"];
39+
options.RedactedJsonFields = ["password"];
40+
options.RedactionText = "***";
3341
});
3442

3543
using var provider = services.BuildServiceProvider();
@@ -40,6 +48,10 @@ public void Custom_options_are_registered_and_used()
4048
Assert.Equal(4, options.MaxBodyCaptureSizeKb);
4149
Assert.True(options.AllowLocalCompareTargets);
4250
Assert.Equal(["/health"], options.IgnorePaths);
51+
Assert.Equal(["X-Api-Key"], options.RedactedHeaders);
52+
Assert.Equal(["token"], options.RedactedQueryParameters);
53+
Assert.Equal(["password"], options.RedactedJsonFields);
54+
Assert.Equal("***", options.RedactionText);
4355
Assert.NotNull(store.Environment);
4456
}
4557
}

‎DebugProbe.AspNetCore.Tests/Handlers/DebugProbeHttpClientHandlerTests.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,49 @@ public async Task Captures_outgoing_http_call_on_active_trace()
5252
Assert.Contains("\"ok\": true", outgoing.ResponseBody);
5353
}
5454

55+
[Fact]
56+
public async Task Redacts_configured_outgoing_url_headers_and_json_fields()
57+
{
58+
var entry = new DebugEntry();
59+
var context = new DefaultHttpContext();
60+
context.Items["DebugProbeEntry"] = entry;
61+
62+
using var handler = new DebugProbeHttpClientHandler(
63+
new HttpContextAccessor { HttpContext = context },
64+
new DebugProbeOptions
65+
{
66+
RedactedHeaders = ["Authorization", "X-Api-Key"],
67+
RedactedQueryParameters = ["token"],
68+
RedactedJsonFields = ["password", "refreshToken"]
69+
})
70+
{
71+
InnerHandler = new StubHandler(_ =>
72+
{
73+
var response = new HttpResponseMessage(HttpStatusCode.OK)
74+
{
75+
Content = new StringContent("{\"refreshToken\":\"response-token\"}", Encoding.UTF8, "application/json")
76+
};
77+
return response;
78+
})
79+
};
80+
81+
using var client = new HttpClient(handler);
82+
using var request = new HttpRequestMessage(HttpMethod.Post, "https://api.example.test/orders?token=query-secret&safe=yes")
83+
{
84+
Content = new StringContent("{\"password\":\"body-secret\"}", Encoding.UTF8, "application/json")
85+
};
86+
request.Headers.Add("X-Api-Key", "header-secret");
87+
88+
await client.SendAsync(request);
89+
90+
var outgoing = Assert.Single(entry.OutgoingRequests);
91+
92+
Assert.Equal("https://api.example.test/orders?token=[REDACTED]&safe=yes", outgoing.Url);
93+
Assert.Equal("[REDACTED]", outgoing.RequestHeaders["X-Api-Key"]);
94+
Assert.Contains("\"password\": \"[REDACTED]\"", outgoing.RequestBody);
95+
Assert.Contains("\"refreshToken\": \"[REDACTED]\"", outgoing.ResponseBody);
96+
}
97+
5598
private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> send) : HttpMessageHandler
5699
{
57100
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)

‎DebugProbe.AspNetCore.Tests/Middleware/RedactionTests.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
using System.Text;
2+
using DebugProbe.AspNetCore.Options;
3+
using DebugProbe.AspNetCore.Tests.Infrastructure;
4+
5+
namespace DebugProbe.AspNetCore.Tests.Middleware;
6+
7+
public class RedactionTests
8+
{
9+
[Fact]
10+
public async Task Redacts_configured_headers_query_parameters_and_json_fields()
11+
{
12+
await using var app = await DebugProbeTestApp.CreateAsync(
13+
endpoints => endpoints.MapPost("/orders", async context =>
14+
{
15+
context.Response.Headers["X-Session-Token"] = "response-secret";
16+
context.Response.ContentType = "application/json";
17+
await context.Response.WriteAsync("{\"ok\":true,\"refreshToken\":\"response-token\"}");
18+
}),
19+
ConfigureRedaction);
20+
21+
using var request = new HttpRequestMessage(HttpMethod.Post, "/orders?api_key=query-secret&safe=yes")
22+
{
23+
Content = new StringContent(
24+
"{\"name\":\"Ada\",\"password\":\"secret\",\"profile\":{\"refreshToken\":\"nested-token\"}}",
25+
Encoding.UTF8,
26+
"application/json")
27+
};
28+
request.Headers.Add("X-Api-Key", "header-secret");
29+
30+
await app.Client.SendAsync(request);
31+
32+
var entry = app.SingleEntry;
33+
34+
Assert.Equal("[REDACTED]", entry.RequestHeaders["X-Api-Key"]);
35+
Assert.Equal("[REDACTED]", entry.ResponseHeaders["X-Session-Token"]);
36+
Assert.Equal("?api_key=[REDACTED]&safe=yes", entry.Query);
37+
Assert.Equal("http://localhost/orders?api_key=[REDACTED]&safe=yes", entry.RequestUrl);
38+
Assert.Contains("\"password\":\"[REDACTED]\"", entry.RequestBody);
39+
Assert.Contains("\"refreshToken\":\"[REDACTED]\"", entry.RequestBody);
40+
Assert.Contains("\"refreshToken\":\"[REDACTED]\"", entry.ResponseBody);
41+
Assert.DoesNotContain("secret", entry.RequestBody);
42+
Assert.DoesNotContain("response-token", entry.ResponseBody);
43+
}
44+
45+
[Fact]
46+
public async Task Leaves_invalid_json_body_unchanged_when_json_fields_are_configured()
47+
{
48+
await using var app = await DebugProbeTestApp.CreateAsync(
49+
endpoints => endpoints.MapPost("/text", async context =>
50+
{
51+
context.Response.ContentType = "text/plain";
52+
await context.Response.WriteAsync("ok");
53+
}),
54+
options => options.RedactedJsonFields = ["password"]);
55+
56+
await app.Client.PostAsync(
57+
"/text",
58+
new StringContent("password=secret", Encoding.UTF8, "text/plain"));
59+
60+
Assert.Equal("password=secret", app.SingleEntry.RequestBody);
61+
}
62+
63+
private static void ConfigureRedaction(DebugProbeOptions options)
64+
{
65+
options.RedactedHeaders =
66+
[
67+
..options.RedactedHeaders,
68+
"X-Api-Key",
69+
"X-Session-Token"
70+
];
71+
72+
options.RedactedQueryParameters = ["api_key"];
73+
options.RedactedJsonFields = ["password", "refreshToken"];
74+
}
75+
}

‎DebugProbe.AspNetCore/Handlers/DebugProbeHttpClientHandler.cs‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
7171
{
7272
Method = request.Method.Method,
7373

74-
Url = request.RequestUri?.ToString() ?? string.Empty,
74+
Url = RedactionUtils.RedactUrl(request.RequestUri?.ToString(), _options),
7575

7676
StatusCode = response != null ? (int)response.StatusCode : null,
7777

@@ -83,9 +83,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
8383

8484
IsSuccessStatusCode = response?.IsSuccessStatusCode ?? false,
8585

86-
RequestHeaders = request.Headers.ToDictionary(x => x.Key, x => HeaderUtils.RedactIfSensitive(x.Key, string.Join(", ", x.Value))),
86+
RequestHeaders = request.Headers.ToDictionary(x => x.Key, x => RedactionUtils.RedactHeader(x.Key, string.Join(", ", x.Value), _options)),
8787

88-
ResponseHeaders = response != null ? response.Headers.ToDictionary(x => x.Key, x => HeaderUtils.RedactIfSensitive(x.Key, string.Join(", ", x.Value))) : []
88+
ResponseHeaders = response != null ? response.Headers.ToDictionary(x => x.Key, x => RedactionUtils.RedactHeader(x.Key, string.Join(", ", x.Value), _options)) : []
8989
};
9090

9191
if (request.Content != null)
@@ -96,7 +96,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
9696
{
9797
var body = await request.Content.ReadAsStringAsync();
9898

99-
outgoing.RequestBody = JsonUtils.Format(HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes));
99+
outgoing.RequestBody = JsonUtils.Format(RedactionUtils.RedactJsonFields(
100+
HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes),
101+
_options));
100102
}
101103
}
102104

@@ -108,7 +110,9 @@ private async Task CaptureRequest(HttpRequestMessage request, HttpResponseMessag
108110
{
109111
var body = await response.Content.ReadAsStringAsync();
110112

111-
outgoing.ResponseBody = JsonUtils.Format(HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes));
113+
outgoing.ResponseBody = JsonUtils.Format(RedactionUtils.RedactJsonFields(
114+
HttpContentUtils.Trim(body, _options.MaxBodyCaptureSizeBytes),
115+
_options));
112116
}
113117
}
114118

‎DebugProbe.AspNetCore/Internal/Utils/RedactionUtils.cs‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
using System.Text.Encodings.Web;
2+
using System.Text.Json;
3+
using System.Text.Json.Nodes;
4+
using DebugProbe.AspNetCore.Options;
5+
6+
namespace DebugProbe.AspNetCore.Internal.Utils;
7+
8+
internal static class RedactionUtils
9+
{
10+
public static string RedactHeader(string name, string value, DebugProbeOptions options)
11+
{
12+
return IsMatch(name, options.RedactedHeaders) ? options.RedactionText : value;
13+
}
14+
15+
public static string RedactQueryString(string? queryString, DebugProbeOptions options)
16+
{
17+
if (string.IsNullOrEmpty(queryString) || options.RedactedQueryParameters.Length == 0)
18+
{
19+
return queryString ?? string.Empty;
20+
}
21+
22+
var prefix = queryString.StartsWith('?') ? "?" : string.Empty;
23+
var query = prefix.Length == 0 ? queryString : queryString[1..];
24+
25+
return prefix + RedactQuery(query, options);
26+
}
27+
28+
public static string RedactUrl(string? url, DebugProbeOptions options)
29+
{
30+
if (string.IsNullOrEmpty(url) || options.RedactedQueryParameters.Length == 0)
31+
{
32+
return url ?? string.Empty;
33+
}
34+
35+
var fragmentIndex = url.IndexOf('#');
36+
var fragment = fragmentIndex >= 0 ? url[fragmentIndex..] : string.Empty;
37+
var withoutFragment = fragmentIndex >= 0 ? url[..fragmentIndex] : url;
38+
39+
var queryIndex = withoutFragment.IndexOf('?');
40+
if (queryIndex < 0)
41+
{
42+
return url;
43+
}
44+
45+
var beforeQuery = withoutFragment[..queryIndex];
46+
var query = withoutFragment[(queryIndex + 1)..];
47+
48+
return $"{beforeQuery}?{RedactQuery(query, options)}{fragment}";
49+
}
50+
51+
public static string RedactJsonFields(string? body, DebugProbeOptions options)
52+
{
53+
if (string.IsNullOrWhiteSpace(body) || options.RedactedJsonFields.Length == 0)
54+
{
55+
return body ?? string.Empty;
56+
}
57+
58+
try
59+
{
60+
var node = JsonNode.Parse(body);
61+
if (node is null)
62+
{
63+
return body;
64+
}
65+
66+
RedactNode(node, options);
67+
68+
return JsonSerializer.Serialize(
69+
node,
70+
new JsonSerializerOptions
71+
{
72+
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
73+
});
74+
}
75+
catch
76+
{
77+
return body;
78+
}
79+
}
80+
81+
private static string RedactQuery(string query, DebugProbeOptions options)
82+
{
83+
if (query.Length == 0)
84+
{
85+
return query;
86+
}
87+
88+
var parts = query.Split('&');
89+
90+
for (var i = 0; i < parts.Length; i++)
91+
{
92+
var part = parts[i];
93+
var equalsIndex = part.IndexOf('=');
94+
var name = equalsIndex >= 0 ? part[..equalsIndex] : part;
95+
96+
if (!IsMatch(DecodeQueryValue(name), options.RedactedQueryParameters))
97+
{
98+
continue;
99+
}
100+
101+
parts[i] = equalsIndex >= 0? $"{name}={options.RedactionText}" : $"{name}={options.RedactionText}";
102+
}
103+
104+
return string.Join("&", parts);
105+
}
106+
107+
private static void RedactNode(JsonNode node, DebugProbeOptions options)
108+
{
109+
if (node is JsonObject jsonObject)
110+
{
111+
foreach (var property in jsonObject.ToList())
112+
{
113+
if (IsMatch(property.Key, options.RedactedJsonFields))
114+
{
115+
jsonObject[property.Key] = options.RedactionText;
116+
continue;
117+
}
118+
119+
if (property.Value is not null)
120+
{
121+
RedactNode(property.Value, options);
122+
}
123+
}
124+
125+
return;
126+
}
127+
128+
if (node is JsonArray jsonArray)
129+
{
130+
foreach (var item in jsonArray)
131+
{
132+
if (item is not null)
133+
{
134+
RedactNode(item, options);
135+
}
136+
}
137+
}
138+
}
139+
140+
private static string DecodeQueryValue(string value)
141+
{
142+
return Uri.UnescapeDataString(value.Replace("+", " "));
143+
}
144+
145+
private static bool IsMatch(string value, string[] candidates)
146+
{
147+
return candidates.Any(candidate =>
148+
string.Equals(value, candidate, StringComparison.OrdinalIgnoreCase));
149+
}
150+
}

‎DebugProbe.AspNetCore/Middleware/DebugProbeMiddleware.cs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public async Task Invoke(HttpContext context, DebugEntryStore store)
118118

119119
entry.Path = context.Request.Path;
120120

121-
entry.Query = context.Request.QueryString.ToString();
121+
entry.Query = RedactionUtils.RedactQueryString(context.Request.QueryString.ToString(), _options);
122122

123123
entry.StatusCode = statusCode;
124124

@@ -133,20 +133,26 @@ public async Task Invoke(HttpContext context, DebugEntryStore store)
133133
entry.RequestHeaders =
134134
context.Request.Headers.ToDictionary(
135135
x => x.Key,
136-
x => HeaderUtils.RedactIfSensitive(x.Key, x.Value.ToString()));
136+
x => RedactionUtils.RedactHeader(x.Key, x.Value.ToString(), _options));
137137

138138
entry.RequestUrl =
139-
$"{context.Request.Scheme}://{context.Request.Host}" +
140-
$"{context.Request.Path}{context.Request.QueryString}";
139+
RedactionUtils.RedactUrl(
140+
$"{context.Request.Scheme}://{context.Request.Host}" +
141+
$"{context.Request.Path}{context.Request.QueryString}",
142+
_options);
141143

142-
entry.RequestBody = HttpContentUtils.Trim(requestBody, maxBodySize);
144+
entry.RequestBody = RedactionUtils.RedactJsonFields(
145+
HttpContentUtils.Trim(requestBody, maxBodySize),
146+
_options);
143147

144-
entry.ResponseBody = HttpContentUtils.Trim(responseBody, maxBodySize);
148+
entry.ResponseBody = RedactionUtils.RedactJsonFields(
149+
HttpContentUtils.Trim(responseBody, maxBodySize),
150+
_options);
145151

146152
entry.ResponseHeaders =
147153
context.Response.Headers.ToDictionary(
148154
x => x.Key,
149-
x => HeaderUtils.RedactIfSensitive(x.Key, x.Value.ToString()));
155+
x => RedactionUtils.RedactHeader(x.Key, x.Value.ToString(), _options));
150156

151157
store.Add(entry);
152158
}

0 commit comments

Comments
 (0)