From 4479738a5f6dbb190c5a8741eccd2de4e28564c5 Mon Sep 17 00:00:00 2001 From: adeleon Date: Thu, 28 May 2026 15:53:38 -0300 Subject: [PATCH 1/2] feat: add model sanitization --- .github/workflows/go.yaml | 2 +- .gitignore | 2 + configstore/configstore.go | 33 ++-- configstore/configstore_test.go | 101 +++++++++++ sanitize.go | 46 +++++ sanitize_test.go | 287 ++++++++++++++++++++++++++++++++ testdata/plugins/model/param.go | 2 +- wacecore.go | 34 +++- 8 files changed, 485 insertions(+), 22 deletions(-) create mode 100644 .gitignore create mode 100644 sanitize.go create mode 100644 sanitize_test.go diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index de4e22a..c935414 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.22.2' + go-version: '1.26.2' - name: Test run: go run mage.go test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8dcfc82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.so +coverage.* \ No newline at end of file diff --git a/configstore/configstore.go b/configstore/configstore.go index 573a372..d0f1390 100644 --- a/configstore/configstore.go +++ b/configstore/configstore.go @@ -84,6 +84,7 @@ type modelPluginConfig struct { remote bool training bool TrainingData TrainingData + sanitize bool } // DecisionPluginConfig stores the configuration of a decision plugin @@ -95,12 +96,13 @@ type decisionPluginConfig struct { // ConfigStore stores all wacecore configuration from the config file. type ConfigStore struct { - ModelPlugins map[string]modelPluginConfig - DecisionPlugins map[string]decisionPluginConfig - LogPath string - LogLevel logging.LogLevel - NatsURL string - ApplicationId string + ModelPlugins map[string]modelPluginConfig + DecisionPlugins map[string]decisionPluginConfig + LogPath string + LogLevel logging.LogLevel + NatsURL string + ApplicationId string + CredentialHeaders []string } var config *ConfigStore @@ -138,6 +140,7 @@ type configFileModelPlugin struct { Remote bool Training bool TrainingData TrainingData `yaml:"training_data"` + Sanitize bool } type configFileDecisionPlugin struct { @@ -147,11 +150,12 @@ type configFileDecisionPlugin struct { } type ConfigFileData struct { - Logpath string - Loglevel string - Modelplugins []configFileModelPlugin - Decisionplugins []configFileDecisionPlugin - NatsURL string + Logpath string + Loglevel string + Modelplugins []configFileModelPlugin + Decisionplugins []configFileDecisionPlugin + NatsURL string + CredentialHeaders []string `yaml:"credential_headers"` } // IsAsync returns true if the model plugin is async @@ -169,6 +173,10 @@ func (c *ConfigStore) IsInTraining(modelID string) bool { return c.ModelPlugins[modelID].training } +func (c *ConfigStore) ShouldSanitize(modelID string) bool { + return c.ModelPlugins[modelID].sanitize +} + // CheckLogging verifies if the log path is valid func checkLogging(inConf ConfigFileData) error { // check logpath @@ -258,6 +266,7 @@ func (cs *ConfigStore) SetConfig(inConf ConfigFileData) error { modelConfig.remote = modelP.Remote modelConfig.training = modelP.Training modelConfig.TrainingData = modelP.TrainingData + modelConfig.sanitize = modelP.Sanitize if err != nil { return err } @@ -275,5 +284,7 @@ func (cs *ConfigStore) SetConfig(inConf ConfigFileData) error { cs.NatsURL = inConf.NatsURL + cs.CredentialHeaders = inConf.CredentialHeaders + return nil } diff --git a/configstore/configstore_test.go b/configstore/configstore_test.go index 01f18e4..394cc9e 100644 --- a/configstore/configstore_test.go +++ b/configstore/configstore_test.go @@ -3,6 +3,7 @@ package configstore import ( "fmt" "os" + "reflect" "testing" "gopkg.in/yaml.v3" @@ -675,6 +676,106 @@ modelplugins: } } +func TestShouldSanitize(t *testing.T) { + tests := []struct { + name string + sanitize bool + wantSanitize bool + }{ + {"sanitize omitted defaults to false", false, false}, + {"sanitize: true propagates correctly", true, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs, err := New() + if err != nil { + t.Fatal(err) + } + defer Clean() + + config := fmt.Sprintf(`--- +loglevel: ERROR +logpath: /dev/null +modelplugins: + - id: "testplugin" + path: "../testdata/plugins/model/trivial.so" + plugintype: "RequestHeaders" + sanitize: %v +`, tt.sanitize) + if err := initialize([]byte(config)); err != nil { + t.Fatalf("initialize: %v", err) + } + + if got := cs.ShouldSanitize("testplugin"); got != tt.wantSanitize { + t.Errorf("ShouldSanitize = %v, want %v", got, tt.wantSanitize) + } + }) + } +} + +func TestShouldSanitizeUnknownModel(t *testing.T) { + cs, err := New() + if err != nil { + t.Fatal(err) + } + defer Clean() + + if err := initialize(validConfig); err != nil { + t.Fatalf("initialize: %v", err) + } + + if cs.ShouldSanitize("nonexistent") { + t.Error("ShouldSanitize for unknown model ID should return false") + } +} + +func TestCredentialHeaders(t *testing.T) { + tests := []struct { + name string + config string + wantHeaders []string + }{ + { + name: "no credential_headers field defaults to nil", + config: `--- +loglevel: ERROR +logpath: /dev/null +`, + wantHeaders: nil, + }, + { + name: "credential_headers values are stored", + config: `--- +loglevel: ERROR +logpath: /dev/null +credential_headers: + - x-api-key + - x-secret-token +`, + wantHeaders: []string{"x-api-key", "x-secret-token"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs, err := New() + if err != nil { + t.Fatal(err) + } + defer Clean() + + if err := initialize([]byte(tt.config)); err != nil { + t.Fatalf("initialize: %v", err) + } + + if !reflect.DeepEqual(cs.CredentialHeaders, tt.wantHeaders) { + t.Errorf("CredentialHeaders = %v, want %v", cs.CredentialHeaders, tt.wantHeaders) + } + }) + } +} + func TestNatsURL(t *testing.T) { tests := []struct { name string diff --git a/sanitize.go b/sanitize.go new file mode 100644 index 0000000..ddf44a0 --- /dev/null +++ b/sanitize.go @@ -0,0 +1,46 @@ +package wace + +import ( + "regexp" + "slices" + "strings" + + "github.com/tilsor/ModSecIntl_wace_lib/waceapi" +) + +const passwordRegex = `(?i)((new|old|form_|)(v)?password|clave|pass(|2))=([^*&\n]*)` + +var ( + credentialRegex = regexp.MustCompile(passwordRegex) + credentialHeaders = []string{"authorization", "cookie", "set-cookie"} +) + +// sanitizeCredentials replaces password-like fields in the request body and +// known credential headers (Authorization, Cookie, Set-Cookie) with "********". +func sanitizeCredentials(p waceapi.HTTPPayload) waceapi.HTTPPayload { + p.RequestBody = string(credentialRegex.ReplaceAll([]byte(p.RequestBody), []byte("$1=********"))) + + p.URI = string(credentialRegex.ReplaceAll([]byte(p.URI), []byte("$1=********"))) + + for i := range p.RequestHeaders { + if slices.Contains(credentialHeaders, strings.ToLower(p.RequestHeaders[i].Key)) { + p.RequestHeaders[i] = waceapi.HTTPHeader{Key: p.RequestHeaders[i].Key, Value: "********"} + } + } + + for i := range p.ResponseHeaders { + if slices.Contains(credentialHeaders, strings.ToLower(p.ResponseHeaders[i].Key)) { + p.ResponseHeaders[i] = waceapi.HTTPHeader{Key: p.ResponseHeaders[i].Key, Value: "********"} + } + } + + return p +} + +// SetCredentialHeaders allows the user to change the list of headers to be sanitized. +func SetCredentialHeaders(headers []string) { + for i := range headers { + headers[i] = strings.ToLower(headers[i]) + } + credentialHeaders = headers +} diff --git a/sanitize_test.go b/sanitize_test.go new file mode 100644 index 0000000..d4f6e26 --- /dev/null +++ b/sanitize_test.go @@ -0,0 +1,287 @@ +package wace + +import ( + "reflect" + "testing" + + "github.com/tilsor/ModSecIntl_wace_lib/waceapi" +) + +func TestSanitizeCredentials(t *testing.T) { + tests := []struct { + name string + input waceapi.HTTPPayload + want waceapi.HTTPPayload + }{ + // --- Body: password field variants --- + { + name: "password= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "user=john&password=secret"}, + want: waceapi.HTTPPayload{RequestBody: "user=john&password=********"}, + }, + { + name: "newpassword= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "newpassword=newsecret"}, + want: waceapi.HTTPPayload{RequestBody: "newpassword=********"}, + }, + { + name: "oldpassword= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "oldpassword=oldsecret"}, + want: waceapi.HTTPPayload{RequestBody: "oldpassword=********"}, + }, + { + name: "vpassword= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "vpassword=vsecret"}, + want: waceapi.HTTPPayload{RequestBody: "vpassword=********"}, + }, + { + name: "form_password= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "form_password=formsecret"}, + want: waceapi.HTTPPayload{RequestBody: "form_password=********"}, + }, + { + name: "clave= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "clave=mysecret"}, + want: waceapi.HTTPPayload{RequestBody: "clave=********"}, + }, + { + name: "pass= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "pass=mypass"}, + want: waceapi.HTTPPayload{RequestBody: "pass=********"}, + }, + { + name: "pass2= is sanitized", + input: waceapi.HTTPPayload{RequestBody: "pass2=mypass"}, + want: waceapi.HTTPPayload{RequestBody: "pass2=********"}, + }, + // --- Body: multiple and edge cases --- + { + name: "multiple credential fields are all sanitized", + input: waceapi.HTTPPayload{RequestBody: "password=first&pass2=second"}, + want: waceapi.HTTPPayload{RequestBody: "password=********&pass2=********"}, + }, + { + name: "empty password value is sanitized", + input: waceapi.HTTPPayload{RequestBody: "password="}, + want: waceapi.HTTPPayload{RequestBody: "password=********"}, + }, + { + name: "non-credential fields are unchanged", + input: waceapi.HTTPPayload{RequestBody: "user=john&email=john@example.com"}, + want: waceapi.HTTPPayload{RequestBody: "user=john&email=john@example.com"}, + }, + { + name: "empty body is unchanged", + input: waceapi.HTTPPayload{RequestBody: ""}, + want: waceapi.HTTPPayload{RequestBody: ""}, + }, + // --- Headers --- + { + name: "Authorization header is sanitized", + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "Bearer token123"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "********"}, + }}, + }, + { + name: "Cookie header is sanitized", + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Cookie", Value: "session=abc123"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Cookie", Value: "********"}, + }}, + }, + { + name: "Set-Cookie header is sanitized", + input: waceapi.HTTPPayload{ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "session=abc; Path=/"}, + }}, + want: waceapi.HTTPPayload{ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "********"}, + }}, + }, + { + name: "header matching is case-insensitive", + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "AUTHORIZATION", Value: "Bearer token"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "AUTHORIZATION", Value: "********"}, + }}, + }, + { + name: "header key is preserved after sanitization", + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "secret"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "********"}, + }}, + }, + { + name: "non-credential headers are unchanged", + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Content-Type", Value: "application/json"}, + {Key: "User-Agent", Value: "TestAgent/1.0"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Content-Type", Value: "application/json"}, + {Key: "User-Agent", Value: "TestAgent/1.0"}, + }}, + }, + { + name: "multiple credential headers are all sanitized", + input: waceapi.HTTPPayload{ + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "Bearer tok"}, + {Key: "Cookie", Value: "s=xyz"}, + }, + ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "s=xyz; Path=/"}, + }}, + want: waceapi.HTTPPayload{ + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "********"}, + {Key: "Cookie", Value: "********"}, + }, + ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "********"}, + }}, + }, + // --- Combined --- + { + name: "body and headers are both sanitized", + input: waceapi.HTTPPayload{ + RequestBody: "user=john&password=secret", + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "Bearer token"}, + {Key: "Content-Type", Value: "application/json"}, + }, + }, + want: waceapi.HTTPPayload{ + RequestBody: "user=john&password=********", + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "********"}, + {Key: "Content-Type", Value: "application/json"}, + }, + }, + }, + { + name: "payload with no sensitive data is returned unchanged", + input: waceapi.HTTPPayload{URI: "/health", Method: "GET"}, + want: waceapi.HTTPPayload{URI: "/health", Method: "GET"}, + }, + // --- Case-insensitive body matching --- + { + name: "Password= (mixed case) is sanitized", + input: waceapi.HTTPPayload{RequestBody: "Password=Secret"}, + want: waceapi.HTTPPayload{RequestBody: "Password=********"}, + }, + { + name: "PASSWORD= (uppercase) is sanitized", + input: waceapi.HTTPPayload{RequestBody: "PASSWORD=secret"}, + want: waceapi.HTTPPayload{RequestBody: "PASSWORD=********"}, + }, + { + name: "CLAVE= (uppercase) is sanitized", + input: waceapi.HTTPPayload{RequestBody: "CLAVE=mysecret"}, + want: waceapi.HTTPPayload{RequestBody: "CLAVE=********"}, + }, + // --- URI sanitization --- + { + name: "password in URI query string is sanitized", + input: waceapi.HTTPPayload{URI: "/login?user=john&password=secret"}, + want: waceapi.HTTPPayload{URI: "/login?user=john&password=********"}, + }, + { + name: "URI with no credentials is unchanged", + input: waceapi.HTTPPayload{URI: "/api/v1/resource?filter=active"}, + want: waceapi.HTTPPayload{URI: "/api/v1/resource?filter=active"}, + }, + { + name: "URI and body are both sanitized", + input: waceapi.HTTPPayload{URI: "/login?pass=abc", RequestBody: "password=xyz"}, + want: waceapi.HTTPPayload{URI: "/login?pass=********", RequestBody: "password=********"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := sanitizeCredentials(tt.input) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("SanitizeCredentials() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestSetCredentialHeaders(t *testing.T) { + original := credentialHeaders + defer func() { credentialHeaders = original }() + + tests := []struct { + name string + headers []string + input waceapi.HTTPPayload + want waceapi.HTTPPayload + }{ + { + name: "custom header replaces defaults", + headers: []string{"X-Api-Key"}, + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "X-Api-Key", Value: "secret"}, + {Key: "Authorization", Value: "Bearer token"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "X-Api-Key", Value: "********"}, + {Key: "Authorization", Value: "Bearer token"}, + }}, + }, + { + name: "input header name casing is normalized before matching", + headers: []string{"X-API-KEY"}, + input: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "x-api-key", Value: "secret"}, + }}, + want: waceapi.HTTPPayload{RequestHeaders: []waceapi.HTTPHeader{ + {Key: "x-api-key", Value: "********"}, + }}, + }, + { + name: "empty list means no headers are sanitized", + headers: []string{}, + input: waceapi.HTTPPayload{ + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "Bearer token"}, + {Key: "Cookie", Value: "session=abc"}, + }, + ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "s=xyz; Path=/"}, + }, + }, + want: waceapi.HTTPPayload{ + RequestHeaders: []waceapi.HTTPHeader{ + {Key: "Authorization", Value: "Bearer token"}, + {Key: "Cookie", Value: "session=abc"}, + }, + ResponseHeaders: []waceapi.HTTPHeader{ + {Key: "Set-Cookie", Value: "s=xyz; Path=/"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SetCredentialHeaders(tt.headers) + got := sanitizeCredentials(tt.input) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("after SetCredentialHeaders(%v): sanitizeCredentials() = %+v, want %+v", + tt.headers, got, tt.want) + } + }) + } +} diff --git a/testdata/plugins/model/param.go b/testdata/plugins/model/param.go index a9a7862..5f083b7 100644 --- a/testdata/plugins/model/param.go +++ b/testdata/plugins/model/param.go @@ -46,7 +46,7 @@ func Process(input waceapi.ModelInput) (waceapi.ModelResults, error) { logger.TPrintf(lg.WARN, input.TransactionId, "[param:Process] \"%v\"\n", input.Payload) return waceapi.ModelResults{ ProbAttack: result, - Data: make(map[string]interface{}), + Data: input, }, nil } diff --git a/wacecore.go b/wacecore.go index 310b9c6..51889d3 100644 --- a/wacecore.go +++ b/wacecore.go @@ -82,24 +82,33 @@ func callPlugins(input waceapi.HTTPPayload, models []string, t configstore.Model startTime := time.Now() + sanitizedPayload := sanitizeCredentials(input) + for _, id := range models { logger.TPrintf(logging.DEBUG, transactionID, "%s | calling from core", id) + if _, ok := conf.ModelPlugins[id]; !ok { logger.TPrintf(logging.ERROR, transactionID, "core | model plugin %s not found", id) } else if conf.ModelPlugins[id].PluginType != t { logger.TPrintf(logging.ERROR, transactionID, "core | model plugin %s is not of type %s", id, t) - } else if conf.IsAsync(id) { - asyncCounter++ - go plugins.AddToQueue(id, transactionID, input) - } else if conf.IsInTraining(id) { - go plugins.ProcessTraining(id, transactionID, input, t) } else { - if conf.IsRemote(id) { - go plugins.AddToQueue(id, transactionID, input) + payload := input + if conf.ShouldSanitize(id) { + payload = sanitizedPayload + } + if conf.IsAsync(id) { + asyncCounter++ + go plugins.AddToQueue(id, transactionID, payload) + } else if conf.IsInTraining(id) { + go plugins.ProcessTraining(id, transactionID, payload, t) } else { - go plugins.Process(id, transactionID, input, t, modelPluginStatus) + if conf.IsRemote(id) { + go plugins.AddToQueue(id, transactionID, payload) + } else { + go plugins.Process(id, transactionID, payload, t, modelPluginStatus) + } + syncCounter++ } - syncCounter++ } } @@ -259,6 +268,9 @@ func Reload(met metric.Meter, conf configstore.ConfigFileData) error { if err = cs.SetConfig(conf); err != nil { return err } + if len(cs.CredentialHeaders) != 0 { + SetCredentialHeaders(cs.CredentialHeaders) + } if err = logger.LoadLogger(cs.LogPath, cs.LogLevel); err != nil { return err } @@ -280,6 +292,10 @@ func Init(met metric.Meter, conf configstore.ConfigFileData) error { return err } + if len(cs.CredentialHeaders) != 0 { + SetCredentialHeaders(cs.CredentialHeaders) + } + meter = met err = logger.LoadLogger(cs.LogPath, cs.LogLevel) From 2d3506ddbb23690cb994ae14823802999df4c419 Mon Sep 17 00:00:00 2001 From: adeleon Date: Thu, 28 May 2026 16:06:00 -0300 Subject: [PATCH 2/2] refactor: update setCredentialHeaders function name --- sanitize.go | 4 ++-- sanitize_test.go | 2 +- wacecore.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sanitize.go b/sanitize.go index ddf44a0..73f4aa1 100644 --- a/sanitize.go +++ b/sanitize.go @@ -37,8 +37,8 @@ func sanitizeCredentials(p waceapi.HTTPPayload) waceapi.HTTPPayload { return p } -// SetCredentialHeaders allows the user to change the list of headers to be sanitized. -func SetCredentialHeaders(headers []string) { +// setCredentialHeaders allows the user to change the list of headers to be sanitized. +func setCredentialHeaders(headers []string) { for i := range headers { headers[i] = strings.ToLower(headers[i]) } diff --git a/sanitize_test.go b/sanitize_test.go index d4f6e26..0b0936a 100644 --- a/sanitize_test.go +++ b/sanitize_test.go @@ -276,7 +276,7 @@ func TestSetCredentialHeaders(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - SetCredentialHeaders(tt.headers) + setCredentialHeaders(tt.headers) got := sanitizeCredentials(tt.input) if !reflect.DeepEqual(got, tt.want) { t.Errorf("after SetCredentialHeaders(%v): sanitizeCredentials() = %+v, want %+v", diff --git a/wacecore.go b/wacecore.go index 51889d3..cd5330c 100644 --- a/wacecore.go +++ b/wacecore.go @@ -269,7 +269,7 @@ func Reload(met metric.Meter, conf configstore.ConfigFileData) error { return err } if len(cs.CredentialHeaders) != 0 { - SetCredentialHeaders(cs.CredentialHeaders) + setCredentialHeaders(cs.CredentialHeaders) } if err = logger.LoadLogger(cs.LogPath, cs.LogLevel); err != nil { return err @@ -293,7 +293,7 @@ func Init(met metric.Meter, conf configstore.ConfigFileData) error { } if len(cs.CredentialHeaders) != 0 { - SetCredentialHeaders(cs.CredentialHeaders) + setCredentialHeaders(cs.CredentialHeaders) } meter = met