From 3adf9cef13b9f69413f4598b04069c43493d8f81 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 22 Mar 2026 13:35:16 +0100 Subject: [PATCH] fixing output writing --- runner/options.go | 26 +++++++- runner/runner.go | 89 +++++++++++++------------ runner/runner_test.go | 152 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 44 deletions(-) diff --git a/runner/options.go b/runner/options.go index d12a7dad..50a08e77 100644 --- a/runner/options.go +++ b/runner/options.go @@ -343,7 +343,7 @@ type Options struct { HeadlessOptionalArguments goflags.StringSlice Protocol string OutputFilterErrorPagePath string - DisableStdout bool + DisableStdout bool JavascriptCodes goflags.StringSlice @@ -696,6 +696,30 @@ func ParseOptions() *Options { return options } +func (options *Options) HasMatcherOrFilter() bool { + return len(options.matchStatusCode) > 0 || + len(options.matchContentLength) > 0 || + len(options.filterStatusCode) > 0 || + len(options.filterContentLength) > 0 || + len(options.matchRegexes) > 0 || + len(options.filterRegexes) > 0 || + len(options.matchLinesCount) > 0 || + len(options.matchWordsCount) > 0 || + len(options.filterLinesCount) > 0 || + len(options.filterWordsCount) > 0 || + len(options.OutputMatchString) > 0 || + len(options.OutputFilterString) > 0 || + len(options.OutputMatchFavicon) > 0 || + len(options.OutputFilterFavicon) > 0 || + len(options.OutputMatchCdn) > 0 || + len(options.OutputFilterCdn) > 0 || + len(options.OutputFilterPageType) > 0 || + options.OutputMatchCondition != "" || + options.OutputFilterCondition != "" || + options.OutputMatchResponseTime != "" || + options.OutputFilterResponseTime != "" +} + func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { return fmt.Errorf("file '%s' does not exist", options.InputFile) diff --git a/runner/runner.go b/runner/runner.go index ae90cfbc..defeea9d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2456,53 +2456,56 @@ retry: responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) var responsePath, fileNameHash string - // store response + // store response — when matchers/filters are active, defer writing to the + // output loop so only matched responses are persisted to disk. if scanopts.StoreResponse || scanopts.StoreChain { - if r.options.OmitBody { - resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "") - } - responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) - // URL.EscapedString returns that can be used as filename - respRaw := resp.Raw - reqRaw := requestDump - if len(respRaw) > scanopts.MaxResponseBodySizeToSave { - respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] - } - data := reqRaw - if scanopts.StoreChain && resp.HasChain() { - data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...) - } - data = append(data, respRaw...) - data = append(data, []byte("\n\n\n")...) - data = append(data, []byte(fullURL)...) - _ = fileutil.CreateFolder(responseBaseDir) - - basePath := strings.TrimSuffix(responsePath, ".txt") - var idx int - for idx = 0; ; idx++ { - targetPath := responsePath - if idx > 0 { - targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx) - } - f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) - if err == nil { - _, writeErr := f.Write(data) - _ = f.Close() - if writeErr != nil { - gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr) + fileNameHash = hash + + if !r.options.HasMatcherOrFilter() { + if r.options.OmitBody { + resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "") + } + responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) + // URL.EscapedString returns that can be used as filename + respRaw := resp.Raw + reqRaw := requestDump + if len(respRaw) > scanopts.MaxResponseBodySizeToSave { + respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] + } + data := reqRaw + if scanopts.StoreChain && resp.HasChain() { + data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...) + } + data = append(data, respRaw...) + data = append(data, []byte("\n\n\n")...) + data = append(data, []byte(fullURL)...) + _ = fileutil.CreateFolder(responseBaseDir) + + basePath := strings.TrimSuffix(responsePath, ".txt") + var idx int + for idx = 0; ; idx++ { + targetPath := responsePath + if idx > 0 { + targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx) + } + f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err == nil { + _, writeErr := f.Write(data) + _ = f.Close() + if writeErr != nil { + gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr) + } + break + } + if !os.IsExist(err) { + gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err) + break } - break - } - if !os.IsExist(err) { - gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err) - break } - } - if idx == 0 { - fileNameHash = hash - } else { - fileNameHash = fmt.Sprintf("%s_%d", hash, idx) + if idx > 0 { + fileNameHash = fmt.Sprintf("%s_%d", hash, idx) + } } } diff --git a/runner/runner_test.go b/runner/runner_test.go index bd96b81d..0d346567 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -386,6 +386,158 @@ func TestRunner_testAndSet_concurrent(t *testing.T) { require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key") } +func TestOptions_hasMatcherOrFilter(t *testing.T) { + tests := []struct { + name string + options Options + expected bool + }{ + { + name: "no matchers or filters", + options: Options{}, + expected: false, + }, + { + name: "match status code", + options: Options{OutputMatchStatusCode: "200"}, + expected: true, + }, + { + name: "filter status code", + options: Options{OutputFilterStatusCode: "403,401"}, + expected: true, + }, + { + name: "match string", + options: Options{OutputMatchString: []string{"admin"}}, + expected: true, + }, + { + name: "filter string", + options: Options{OutputFilterString: []string{"error"}}, + expected: true, + }, + { + name: "match content length", + options: Options{OutputMatchContentLength: "100"}, + expected: true, + }, + { + name: "filter content length", + options: Options{OutputFilterContentLength: "0"}, + expected: true, + }, + { + name: "match regex", + options: Options{OutputMatchRegex: []string{"admin.*panel"}}, + expected: true, + }, + { + name: "filter regex", + options: Options{OutputFilterRegex: []string{"error"}}, + expected: true, + }, + { + name: "match lines count", + options: Options{OutputMatchLinesCount: "50"}, + expected: true, + }, + { + name: "filter lines count", + options: Options{OutputFilterLinesCount: "0"}, + expected: true, + }, + { + name: "match words count", + options: Options{OutputMatchWordsCount: "100"}, + expected: true, + }, + { + name: "filter words count", + options: Options{OutputFilterWordsCount: "0"}, + expected: true, + }, + { + name: "match favicon", + options: Options{OutputMatchFavicon: []string{"1494302000"}}, + expected: true, + }, + { + name: "filter favicon", + options: Options{OutputFilterFavicon: []string{"1494302000"}}, + expected: true, + }, + { + name: "match cdn", + options: Options{OutputMatchCdn: []string{"cloudflare"}}, + expected: true, + }, + { + name: "filter cdn", + options: Options{OutputFilterCdn: []string{"cloudflare"}}, + expected: true, + }, + { + name: "match condition", + options: Options{OutputMatchCondition: "status_code == 200"}, + expected: true, + }, + { + name: "filter condition", + options: Options{OutputFilterCondition: "status_code == 403"}, + expected: true, + }, + { + name: "match response time", + options: Options{OutputMatchResponseTime: "< 1"}, + expected: true, + }, + { + name: "filter response time", + options: Options{OutputFilterResponseTime: "> 5"}, + expected: true, + }, + { + name: "filter page type", + options: Options{OutputFilterPageType: []string{"error"}}, + expected: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + opts := tc.options + err := opts.ValidateOptions() + require.Nil(t, err) + require.Equal(t, tc.expected, opts.HasMatcherOrFilter(), + "HasMatcherOrFilter() should be %v for %s", tc.expected, tc.name) + }) + } +} + +func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) { + dir := t.TempDir() + opts := &Options{ + StoreResponse: true, + StoreResponseDir: dir, + } + err := opts.ValidateOptions() + require.Nil(t, err) + require.False(t, opts.HasMatcherOrFilter()) +} + +func TestStoreResponse_withMatcherSetsFlag(t *testing.T) { + dir := t.TempDir() + opts := &Options{ + StoreResponse: true, + StoreResponseDir: dir, + OutputMatchStatusCode: "200", + } + err := opts.ValidateOptions() + require.Nil(t, err) + require.True(t, opts.HasMatcherOrFilter()) +} + func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { runner := &Runner{}