From 05a0bd5681165543b870c8f98c2352a49c85e562 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 31 May 2026 13:51:43 +0200 Subject: [PATCH 1/3] feat: referer detector --- lib/clickguard/detector/freq_ip.ex | 10 +- lib/clickguard/detector/referer.ex | 108 +++++++++++++ lib/clickguard/event.ex | 4 + test/clickguard/detector/referer_test.exs | 184 ++++++++++++++++++++++ 4 files changed, 297 insertions(+), 9 deletions(-) create mode 100644 lib/clickguard/detector/referer.ex create mode 100644 test/clickguard/detector/referer_test.exs diff --git a/lib/clickguard/detector/freq_ip.ex b/lib/clickguard/detector/freq_ip.ex index ae11e68..acec2a8 100644 --- a/lib/clickguard/detector/freq_ip.ex +++ b/lib/clickguard/detector/freq_ip.ex @@ -84,7 +84,7 @@ defmodule Clickguard.Detector.FreqIp do severity: :low, subject: Event.format_ip(ip), evidence: build_evidence(window, count, threshold, window_ms), - sample_events: sample(window), + sample_events: Event.sample(window), detected_at: detected_at } end @@ -99,12 +99,4 @@ defmodule Clickguard.Detector.FreqIp do window_end: List.last(window).timestamp } end - - defp sample(window) do - if length(window) < 5 do - window - else - Enum.take(window, 3) ++ Enum.take(window, -2) - end - end end diff --git a/lib/clickguard/detector/referer.ex b/lib/clickguard/detector/referer.ex new file mode 100644 index 0000000..bb47f68 --- /dev/null +++ b/lib/clickguard/detector/referer.ex @@ -0,0 +1,108 @@ +defmodule Clickguard.Detector.Referer do + @moduledoc """ + Surfaces requests with empty or known-spam referers + + Rules: + * :empty_referer - referer is nil/blank. Weak signal (privacy browsers, + Referrer-Policy, HTTPS->HTTP downgrade all strip it legitimately). + * :spam_referer - referer host matches a known referer-spam domain. + Subject is the matched domain. + + NOTE: subject semantics differ by rule (sentinel vs domain). Never reason + about a referer finding's subject without branching on :rule. + + Deferred (need deployment config): publisher mismatch, self-referencing loops. + + Severity: low + """ + alias Clickguard.{Event, Finding} + @behaviour Clickguard.Detector + + @type evidence :: %{ + event_count: pos_integer(), + ips: [String.t()], + ips_truncated: boolean(), + matched_value: String.t() | nil + } + + @empty_subject "(none)" + + @default_spam_domains ~w( + brandedleadgeneration.com addshoppers.com 7minuteworkout.com + ) + + @impl true + def name, do: :referer + + @impl true + def detect(events, opts) do + spam = Keyword.get(opts, :spam_domains, @default_spam_domains) |> MapSet.new() + detected_at = DateTime.now!("Etc/UTC") + + empty_findings(events, detected_at) ++ spam_findings(events, spam, detected_at) + end + + defp empty_findings(events, detected_at) do + case Enum.filter(events, &blank_referer?/1) do + [] -> [] + evts -> [build_finding(:empty_referer, @empty_subject, nil, evts, detected_at)] + end + end + + defp spam_findings(events, spam, detected_at) do + events + |> Enum.flat_map(fn e -> + case spam_host(e, spam) do + nil -> + [] + + host -> + [{host, e}] + end + end) + |> Enum.group_by(fn {host, _} -> host end, fn {_, e} -> e end) + |> Enum.map(fn {host, evts} -> + build_finding(:spam_referer, host, host, evts, detected_at) + end) + end + + defp build_finding(rule, subject, matched, evts, detected_at) do + ips = evts |> Enum.map(&Event.ip_string/1) |> Enum.uniq() |> Enum.reject(&is_nil/1) + + %Finding{ + rule: rule, + severity: :low, + subject: subject, + evidence: %{ + event_count: length(evts), + ips: ips |> Enum.take(50), + ips_truncated: length(ips) > 50, + matched_value: matched + }, + sample_events: Event.sample(evts), + detected_at: detected_at + } + end + + defp blank_referer?(%Event{referer: nil}), do: true + defp blank_referer?(%Event{referer: r}), do: String.trim(r) == "" + + defp spam_host(%Event{referer: r}, _spam) when is_nil(r), do: nil + + defp spam_host(%Event{referer: r}, spam) do + case String.trim(r) do + "" -> nil + trimmed -> match_host(URI.parse(trimmed).host, spam) + end + end + + defp match_host(nil, _spam), do: nil + + defp match_host(host, spam) do + normalized = + String.downcase(host) + |> String.replace_prefix("www.", "") + + if MapSet.member?(spam, normalized), do: normalized, else: nil + end +end diff --git a/lib/clickguard/event.ex b/lib/clickguard/event.ex index 0515987..3127e8d 100644 --- a/lib/clickguard/event.ex +++ b/lib/clickguard/event.ex @@ -52,4 +52,8 @@ defmodule Clickguard.Event do @spec format_ip(:inet.ip_address() | nil) :: String.t() | nil def format_ip(nil), do: nil def format_ip(ip), do: ip |> :inet.ntoa() |> to_string() + + @spec sample([%__MODULE__{}]) :: [%__MODULE__{}] + def sample(events) when length(events) < 5, do: events + def sample(events), do: Enum.take(events, 3) ++ Enum.take(events, -2) end diff --git a/test/clickguard/detector/referer_test.exs b/test/clickguard/detector/referer_test.exs new file mode 100644 index 0000000..b09c960 --- /dev/null +++ b/test/clickguard/detector/referer_test.exs @@ -0,0 +1,184 @@ +defmodule Clickguard.Detector.RefererTest do + use ExUnit.Case, async: true + + alias Clickguard.Detector.Referer + alias Clickguard.EventBuilder, as: EB + + @base_ts ~U[2016-05-24 13:26:08.003Z] + @custom_spam_domains ["semalt.com", "badtestdomain.com"] + + describe "detect/2 - :empty_referer rule" do + test "event with valid referer produces no findings" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: "http://gooddomain.com") + assert Referer.detect([event], []) == [] + end + + test "event without referer produces (none) finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: nil) + assert [f] = Referer.detect([event], []) + assert f.subject == "(none)" + end + + test "finding shape" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: nil) + + assert [f] = Referer.detect([event], []) + assert f.rule == :empty_referer + assert f.severity == :low + assert f.subject == "(none)" + assert f.evidence.event_count == 1 + assert f.evidence.ips == ["127.0.0.1"] + assert f.evidence.ips_truncated == false + assert f.evidence.matched_value == nil + assert f.sample_events == [event] + end + + test "event with whitespace-only referer produces (none) finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: " ") + assert [f] = Referer.detect([event], []) + assert f.subject == "(none)" + end + + test "events with nil and whitespace-only referers collapse into one finding" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: nil), + EB.event({127, 0, 0, 2}, @base_ts, referer: nil), + EB.event({127, 0, 0, 3}, @base_ts, referer: " "), + EB.event({127, 0, 0, 4}, @base_ts, referer: " ") + ] + + assert [f] = Referer.detect(events, []) + assert f.subject == "(none)" + assert f.evidence.event_count == 4 + assert length(f.evidence.ips) == 4 + assert "127.0.0.1" in f.evidence.ips + assert "127.0.0.2" in f.evidence.ips + assert "127.0.0.3" in f.evidence.ips + assert "127.0.0.4" in f.evidence.ips + assert f.evidence.ips_truncated == false + end + + test "there are only unique IPs in finding" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: nil), + EB.event({127, 0, 0, 1}, @base_ts, referer: nil), + EB.event({127, 0, 0, 1}, @base_ts, referer: " "), + EB.event({127, 0, 0, 1}, @base_ts, referer: " ") + ] + + assert [f] = Referer.detect(events, []) + assert f.subject == "(none)" + assert f.evidence.event_count == 4 + assert length(f.evidence.ips) == 1 + assert "127.0.0.1" in f.evidence.ips + end + end + + describe "detect/2 - :spam_referer rule" do + test "normalizes domain name" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://www.semalt.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://www.Semalt.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://SEMALT.COM") + ] + + assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) + assert f.subject == "semalt.com" + assert f.evidence.event_count == 4 + assert length(f.evidence.ips) == 1 + assert "127.0.0.1" in f.evidence.ips + end + + test "finding shape" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com") + + assert [f] = Referer.detect([event], spam_domains: @custom_spam_domains) + assert f.rule == :spam_referer + assert f.severity == :low + assert f.subject == "semalt.com" + assert f.evidence.event_count == 1 + assert f.evidence.ips == ["127.0.0.1"] + assert f.evidence.ips_truncated == false + assert f.evidence.matched_value == "semalt.com" + assert f.sample_events == [event] + end + + test "skips garbage referer" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: "garbage"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "Garbage"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "GARBAGE") + ] + + assert Referer.detect(events, []) == [] + end + + test "two different spam domains produce two findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "https://badtestdomain.com") + ] + + assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) + assert "semalt.com" in [f1.subject, f2.subject] + assert "badtestdomain.com" in [f1.subject, f2.subject] + end + + test "ips is truncated" do + events = + for n <- 1..51, + do: EB.event({127, 0, 0, n}, @base_ts, referer: "http://semalt.com") + + assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) + assert f.subject == "semalt.com" + assert f.evidence.event_count == 51 + assert length(f.evidence.ips) == 50 + assert f.evidence.ips_truncated == true + end + + test "ips_truncated boundary" do + events = + for n <- 1..50, + do: EB.event({127, 0, 0, n}, @base_ts, referer: "http://semalt.com") + + assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) + assert f.subject == "semalt.com" + assert f.evidence.event_count == 50 + assert length(f.evidence.ips) == 50 + assert f.evidence.ips_truncated == false + end + + test "use default domain list" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://brandedleadgeneration.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://www.brandedleadgeneration.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://www.Brandedleadgeneration.com"), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://BRANDEDLEADGENERATION.COM") + ] + + assert [f] = Referer.detect(events, []) + assert f.subject == "brandedleadgeneration.com" + assert f.evidence.event_count == 4 + assert length(f.evidence.ips) == 1 + assert "127.0.0.1" in f.evidence.ips + end + end + + describe "detect/2 both rules in one batch" do + test "one :empty_referer event and one :spam_referer event produce exactly two findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com"), + EB.event({127, 0, 0, 2}, @base_ts, referer: nil) + ] + + assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) + assert f1.subject in ["semalt.com", "(none)"] + assert f2.subject in ["semalt.com", "(none)"] + assert f1.rule in [:empty_referer, :spam_referer] + assert f2.rule in [:empty_referer, :spam_referer] + assert f1.rule != f2.rule + assert f1.subject != f2.subject + end + end +end From 5ade08ed2e8200550a511d973377ec4aae17d199 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 31 May 2026 19:33:59 +0200 Subject: [PATCH 2/3] feat: add user-agent detector --- config/config.exs | 3 +- lib/clickguard/detector/user_agent.ex | 78 +++++++ lib/clickguard/finding.ex | 1 + test/clickguard/detector/user_agent_test.exs | 203 +++++++++++++++++++ test/fixtures/sample_clf.log | 1 + 5 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 lib/clickguard/detector/user_agent.ex create mode 100644 test/clickguard/detector/user_agent_test.exs diff --git a/config/config.exs b/config/config.exs index 468e10a..11d4e0b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -2,5 +2,6 @@ import Config config :clickguard, detectors: [ - Clickguard.Detector.FreqIp + Clickguard.Detector.FreqIp, + Clickguard.Detector.UserAgent ] diff --git a/lib/clickguard/detector/user_agent.ex b/lib/clickguard/detector/user_agent.ex new file mode 100644 index 0000000..ee61762 --- /dev/null +++ b/lib/clickguard/detector/user_agent.ex @@ -0,0 +1,78 @@ +defmodule Clickguard.Detector.UserAgent do + @moduledoc """ + Flags requests by automated/headless user agents, keyed by IP. + + Rules (Subject IP): + * :empty_ua - UA nil/blank + * :automation_tool - python-requests, curl, wget, Go-http-client, Scrapy + * :headless_browser - HeadlessChrome, PhantomJS + + Catches lazy bots only - misses real-UA automation. + A filter, strongest in combination, not a standalone verdict. + + Severity: low + """ + @behaviour Clickguard.Detector + alias Clickguard.{Event, Finding} + + @type evidence :: %{ + event_count: pos_integer(), + matched_uas: [String.t()] + } + + @automation ~w(python-requests curl wget go-http-client scrapy) + @headless ~w(headlesschrome phantomjs) + + @impl true + def name, do: :user_agent + + @impl true + def detect(events, _opts) do + detected_at = DateTime.now!("Etc/UTC") + + events + |> Enum.flat_map(fn e -> + case classify(e) do + nil -> [] + rule -> [{{Event.ip_string(e), rule}, e}] + end + end) + |> Enum.group_by(fn {key, _} -> key end, fn {_, e} -> e end) + |> Enum.map(fn {{ip, rule}, evts} -> build_finding(rule, ip, evts, detected_at) end) + end + + defp classify(%Event{user_agent: ua}) do + if blank?(ua), do: :empty_ua, else: match_token(String.downcase(ua)) + end + + defp match_token(ua) do + cond do + contains_any?(ua, @automation) -> :automation_tool + contains_any?(ua, @headless) -> :headless_browser + true -> nil + end + end + + defp contains_any?(ua, tokens), do: Enum.any?(tokens, &String.contains?(ua, &1)) + + defp build_finding(rule, ip, events, detected_at) do + %Finding{ + rule: rule, + severity: :low, + subject: ip, + evidence: build_evidence(events), + sample_events: Event.sample(events), + detected_at: detected_at + } + end + + defp build_evidence(events) do + %{ + event_count: length(events), + matched_uas: events |> Enum.map(& &1.user_agent) |> Enum.uniq() |> Enum.reject(&blank?/1) + } + end + + defp blank?(nil), do: true + defp blank?(ua), do: String.trim(ua) == "" +end diff --git a/lib/clickguard/finding.ex b/lib/clickguard/finding.ex index 4e20432..5adea47 100644 --- a/lib/clickguard/finding.ex +++ b/lib/clickguard/finding.ex @@ -12,6 +12,7 @@ defmodule Clickguard.Finding do severity: severity(), subject: String.t(), evidence: map(), + # nonempty_list() ? sample_events: [Event.t()], detected_at: DateTime.t() } diff --git a/test/clickguard/detector/user_agent_test.exs b/test/clickguard/detector/user_agent_test.exs new file mode 100644 index 0000000..7161024 --- /dev/null +++ b/test/clickguard/detector/user_agent_test.exs @@ -0,0 +1,203 @@ +defmodule Clickguard.Detector.UserAgentTest do + use ExUnit.Case, async: true + + alias Clickguard.Detector.UserAgent + alias Clickguard.EventBuilder, as: EB + + @base_ts ~U[2016-05-24 13:26:08.003Z] + @automation_ua "python-requests/2.29.0" + @headless_ua "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/534.34 (KHTML, like Gecko) PhantomJS/1.9.2 Safari/534.34" + @good_ua "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:151.0) Gecko/20100101 Firefox/151.0" + + describe "detect/2 common tests for all rules" do + test "event with valid user-agent produces no findings" do + event = + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @good_ua) + + assert UserAgent.detect([event], []) == [] + end + + test "multiple rules per IP produce separate findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: nil), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @good_ua) + ] + + assert [f1, f2, f3] = UserAgent.detect(events, []) + rules = [f1, f2, f3] |> Enum.map(& &1.rule) |> Enum.sort() + assert rules == [:automation_tool, :empty_ua, :headless_browser] + end + + test "one rule per IP produces separate findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: nil), + EB.event({127, 0, 0, 2}, @base_ts, user_agent: @automation_ua), + EB.event({127, 0, 0, 3}, @base_ts, user_agent: @headless_ua), + EB.event({127, 0, 0, 4}, @base_ts, user_agent: @good_ua) + ] + + assert [f1, f2, f3] = UserAgent.detect(events, []) + rules = [f1, f2, f3] |> Enum.map(& &1.rule) |> Enum.sort() + subjects = [f1, f2, f3] |> Enum.map(& &1.subject) |> Enum.sort() + assert rules == [:automation_tool, :empty_ua, :headless_browser] + assert subjects == ["127.0.0.1", "127.0.0.2", "127.0.0.3"] + end + end + + describe "detect/2 - :empty_ua rule" do + test "event without user-agent produces one finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, user_agent: nil) + assert [f] = UserAgent.detect([event], []) + assert f.subject == "127.0.0.1" + assert f.rule == :empty_ua + assert f.evidence.event_count == 1 + assert f.evidence.matched_uas == [] + assert f.severity == :low + end + + test "event with whitespace-only user-agent produces one finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, user_agent: " ") + assert [f] = UserAgent.detect([event], []) + assert f.rule == :empty_ua + assert f.evidence.matched_uas == [] + end + + test "two events with different IPs produce two findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: nil), + EB.event({127, 0, 0, 2}, @base_ts, user_agent: nil) + ] + + assert [f1, f2] = UserAgent.detect(events, []) + assert f1.subject in ["127.0.0.1", "127.0.0.2"] + assert f2.subject in ["127.0.0.1", "127.0.0.2"] + assert f1.subject != f2.subject + assert f1.evidence.matched_uas == f2.evidence.matched_uas + end + + test "two events with the same IP produce one finding" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: nil), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: " ") + ] + + assert [f] = UserAgent.detect(events, []) + assert f.subject == "127.0.0.1" + assert f.evidence.matched_uas == [] + assert f.evidence.event_count == 2 + end + end + + describe "detect/2 - :automation_tool rule" do + test "automation_tool user-agent produces one finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua) + assert [f] = UserAgent.detect([event], []) + assert f.subject == "127.0.0.1" + assert f.rule == :automation_tool + assert f.evidence.event_count == 1 + assert f.evidence.matched_uas == [@automation_ua] + assert f.severity == :low + end + + test "two events with different IPs produce two findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua), + EB.event({127, 0, 0, 2}, @base_ts, user_agent: @automation_ua) + ] + + assert [f1, f2] = UserAgent.detect(events, []) + assert f1.subject in ["127.0.0.1", "127.0.0.2"] + assert f2.subject in ["127.0.0.1", "127.0.0.2"] + assert f1.subject != f2.subject + assert f1.evidence.matched_uas == f2.evidence.matched_uas + assert f1.evidence.matched_uas == [@automation_ua] + end + + test "two events with the same IP and user-agent produce one finding" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua) + ] + + assert [f] = UserAgent.detect(events, []) + assert f.subject == "127.0.0.1" + assert f.evidence.matched_uas == [@automation_ua] + assert f.evidence.event_count == 2 + end + + test "two events with the same IP produce one finding" do + second_ua = "curl/8.3.0" + + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: second_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @automation_ua) + ] + + assert [f1] = UserAgent.detect(events, []) + assert f1.subject == "127.0.0.1" + assert f1.evidence.event_count == 2 + assert length(f1.evidence.matched_uas) == 2 + assert f1.rule == :automation_tool + assert second_ua in f1.evidence.matched_uas + assert @automation_ua in f1.evidence.matched_uas + end + end + + describe "detect/2 - :headless_browser rule" do + test "headless_browser user-agent produces one finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua) + assert [f] = UserAgent.detect([event], []) + assert f.subject == "127.0.0.1" + assert f.rule == :headless_browser + assert f.evidence.event_count == 1 + assert f.evidence.matched_uas == [@headless_ua] + assert f.severity == :low + end + + test "two events with different IPs produce two findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua), + EB.event({127, 0, 0, 2}, @base_ts, user_agent: @headless_ua) + ] + + assert [f1, f2] = UserAgent.detect(events, []) + assert f1.subject in ["127.0.0.1", "127.0.0.2"] + assert f2.subject in ["127.0.0.1", "127.0.0.2"] + assert f1.subject != f2.subject + assert f1.evidence.matched_uas == f2.evidence.matched_uas + assert f1.evidence.matched_uas == [@headless_ua] + end + + test "two events with the same IP and user-agent produce one finding" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua) + ] + + assert [f] = UserAgent.detect(events, []) + assert f.subject == "127.0.0.1" + assert f.evidence.matched_uas == [@headless_ua] + assert f.evidence.event_count == 2 + end + + test "two events with the same IP produce one finding" do + second_ua = + "Mozilla/5.0 (X14; Linux x82_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/80.0.3987.132 Safari/537.36" + + events = [ + EB.event({127, 0, 0, 1}, @base_ts, user_agent: second_ua), + EB.event({127, 0, 0, 1}, @base_ts, user_agent: @headless_ua) + ] + + assert [f] = UserAgent.detect(events, []) + assert f.subject == "127.0.0.1" + assert f.evidence.event_count == 2 + assert length(f.evidence.matched_uas) == 2 + assert f.rule == :headless_browser + assert second_ua in f.evidence.matched_uas + assert @headless_ua in f.evidence.matched_uas + end + end +end diff --git a/test/fixtures/sample_clf.log b/test/fixtures/sample_clf.log index dc38c30..9e64cd1 100644 --- a/test/fixtures/sample_clf.log +++ b/test/fixtures/sample_clf.log @@ -22,3 +22,4 @@ 122.0.2.99 - - [22/May/2026:09:21:13 +0000] "GET /click?aff=42&cid=101 HTTP/1.1" 302 0 "-" "python-requests/2.31.0" 122.0.2.99 - - [22/May/2026:09:22:13 +0000] "GET /click?aff=42&cid=101 HTTP/1.1" 302 0 "-" "python-requests/2.31.0" 122.0.2.99 - - [22/May/2026:09:23:13 +0000] "GET /click?aff=42&cid=101 HTTP/1.1" 302 0 "-" "python-requests/2.31.0" +122.0.2.99 - - [22/May/2026:09:23:13 +0000] "GET /click?aff=42&cid=101 HTTP/1.1" 302 0 "" "" From 1786bcc7116f2b697ceae14eb21e0fdc2a347c83 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 31 May 2026 23:13:14 +0200 Subject: [PATCH 3/3] feat: wired ua and referer detectors into fixtures --- config/config.exs | 3 +- lib/clickguard/fixtures.ex | 107 +++++++++++++++++------- lib/mix/tasks/clickguard.gen.fixture.ex | 18 +++- 3 files changed, 92 insertions(+), 36 deletions(-) diff --git a/config/config.exs b/config/config.exs index 11d4e0b..59bc2d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -3,5 +3,6 @@ import Config config :clickguard, detectors: [ Clickguard.Detector.FreqIp, - Clickguard.Detector.UserAgent + Clickguard.Detector.UserAgent, + Clickguard.Detector.Referer ] diff --git a/lib/clickguard/fixtures.ex b/lib/clickguard/fixtures.ex index 303a083..4704b59 100644 --- a/lib/clickguard/fixtures.ex +++ b/lib/clickguard/fixtures.ex @@ -9,6 +9,7 @@ defmodule Clickguard.Fixtures do @targets ["/index.html", "/index.php", "/", "/admin-login", "/postback?aff_id=123&offer_id=321"] @methods ["GET", "POST"] @base_ts ~U[2026-01-01 00:00:00.00Z] + @ts_format "[%d/%b/%Y:%H:%M:%S +0000]" @referers [ "https://www.example.com/search?q=open+source+database+tools", "https://news.example.com/item?id=482942418", @@ -23,52 +24,93 @@ defmodule Clickguard.Fixtures do "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0" ] + @bad_user_agents [ + "python-requests/2.31.0", + "curl/8.4.0", + "Mozilla/5.0 (Unknown; Linux i686) AppleWebKit/534.34 (KHTML, like Gecko) PhantomJS/1.9.8 Safari/534.34", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/143.0.7499.4 Safari/537.36", + "Wget/1.21.4", + "Go-http-client/2.0", + "" + ] + + # must stay in sync with Referer @default_spam_domains + @bad_referers [ + "https://brandedleadgeneration.com/", + "https://www.addshoppers.com", + "https://7minuteworkout.com", + "" + ] + def generate(opts \\ []) do lines = Keyword.get(opts, :lines, 500) out = Keyword.get(opts, :out, "test/fixtures/sample_clf.log") freqip = Keyword.get(opts, :freqip, false) + bad_ua = Keyword.get(opts, :bad_ua, false) + bad_referer = Keyword.get(opts, :bad_referer, false) :rand.seed(:exsss, 4711) - good_lines = - if lines >= 1 do - for n <- 1..lines do - ts = - DateTime.add(@base_ts, n, :second) - |> Calendar.strftime("[%d/%b/%Y:%H:%M:%S +0000]") - - clf_line(ts: ts) - end - else - [] - end - - freqip_lines = - if freqip do - for n <- 0..299 do - ip = "127.0.0.1" - - ts = - DateTime.add(@base_ts, n * 200, :millisecond) - |> Calendar.strftime("[%d/%b/%Y:%H:%M:%S +0000]") - - clf_line(ip: ip, ts: ts) - end - else - [] - end - - # |> Enum.shuffle() |> Enum.join("\n") - output = (freqip_lines ++ good_lines) |> Enum.join("\n") + good_lines = if lines >= 1, do: generate_good_lines(lines), else: [] + freqip_lines = if freqip, do: generate_freqip_lines(), else: [] + bad_ua_lines = if bad_ua, do: generate_bad_ua_lines(), else: [] + bad_referer_lines = if bad_referer, do: generate_bad_referer_lines(), else: [] + + output = + (freqip_lines ++ good_lines ++ bad_ua_lines ++ bad_referer_lines) + # |> Enum.shuffle() + |> Enum.join("\n") File.mkdir_p!(Path.dirname(out)) File.write!(out, output) - total = length(good_lines) + length(freqip_lines) + total = + length(good_lines) + length(freqip_lines) + length(bad_ua_lines) + length(bad_referer_lines) {total, out} end + defp generate_good_lines(lines) do + for n <- 1..lines, do: clf_line(ts: ts(n)) + end + + # ms spacing: sustained rate across the window, not a spike + defp generate_freqip_lines do + for n <- 0..299 do + ts = + DateTime.add(@base_ts, n * 200, :millisecond) + |> Calendar.strftime(@ts_format) + + clf_line(ip: "127.0.0.1", ts: ts) + end + end + + defp generate_bad_ua_lines do + [first, second | rest] = @bad_user_agents + + single_ua_per_ip = [ + clf_line(ip: "10.0.0.1", ts: ts(0), ua: first), + clf_line(ip: "10.0.0.2", ts: ts(10), ua: second) + ] + + multiple_ua_per_ip = + for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), ua: ua) + + single_ua_per_ip ++ multiple_ua_per_ip + end + + defp generate_bad_referer_lines do + [first | rest] = @bad_referers + + one_domain_across_many_ips = + for n <- 0..4, do: clf_line(ip: "192.168.0.#{n + 1}", ts: ts(n), ref: first) + + other_events = + for ref <- rest, do: clf_line(ip: "192.168.0.10", ts: ts(10), ref: ref) + + one_domain_across_many_ips ++ other_events + end + defp clf_line(fields) do ip = Keyword.get_lazy(fields, :ip, &ip/0) identity = Keyword.get(fields, :identity, "-") @@ -85,6 +127,9 @@ defmodule Clickguard.Fixtures do "#{ip} #{identity} #{username} #{ts} \"#{method} #{target} HTTP/#{http_version}\" #{response_code} #{size} \"#{ref}\" \"#{ua}\"" end + defp ts(offset_s), + do: DateTime.add(@base_ts, offset_s, :second) |> Calendar.strftime(@ts_format) + defp username, do: Enum.random(@usernames) defp method, do: Enum.random(@methods) defp response_code, do: Enum.random(@response_codes) diff --git a/lib/mix/tasks/clickguard.gen.fixture.ex b/lib/mix/tasks/clickguard.gen.fixture.ex index fa891f5..0c72ec5 100644 --- a/lib/mix/tasks/clickguard.gen.fixture.ex +++ b/lib/mix/tasks/clickguard.gen.fixture.ex @@ -5,9 +5,11 @@ defmodule Mix.Tasks.Clickguard.Gen.Fixture do ## Command line options - * `--lines` - number of random generated events - * `--out` - path to the destination file - * `--freqip` - guarantees at least one `FreqIp` finding + * `--lines` - number of random generated events + * `--out` - path to the destination file + * `--freqip` - at least one `FreqIp` finding + * `--bad-ua` - at least one `UserAgent` finding + * `--bad-referer` - at least one `Referer` finding """ @@ -18,7 +20,15 @@ defmodule Mix.Tasks.Clickguard.Gen.Fixture do Mix.Project.get!() {opts, _, _} = - OptionParser.parse(args, strict: [lines: :integer, out: :string, freqip: :boolean]) + OptionParser.parse(args, + strict: [ + lines: :integer, + out: :string, + freqip: :boolean, + bad_ua: :boolean, + bad_referer: :boolean + ] + ) {total, out} = Clickguard.Fixtures.generate(opts) Mix.shell().info("Wrote #{total} lines to #{out}")