feat/ua and referer detectors

config/config.exs

@@ -2,5 +2,7 @@ import Config
 
 config :clickguard,
   detectors: [
-    Clickguard.Detector.FreqIp
+    Clickguard.Detector.FreqIp,
+    Clickguard.Detector.UserAgent,
+    Clickguard.Detector.Referer
   ]

lib/clickguard/detector/freq_ip.ex

@@ -84,7 +84,7 @@ defmodule Clickguard.Detector.FreqIp do
       severity: :low,
       subject: Event.format_ip(ip),
       evidence: build_evidence(window, count, threshold, window_ms),
-      sample_events: sample(window),
+      sample_events: Event.sample(window),
       detected_at: detected_at
     }
   end
@@ -99,12 +99,4 @@ defmodule Clickguard.Detector.FreqIp do
       window_end: List.last(window).timestamp
     }
   end
-
-  defp sample(window) do
-    if length(window) < 5 do
-      window
-    else
-      Enum.take(window, 3) ++ Enum.take(window, -2)
-    end
-  end
 end

lib/clickguard/detector/referer.ex

@@ -0,0 +1,108 @@
+defmodule Clickguard.Detector.Referer do
+  @moduledoc """
+  Surfaces requests with empty or known-spam referers
+
+  Rules:
+    * :empty_referer - referer is nil/blank. Weak signal (privacy browsers,
+      Referrer-Policy, HTTPS->HTTP downgrade all strip it legitimately).
+    * :spam_referer  - referer host matches a known referer-spam domain.
+      Subject is the matched domain.
+
+  NOTE: subject semantics differ by rule (sentinel vs domain). Never reason
+  about a referer finding's subject without branching on :rule.
+
+  Deferred (need deployment config): publisher mismatch, self-referencing loops.
+
+  Severity: low
+  """
+  alias Clickguard.{Event, Finding}
+  @behaviour Clickguard.Detector
+
+  @type evidence :: %{
+          event_count: pos_integer(),
+          ips: [String.t()],
+          ips_truncated: boolean(),
+          matched_value: String.t() | nil
+        }
+
+  @empty_subject "(none)"
+
+  @default_spam_domains ~w(
+    brandedleadgeneration.com addshoppers.com 7minuteworkout.com
+  )
+
+  @impl true
+  def name, do: :referer
+
+  @impl true
+  def detect(events, opts) do
+    spam = Keyword.get(opts, :spam_domains, @default_spam_domains) |> MapSet.new()
+    detected_at = DateTime.now!("Etc/UTC")
+
+    empty_findings(events, detected_at) ++ spam_findings(events, spam, detected_at)
+  end
+
+  defp empty_findings(events, detected_at) do
+    case Enum.filter(events, &blank_referer?/1) do
+      [] -> []
+      evts -> [build_finding(:empty_referer, @empty_subject, nil, evts, detected_at)]
+    end
+  end
+
+  defp spam_findings(events, spam, detected_at) do
+    events
+    |> Enum.flat_map(fn e ->
+      case spam_host(e, spam) do
+        nil ->
+          []
+
+        host ->
+          [{host, e}]
+      end
+    end)
+    |> Enum.group_by(fn {host, _} -> host end, fn {_, e} -> e end)
+    |> Enum.map(fn {host, evts} ->
+      build_finding(:spam_referer, host, host, evts, detected_at)
+    end)
+  end
+
+  defp build_finding(rule, subject, matched, evts, detected_at) do
+    ips = evts |> Enum.map(&Event.ip_string/1) |> Enum.uniq() |> Enum.reject(&is_nil/1)
+
+    %Finding{
+      rule: rule,
+      severity: :low,
+      subject: subject,
+      evidence: %{
+        event_count: length(evts),
+        ips: ips |> Enum.take(50),
+        ips_truncated: length(ips) > 50,
+        matched_value: matched
+      },
+      sample_events: Event.sample(evts),
+      detected_at: detected_at
+    }
+  end
+
+  defp blank_referer?(%Event{referer: nil}), do: true
+  defp blank_referer?(%Event{referer: r}), do: String.trim(r) == ""
+
+  defp spam_host(%Event{referer: r}, _spam) when is_nil(r), do: nil
+
+  defp spam_host(%Event{referer: r}, spam) do
+    case String.trim(r) do
+      "" -> nil
+      trimmed -> match_host(URI.parse(trimmed).host, spam)
+    end
+  end
+
+  defp match_host(nil, _spam), do: nil
+
+  defp match_host(host, spam) do
+    normalized =
+      String.downcase(host)
+      |> String.replace_prefix("www.", "")
+
+    if MapSet.member?(spam, normalized), do: normalized, else: nil
+  end
+end

lib/clickguard/detector/user_agent.ex

@@ -0,0 +1,78 @@
+defmodule Clickguard.Detector.UserAgent do
+  @moduledoc """
+  Flags requests by automated/headless user agents, keyed by IP.
+
+  Rules (Subject IP):
+    * :empty_ua - UA nil/blank
+    * :automation_tool - python-requests, curl, wget, Go-http-client, Scrapy
+    * :headless_browser - HeadlessChrome, PhantomJS
+
+  Catches lazy bots only - misses real-UA automation.
+  A filter, strongest in combination, not a standalone verdict.
+
+  Severity: low
+  """
+  @behaviour Clickguard.Detector
+  alias Clickguard.{Event, Finding}
+
+  @type evidence :: %{
+          event_count: pos_integer(),
+          matched_uas: [String.t()]
+        }
+
+  @automation ~w(python-requests curl wget go-http-client scrapy)
+  @headless ~w(headlesschrome phantomjs)
+
+  @impl true
+  def name, do: :user_agent
+
+  @impl true
+  def detect(events, _opts) do
+    detected_at = DateTime.now!("Etc/UTC")
+
+    events
+    |> Enum.flat_map(fn e ->
+      case classify(e) do
+        nil -> []
+        rule -> [{{Event.ip_string(e), rule}, e}]
+      end
+    end)
+    |> Enum.group_by(fn {key, _} -> key end, fn {_, e} -> e end)
+    |> Enum.map(fn {{ip, rule}, evts} -> build_finding(rule, ip, evts, detected_at) end)
+  end
+
+  defp classify(%Event{user_agent: ua}) do
+    if blank?(ua), do: :empty_ua, else: match_token(String.downcase(ua))
+  end
+
+  defp match_token(ua) do
+    cond do
+      contains_any?(ua, @automation) -> :automation_tool
+      contains_any?(ua, @headless) -> :headless_browser
+      true -> nil
+    end
+  end
+
+  defp contains_any?(ua, tokens), do: Enum.any?(tokens, &String.contains?(ua, &1))
+
+  defp build_finding(rule, ip, events, detected_at) do
+    %Finding{
+      rule: rule,
+      severity: :low,
+      subject: ip,
+      evidence: build_evidence(events),
+      sample_events: Event.sample(events),
+      detected_at: detected_at
+    }
+  end
+
+  defp build_evidence(events) do
+    %{
+      event_count: length(events),
+      matched_uas: events |> Enum.map(& &1.user_agent) |> Enum.uniq() |> Enum.reject(&blank?/1)
+    }
+  end
+
+  defp blank?(nil), do: true
+  defp blank?(ua), do: String.trim(ua) == ""
+end

lib/clickguard/event.ex

@@ -52,4 +52,8 @@ defmodule Clickguard.Event do
   @spec format_ip(:inet.ip_address() | nil) :: String.t() | nil
   def format_ip(nil), do: nil
   def format_ip(ip), do: ip |> :inet.ntoa() |> to_string()
+
+  @spec sample([%__MODULE__{}]) :: [%__MODULE__{}]
+  def sample(events) when length(events) < 5, do: events
+  def sample(events), do: Enum.take(events, 3) ++ Enum.take(events, -2)
 end

lib/clickguard/finding.ex

@@ -12,6 +12,7 @@ defmodule Clickguard.Finding do
           severity: severity(),
           subject: String.t(),
           evidence: map(),
+          # nonempty_list() ?
           sample_events: [Event.t()],
           detected_at: DateTime.t()
         }

lib/clickguard/fixtures.ex

@@ -9,6 +9,7 @@ defmodule Clickguard.Fixtures do
   @targets ["/index.html", "/index.php", "/", "/admin-login", "/postback?aff_id=123&offer_id=321"]
   @methods ["GET", "POST"]
   @base_ts ~U[2026-01-01 00:00:00.00Z]
+  @ts_format "[%d/%b/%Y:%H:%M:%S +0000]"
   @referers [
     "https://www.example.com/search?q=open+source+database+tools",
     "https://news.example.com/item?id=482942418",
@@ -23,52 +24,93 @@ defmodule Clickguard.Fixtures do
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"
   ]
 
+  @bad_user_agents [
+    "python-requests/2.31.0",
+    "curl/8.4.0",
+    "Mozilla/5.0 (Unknown; Linux i686) AppleWebKit/534.34 (KHTML, like Gecko) PhantomJS/1.9.8 Safari/534.34",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/143.0.7499.4 Safari/537.36",
+    "Wget/1.21.4",
+    "Go-http-client/2.0",
+    ""
+  ]
+
+  # must stay in sync with Referer @default_spam_domains
+  @bad_referers [
+    "https://brandedleadgeneration.com/",
+    "https://www.addshoppers.com",
+    "https://7minuteworkout.com",
+    ""
+  ]
+
   def generate(opts \\ []) do
     lines = Keyword.get(opts, :lines, 500)
     out = Keyword.get(opts, :out, "test/fixtures/sample_clf.log")
     freqip = Keyword.get(opts, :freqip, false)
+    bad_ua = Keyword.get(opts, :bad_ua, false)
+    bad_referer = Keyword.get(opts, :bad_referer, false)
 
     :rand.seed(:exsss, 4711)
 
-    good_lines =
-      if lines >= 1 do
-        for n <- 1..lines do
-          ts =
-            DateTime.add(@base_ts, n, :second)
-            |> Calendar.strftime("[%d/%b/%Y:%H:%M:%S +0000]")
-
-          clf_line(ts: ts)
-        end
-      else
-        []
-      end
-
-    freqip_lines =
-      if freqip do
-        for n <- 0..299 do
-          ip = "127.0.0.1"
-
-          ts =
-            DateTime.add(@base_ts, n * 200, :millisecond)
-            |> Calendar.strftime("[%d/%b/%Y:%H:%M:%S +0000]")
-
-          clf_line(ip: ip, ts: ts)
-        end
-      else
-        []
-      end
-
-    # |> Enum.shuffle() |> Enum.join("\n")
-    output = (freqip_lines ++ good_lines) |> Enum.join("\n")
+    good_lines = if lines >= 1, do: generate_good_lines(lines), else: []
+    freqip_lines = if freqip, do: generate_freqip_lines(), else: []
+    bad_ua_lines = if bad_ua, do: generate_bad_ua_lines(), else: []
+    bad_referer_lines = if bad_referer, do: generate_bad_referer_lines(), else: []
+
+    output =
+      (freqip_lines ++ good_lines ++ bad_ua_lines ++ bad_referer_lines)
+      # |> Enum.shuffle()
+      |> Enum.join("\n")
 
     File.mkdir_p!(Path.dirname(out))
     File.write!(out, output)
 
-    total = length(good_lines) + length(freqip_lines)
+    total =
+      length(good_lines) + length(freqip_lines) + length(bad_ua_lines) + length(bad_referer_lines)
 
     {total, out}
   end
 
+  defp generate_good_lines(lines) do
+    for n <- 1..lines, do: clf_line(ts: ts(n))
+  end
+
+  # ms spacing: sustained rate across the window, not a spike
+  defp generate_freqip_lines do
+    for n <- 0..299 do
+      ts =
+        DateTime.add(@base_ts, n * 200, :millisecond)
+        |> Calendar.strftime(@ts_format)
+
+      clf_line(ip: "127.0.0.1", ts: ts)
+    end
+  end
+
+  defp generate_bad_ua_lines do
+    [first, second | rest] = @bad_user_agents
+
+    single_ua_per_ip = [
+      clf_line(ip: "10.0.0.1", ts: ts(0), ua: first),
+      clf_line(ip: "10.0.0.2", ts: ts(10), ua: second)
+    ]
+
+    multiple_ua_per_ip =
+      for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), ua: ua)
+
+    single_ua_per_ip ++ multiple_ua_per_ip
+  end
+
+  defp generate_bad_referer_lines do
+    [first | rest] = @bad_referers
+
+    one_domain_across_many_ips =
+      for n <- 0..4, do: clf_line(ip: "192.168.0.#{n + 1}", ts: ts(n), ref: first)
+
+    other_events =
+      for ref <- rest, do: clf_line(ip: "192.168.0.10", ts: ts(10), ref: ref)
+
+    one_domain_across_many_ips ++ other_events
+  end
+
   defp clf_line(fields) do
     ip = Keyword.get_lazy(fields, :ip, &ip/0)
     identity = Keyword.get(fields, :identity, "-")
@@ -85,6 +127,9 @@ defmodule Clickguard.Fixtures do
     "#{ip} #{identity} #{username} #{ts} \"#{method} #{target} HTTP/#{http_version}\" #{response_code} #{size} \"#{ref}\" \"#{ua}\""
   end
 
+  defp ts(offset_s),
+    do: DateTime.add(@base_ts, offset_s, :second) |> Calendar.strftime(@ts_format)
+
   defp username, do: Enum.random(@usernames)
   defp method, do: Enum.random(@methods)
   defp response_code, do: Enum.random(@response_codes)