crawler: enforce Crawl-delay, back off on 429/503, surface politeness counters

cmd/cosift/enqueue_lane_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// The lane field must be optional and backward compatible: an omitted/empty
+// lane keeps the historical default (Seed → discovered), never the
+// parseLaneName("") default (submitted).
+func TestHandleCrawlEnqueueLane(t *testing.T) {
+	var seedCalls, laneCalls int
+	var gotLane byte
+	s := &pebbleHTTP{
+		crawlSeed:     func(string) error { seedCalls++; return nil },
+		crawlSeedLane: func(_ string, lane byte) error { laneCalls++; gotLane = lane; return nil },
+	}
+
+	post := func(body string) int {
+		req := httptest.NewRequest(http.MethodPost, "/admin/crawl-enqueue", strings.NewReader(body))
+		rec := httptest.NewRecorder()
+		s.handleCrawlEnqueue(rec, req)
+		return rec.Code
+	}
+
+	if code := post(`{"url":"https://example.com/a"}`); code != http.StatusOK {
+		t.Fatalf("no lane: code %d", code)
+	}
+	if seedCalls != 1 || laneCalls != 0 {
+		t.Errorf("empty lane must use crawlSeed: seed=%d lane=%d", seedCalls, laneCalls)
+	}
+
+	if code := post(`{"url":"https://example.com/b","lane":"bulk"}`); code != http.StatusOK {
+		t.Fatalf("bulk lane: code %d", code)
+	}
+	if laneCalls != 1 || gotLane != 3 {
+		t.Errorf("lane=bulk: laneCalls=%d gotLane=%d, want 1/3", laneCalls, gotLane)
+	}
+
+	if code := post(`{"url":"https://example.com/c","lane":"refresh"}`); code != http.StatusOK {
+		t.Fatalf("refresh lane: code %d", code)
+	}
+	if gotLane != 1 {
+		t.Errorf("lane=refresh: gotLane=%d, want 1", gotLane)
+	}
+}

cmd/cosift/serve_crawl.go

@@ -21,6 +21,9 @@ import (
 // PeerAuthToken Bearer header.
 type crawlEnqueueReq struct {
 	URL string `json:"url"`
+	// Lane optionally targets a frontier lane. Empty keeps the historical
+	// default (discovered) — parseLaneName("") would mean submitted.
+	Lane string `json:"lane,omitempty"`
 }
 
 func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request) {
@@ -42,7 +45,13 @@ func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request)
 		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"...\"}")
 		return
 	}
-	if err := s.crawlSeed(req.URL); err != nil {
+	var err error
+	if req.Lane != "" && s.crawlSeedLane != nil {
+		err = s.crawlSeedLane(req.URL, parseLaneName(req.Lane))
+	} else {
+		err = s.crawlSeed(req.URL)
+	}
+	if err != nil {
 		writeProblem(w, http.StatusInternalServerError, err.Error())
 		return
 	}

cmd/cosift/serve_setup.go

@@ -792,6 +792,9 @@ func (s *pebbleHTTP) startInProcessCrawl(ctx context.Context, ps *store.PebbleSt
 	}
 	// Expose Seed so /admin/crawl-enqueue can hand off forwarded URLs.
 	s.crawlSeed = c.Seed
+	// expose SeedLane so /admin/crawl-enqueue can honor an explicit lane.
+	s.crawlSeedLane = c.SeedLane
+	s.crawlPoliteness = c.PolitenessStats
 	// expose SeedSitemap so /admin/sitemap-import can push
 	// sitemap-discovered URLs into the live frontier.
 	s.crawlSeedSitemap = c.SeedSitemap
@@ -1027,6 +1030,10 @@ type pebbleHTTP struct {
 	// allowlist (used by /admin/allow-domain for organic HN/Reddit growth).
 	crawlAllowDomain func(domain string) error
 	crawlRecrawl     func(ctx context.Context, url string) error
+	// crawlSeedLane backs the optional "lane" field on /admin/crawl-enqueue.
+	crawlSeedLane func(url string, lane byte) error
+	// crawlPoliteness feeds the politeness counters into /stats + /metrics.
+	crawlPoliteness func() (droppedDisallowed, rateDeferrals int64)
 
 	// doc count at startup so /stats can report crawl rate
 	// without persistent counter tables. docs_added = current - startup,

cmd/cosift/serve_stats.go

@@ -455,6 +455,11 @@ func (s *pebbleHTTP) buildStatsBody(ctx context.Context) ([]byte, error) {
 		out["crawl_active"] = true
 		out["docs_added_since_start"] = added
 		out["docs_per_minute"] = rate
+		if s.crawlPoliteness != nil {
+			dropped, deferred := s.crawlPoliteness()
+			out["crawl_dropped_disallowed"] = dropped
+			out["crawl_rate_limited_deferrals"] = deferred
+		}
 	}
 	return json.Marshal(out)
 }
@@ -515,6 +520,15 @@ func (s *pebbleHTTP) handleMetrics(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "# HELP cosift_crawl_docs_per_minute Recent crawl rate (docs added per minute, averaged since process start).\n")
 		fmt.Fprintf(w, "# TYPE cosift_crawl_docs_per_minute gauge\n")
 		fmt.Fprintf(w, "cosift_crawl_docs_per_minute %.2f\n", rate)
+		if s.crawlPoliteness != nil {
+			dropped, deferred := s.crawlPoliteness()
+			fmt.Fprintf(w, "# HELP cosift_crawl_dropped_disallowed_total Frontier claims skipped because the domain is not allowlisted.\n")
+			fmt.Fprintf(w, "# TYPE cosift_crawl_dropped_disallowed_total counter\n")
+			fmt.Fprintf(w, "cosift_crawl_dropped_disallowed_total %d\n", dropped)
+			fmt.Fprintf(w, "# HELP cosift_crawl_rate_limited_deferrals_total Host backoffs triggered by 429/503 rate-limit responses.\n")
+			fmt.Fprintf(w, "# TYPE cosift_crawl_rate_limited_deferrals_total counter\n")
+			fmt.Fprintf(w, "cosift_crawl_rate_limited_deferrals_total %d\n", deferred)
+		}
 	}
 	// HyDE cache effectiveness. Hits/misses both monotonic so
 	// Prometheus rate() over these gives cache pressure under load.

cmd/cosift/stats_politeness_test.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestStatsAndMetricsExposePolitenessCounters(t *testing.T) {
+	f := populatedPebbleStore(t)
+	srv := f.makeServer(nil)
+	srv.crawlActive = true
+	srv.crawlPoliteness = func() (int64, int64) { return 7, 3 }
+
+	body, err := srv.buildStatsBody(context.Background())
+	if err != nil {
+		t.Fatalf("buildStatsBody: %v", err)
+	}
+	var out map[string]any
+	if err := json.Unmarshal(body, &out); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if got := out["crawl_dropped_disallowed"]; got != float64(7) {
+		t.Errorf("crawl_dropped_disallowed: got %v want 7", got)
+	}
+	if got := out["crawl_rate_limited_deferrals"]; got != float64(3) {
+		t.Errorf("crawl_rate_limited_deferrals: got %v want 3", got)
+	}
+
+	rec := httptest.NewRecorder()
+	srv.handleMetrics(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("/metrics: code %d", rec.Code)
+	}
+	metrics := rec.Body.String()
+	for _, want := range []string{
+		"cosift_crawl_dropped_disallowed_total 7",
+		"cosift_crawl_rate_limited_deferrals_total 3",
+	} {
+		if !strings.Contains(metrics, want) {
+			t.Errorf("/metrics missing %q", want)
+		}
+	}
+}
+
+func TestStatsOmitsPolitenessWithoutHook(t *testing.T) {
+	f := populatedPebbleStore(t)
+	srv := f.makeServer(nil)
+	srv.crawlActive = true
+
+	body, err := srv.buildStatsBody(context.Background())
+	if err != nil {
+		t.Fatalf("buildStatsBody: %v", err)
+	}
+	if strings.Contains(string(body), "crawl_dropped_disallowed") {
+		t.Error("politeness keys present without a crawlPoliteness hook")
+	}
+}

internal/config/config.go

@@ -118,6 +118,10 @@ type Crawler struct {
 	//   "per_host_overrides": {"slow-server.example.com": 5000, "fast-cdn.com": 50}
 	PerHostOverrides map[string]int `json:"per_host_overrides,omitempty"`
 
+	// MaxCrawlDelayMs caps the robots.txt Crawl-delay we honor, so a
+	// misconfigured robots.txt can't wedge its host. 0 = default (2 min).
+	MaxCrawlDelayMs int `json:"max_crawl_delay_ms,omitempty"`
+
 	// MaxBodyBytes caps the response size. Bigger pages get truncated.
 	MaxBodyBytes int64 `json:"max_body_bytes"`