Rate limit

cmd/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/atye/wikitable2json/internal/server"
 	"github.com/atye/wikitable2json/internal/server/metrics"
 	"github.com/atye/wikitable2json/pkg/client"
+	"golang.org/x/time/rate"
 )
 
 //go:embed static/dist/*
@@ -26,6 +27,24 @@ var (
 	defaultCacheExpiration = 60 * time.Second
 )
 
+type rateLimitedTransport struct {
+	base    http.RoundTripper
+	limiter *rate.Limiter
+}
+
+func (t *rateLimitedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	if err := t.limiter.Wait(r.Context()); err != nil {
+		return nil, fmt.Errorf("waiting for rate limiter: %w", err)
+	}
+
+	base := t.base
+	if base == nil {
+		base = http.DefaultTransport
+	}
+
+	return base.RoundTrip(r)
+}
+
 func main() {
 	port, ok := os.LookupEnv("PORT")
 	if !ok {
@@ -47,7 +66,17 @@ func main() {
 	googleMeasurementId := os.Getenv("GOOGLE_MEASUREMENT_ID")
 	googleAPISecret := os.Getenv("GOOGLE_API_SECRET")
 
-	httpClient := &http.Client{Timeout: 10 * time.Second}
+	httpClient := &http.Client{
+		Timeout: 10 * time.Second,
+		Transport: &rateLimitedTransport{
+			base: &http.Transport{
+				MaxIdleConns:        500,
+				MaxIdleConnsPerHost: 200,
+				MaxConnsPerHost:     200,
+			},
+			limiter: rate.NewLimiter(rate.Limit(200), 10),
+		},
+	}
 
 	var mp server.MetricsPublisher
 	if googleMeasurementId != "" && googleAPISecret != "" {

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	golang.org/x/net v0.53.0
 	golang.org/x/sync v0.20.0
+	golang.org/x/time v0.15.0
 )
 
 require github.com/andybalholm/cascadia v1.3.3 // indirect

go.sum

@@ -66,6 +66,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

pkg/client/client.go

@@ -575,7 +575,8 @@ func parseTable(tableSelection *goquery.Selection, tableIndex int, brIsNewLine b
 					columns[startCol+j+nextAvailableCell] = cell{
 						set:   true,
 						text:  parseText(s, parseNonTextNodeFuncs...),
-						links: parseLink(s, parseNonTextNodeFuncs...)}
+						links: parseLink(s, parseNonTextNodeFuncs...),
+					}
 					if i == 0 {
 						col++
 					}