diff --git a/cmd/main.go b/cmd/main.go index 4769906..5d7f4a6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,6 +16,7 @@ import ( "github.com/atye/wikitable2json/internal/server" "github.com/atye/wikitable2json/internal/server/metrics" "github.com/atye/wikitable2json/pkg/client" + "golang.org/x/time/rate" ) //go:embed static/dist/* @@ -26,6 +27,24 @@ var ( defaultCacheExpiration = 60 * time.Second ) +type rateLimitedTransport struct { + base http.RoundTripper + limiter *rate.Limiter +} + +func (t *rateLimitedTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if err := t.limiter.Wait(r.Context()); err != nil { + return nil, fmt.Errorf("waiting for rate limiter: %w", err) + } + + base := t.base + if base == nil { + base = http.DefaultTransport + } + + return base.RoundTrip(r) +} + func main() { port, ok := os.LookupEnv("PORT") if !ok { @@ -47,7 +66,17 @@ func main() { googleMeasurementId := os.Getenv("GOOGLE_MEASUREMENT_ID") googleAPISecret := os.Getenv("GOOGLE_API_SECRET") - httpClient := &http.Client{Timeout: 10 * time.Second} + httpClient := &http.Client{ + Timeout: 10 * time.Second, + Transport: &rateLimitedTransport{ + base: &http.Transport{ + MaxIdleConns: 500, + MaxIdleConnsPerHost: 200, + MaxConnsPerHost: 200, + }, + limiter: rate.NewLimiter(rate.Limit(200), 10), + }, + } var mp server.MetricsPublisher if googleMeasurementId != "" && googleAPISecret != "" { diff --git a/go.mod b/go.mod index a6b1796..f9b7288 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 golang.org/x/net v0.53.0 golang.org/x/sync v0.20.0 + golang.org/x/time v0.15.0 ) require github.com/andybalholm/cascadia v1.3.3 // indirect diff --git a/go.sum b/go.sum index a430b4e..9b4b111 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/pkg/client/client.go b/pkg/client/client.go index b0acd5a..d1297fd 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -575,7 +575,8 @@ func parseTable(tableSelection *goquery.Selection, tableIndex int, brIsNewLine b columns[startCol+j+nextAvailableCell] = cell{ set: true, text: parseText(s, parseNonTextNodeFuncs...), - links: parseLink(s, parseNonTextNodeFuncs...)} + links: parseLink(s, parseNonTextNodeFuncs...), + } if i == 0 { col++ }