additional bearer tokens & retry faster after rate-limit

src/apiutils.nim

@@ -32,7 +32,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
 proc genHeaders*(token: Token = nil): HttpHeaders =
   result = newHttpHeaders({
     "connection": "keep-alive",
-    "authorization": auth,
+    "authorization": if token == nil: "" else: token.bearerTok,
     "content-type": "application/json",
     "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",

src/consts.nim

@@ -2,7 +2,26 @@
 import uri, sequtils, strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+  bearerTokens* = [
+      # web
+      "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+      # web old
+      "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw",
+      # android
+      "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F",
+      # developer.twitter.com
+      "Bearer AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb",
+      # tweetdeck old
+      "Bearer AAAAAAAAAAAAAAAAAAAAAF7aAAAAAAAASCiRjWvh7R5wxaKkFp7MM%2BhYBqM%3DbQ0JPmjU9F6ZoMhDfI4uTNAaQuTDm2uO9x3WFVr2xBZ2nhjdP0",
+      # tweetdeck new
+      "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF",
+      # macos
+      "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd",
+      # iphone
+      "Bearer AAAAAAAAAAAAAAAAAAAAAAj4AQAAAAAAPraK64zCZ9CSzdLesbE7LB%2Bw4uE%3DVJQREvQNCZJNiz3rHO7lOXlkVOQkzzdsgu6wWgcazdMUaGoUGm",
+      # ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint?
+      "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR",
+  ]
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")

src/tokens.nim

@@ -2,19 +2,20 @@
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
 import types, consts
+import std/random
+import std/tables
+import std/sequtils
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
   maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
   maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
-  failDelay = initDuration(minutes=30)
+  failDelay = initDuration(minutes=5)
 
 var
   tokenPool: seq[Token]
-  lastFailed: Time
   enableLogging = false
-
-let headers = newHttpHeaders({"authorization": auth})
+  failedBearerTokens: Table[string, Time]
 
 template log(str) =
   if enableLogging: echo "[tokens] ", str
@@ -64,8 +65,16 @@ proc rateLimitError*(): ref RateLimitError =
   newException(RateLimitError, "rate limited")
 
 proc fetchToken(): Future[Token] {.async.} =
-  if getTime() - lastFailed < failDelay:
-    raise rateLimitError()
+  var eligibleBearerTokens = bearerTokens
+    .filter(proc (x: string): bool = not failedBearerTokens.hasKey(x) or getTime() - failedBearerTokens[x] >= failDelay)
+
+  if len(eligibleBearerTokens) == 0:
+    echo "[tokens] all bearer tokens failed recently"
+    eligibleBearerTokens = bearerTokens.toSeq()
+
+  let auth = sample(eligibleBearerTokens)
+  log "using token " & auth
+  let headers = newHttpHeaders({"authorization": auth})
 
   let client = newAsyncHttpClient(headers=headers)
 
@@ -76,12 +85,11 @@ proc fetchToken(): Future[Token] {.async.} =
       tok = tokNode.getStr($(tokNode.getInt))
       time = getTime()
 
-    return Token(tok: tok, init: time, lastUse: time)
+    return Token(tok: tok, bearerTok: auth, init: time, lastUse: time)
   except Exception as e:
     echo "[tokens] fetching token failed: ", e.msg
     if "Try again" notin e.msg:
-      echo "[tokens] fetching tokens paused, resuming in 30 minutes"
-      lastFailed = getTime()
+      failedBearerTokens[auth] = getTime()
   finally:
     client.close()
 
@@ -105,8 +113,11 @@ proc isReady(token: Token; api: Api): bool =
 proc release*(token: Token; used=false; invalid=false) =
   if token.isNil: return
   if invalid or token.expired:
-    if invalid: log "discarding invalid token"
-    elif token.expired: log "discarding expired token"
+    if invalid:
+      log "discarding invalid token " & token.bearerTok
+      failedBearerTokens[token.bearerTok] = getTime()
+    elif token.expired:
+      log "discarding expired token " & token.bearerTok
 
     let idx = tokenPool.find(token)
     if idx > -1: tokenPool.delete(idx)

src/types.nim

@@ -36,6 +36,7 @@ type
 
   Token* = ref object
     tok*: string
+    bearerTok*: string
     init*: Time
     lastUse*: Time
     pending*: int