diff --git a/src/apiutils.nim b/src/apiutils.nim index dbc6cca68..77299e2a8 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -32,7 +32,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; proc genHeaders*(token: Token = nil): HttpHeaders = result = newHttpHeaders({ "connection": "keep-alive", - "authorization": auth, + "authorization": if token == nil: "" else: token.bearerTok, "content-type": "application/json", "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", diff --git a/src/consts.nim b/src/consts.nim index f22581f01..80d20d6c2 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,7 +2,26 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + bearerTokens* = [ + # web + "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", + # web old + "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw", + # android + "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F", + # developer.twitter.com + "Bearer AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb", + # tweetdeck old + "Bearer AAAAAAAAAAAAAAAAAAAAAF7aAAAAAAAASCiRjWvh7R5wxaKkFp7MM%2BhYBqM%3DbQ0JPmjU9F6ZoMhDfI4uTNAaQuTDm2uO9x3WFVr2xBZ2nhjdP0", + # tweetdeck new + "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF", + # macos + "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd", + # iphone + "Bearer AAAAAAAAAAAAAAAAAAAAAAj4AQAAAAAAPraK64zCZ9CSzdLesbE7LB%2Bw4uE%3DVJQREvQNCZJNiz3rHO7lOXlkVOQkzzdsgu6wWgcazdMUaGoUGm", + # ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint? + "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", + ] api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") diff --git a/src/tokens.nim b/src/tokens.nim index 6ef81f5d4..e9e86681d 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -2,19 +2,20 @@ import asyncdispatch, httpclient, times, sequtils, json, random import strutils, tables import types, consts +import std/random +import std/tables +import std/sequtils const maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires maxAge = 2.hours + 55.minutes # tokens expire after 3 hours - failDelay = initDuration(minutes=30) + failDelay = initDuration(minutes=5) var tokenPool: seq[Token] - lastFailed: Time enableLogging = false - -let headers = newHttpHeaders({"authorization": auth}) + failedBearerTokens: Table[string, Time] template log(str) = if enableLogging: echo "[tokens] ", str @@ -64,8 +65,16 @@ proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") proc fetchToken(): Future[Token] {.async.} = - if getTime() - lastFailed < failDelay: - raise rateLimitError() + var eligibleBearerTokens = bearerTokens + .filter(proc (x: string): bool = not failedBearerTokens.hasKey(x) or getTime() - failedBearerTokens[x] >= failDelay) + + if len(eligibleBearerTokens) == 0: + echo "[tokens] all bearer tokens failed recently" + eligibleBearerTokens = bearerTokens.toSeq() + + let auth = sample(eligibleBearerTokens) + log "using token " & auth + let headers = newHttpHeaders({"authorization": auth}) let client = newAsyncHttpClient(headers=headers) @@ -76,12 +85,11 @@ proc fetchToken(): Future[Token] {.async.} = tok = tokNode.getStr($(tokNode.getInt)) time = getTime() - return Token(tok: tok, init: time, lastUse: time) + return Token(tok: tok, bearerTok: auth, init: time, lastUse: time) except Exception as e: echo "[tokens] fetching token failed: ", e.msg if "Try again" notin e.msg: - echo "[tokens] fetching tokens paused, resuming in 30 minutes" - lastFailed = getTime() + failedBearerTokens[auth] = getTime() finally: client.close() @@ -105,8 +113,11 @@ proc isReady(token: Token; api: Api): bool = proc release*(token: Token; used=false; invalid=false) = if token.isNil: return if invalid or token.expired: - if invalid: log "discarding invalid token" - elif token.expired: log "discarding expired token" + if invalid: + log "discarding invalid token " & token.bearerTok + failedBearerTokens[token.bearerTok] = getTime() + elif token.expired: + log "discarding expired token " & token.bearerTok let idx = tokenPool.find(token) if idx > -1: tokenPool.delete(idx) diff --git a/src/types.nim b/src/types.nim index 4dca5f016..94b180d32 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,6 +36,7 @@ type Token* = ref object tok*: string + bearerTok*: string init*: Time lastUse*: Time pending*: int