From 9c97a859de747c68e7c95f71ddc8a506e2f02d25 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:24:05 +0200 Subject: [PATCH 01/12] Update apiutils.nim --- src/apiutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index dbc6cca68..7f8a3e4e9 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -32,7 +32,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; proc genHeaders*(token: Token = nil): HttpHeaders = result = newHttpHeaders({ "connection": "keep-alive", - "authorization": auth, + "authorization": if token == nill: "" else: token.bearerTok, "content-type": "application/json", "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", From 79633fe9b9613104f86e056f0b628e92b2178e67 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:24:39 +0200 Subject: [PATCH 02/12] Update consts.nim --- src/consts.nim | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/consts.nim b/src/consts.nim index f22581f01..9cea9dfbb 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,7 +2,20 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + bearerTokens* = [ + # web + "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", + # web old + "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw", + # android + "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F", + # developer.twitter.com + "Bearer AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb", + # tweetdeck old + "Bearer AAAAAAAAAAAAAAAAAAAAAF7aAAAAAAAASCiRjWvh7R5wxaKkFp7MM%2BhYBqM%3DbQ0JPmjU9F6ZoMhDfI4uTNAaQuTDm2uO9x3WFVr2xBZ2nhjdP0", + # tweetdeck new + "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF", + ] api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") From 4af26022b590ca081e2b269dde1029c2a4ba1b02 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:26:02 +0200 Subject: [PATCH 03/12] Update tokens.nim --- src/tokens.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tokens.nim b/src/tokens.nim index 6ef81f5d4..e07565c8a 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -2,20 +2,19 @@ import asyncdispatch, httpclient, times, sequtils, json, random import strutils, tables import types, consts +import std/random const maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires maxAge = 2.hours + 55.minutes # tokens expire after 3 hours - failDelay = initDuration(minutes=30) + failDelay = initDuration(minutes=0) var tokenPool: seq[Token] lastFailed: Time enableLogging = false -let headers = newHttpHeaders({"authorization": auth}) - template log(str) = if enableLogging: echo "[tokens] ", str @@ -67,6 +66,9 @@ proc fetchToken(): Future[Token] {.async.} = if getTime() - lastFailed < failDelay: raise rateLimitError() + let auth = sample(bearerTokens) + let headers = newHttpHeaders({"authorization": auth}) + let client = newAsyncHttpClient(headers=headers) try: @@ -76,7 +78,7 @@ proc fetchToken(): Future[Token] {.async.} = tok = tokNode.getStr($(tokNode.getInt)) time = getTime() - return Token(tok: tok, init: time, lastUse: time) + return Token(tok: tok, bearerTok: auth, init: time, lastUse: time) except Exception as e: echo "[tokens] fetching token failed: ", e.msg if "Try again" notin e.msg: From a89f1cd0c8ee142fb9583f14bbe166b30fcc799d Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:26:31 +0200 Subject: [PATCH 04/12] Update types.nim --- src/types.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types.nim b/src/types.nim index 4dca5f016..94b180d32 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,6 +36,7 @@ type Token* = ref object tok*: string + bearerTok*: string init*: Time lastUse*: Time pending*: int From 9101b10f2058d184711695936cf6c2c99d261079 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:59:12 +0200 Subject: [PATCH 05/12] Update apiutils.nim --- src/apiutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index 7f8a3e4e9..77299e2a8 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -32,7 +32,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; proc genHeaders*(token: Token = nil): HttpHeaders = result = newHttpHeaders({ "connection": "keep-alive", - "authorization": if token == nill: "" else: token.bearerTok, + "authorization": if token == nil: "" else: token.bearerTok, "content-type": "application/json", "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", From 30f2b83ab5a485ab0da988353ae9e1c581bbff1a Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:47:32 +0200 Subject: [PATCH 06/12] Update consts.nim --- src/consts.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/consts.nim b/src/consts.nim index 9cea9dfbb..a5b0cfa86 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -15,6 +15,8 @@ const "Bearer AAAAAAAAAAAAAAAAAAAAAF7aAAAAAAAASCiRjWvh7R5wxaKkFp7MM%2BhYBqM%3DbQ0JPmjU9F6ZoMhDfI4uTNAaQuTDm2uO9x3WFVr2xBZ2nhjdP0", # tweetdeck new "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF", + # macos + "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd", ] api = parseUri("https://api.twitter.com") From 0489483c1b27fb91bcdd68cc026e0733dc4dbeb0 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:22:01 +0200 Subject: [PATCH 07/12] Update tokens.nim --- src/tokens.nim | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/tokens.nim b/src/tokens.nim index e07565c8a..37ca69450 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -3,17 +3,19 @@ import asyncdispatch, httpclient, times, sequtils, json, random import strutils, tables import types, consts import std/random +import std/tables +import std/sequtils const maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires maxAge = 2.hours + 55.minutes # tokens expire after 3 hours - failDelay = initDuration(minutes=0) + failDelay = initDuration(minutes=5) var tokenPool: seq[Token] - lastFailed: Time enableLogging = false + failedBearerTokens: Table[string, Time] template log(str) = if enableLogging: echo "[tokens] ", str @@ -63,10 +65,15 @@ proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") proc fetchToken(): Future[Token] {.async.} = - if getTime() - lastFailed < failDelay: + let eligibleBearerTokens = bearerTokens + .filter(proc (x: string): bool = not failedBearerTokens.hasKey(x) or getTime() - failedBearerTokens[x] >= failDelay) + + if len(eligibleBearerTokens) == 0: + echo "[tokens] all bearer tokens failed recently" raise rateLimitError() - let auth = sample(bearerTokens) + let auth = sample(eligibleBearerTokens) + log "using token " & auth let headers = newHttpHeaders({"authorization": auth}) let client = newAsyncHttpClient(headers=headers) @@ -82,8 +89,7 @@ proc fetchToken(): Future[Token] {.async.} = except Exception as e: echo "[tokens] fetching token failed: ", e.msg if "Try again" notin e.msg: - echo "[tokens] fetching tokens paused, resuming in 30 minutes" - lastFailed = getTime() + failedBearerTokens[auth] = getTime() finally: client.close() @@ -107,8 +113,11 @@ proc isReady(token: Token; api: Api): bool = proc release*(token: Token; used=false; invalid=false) = if token.isNil: return if invalid or token.expired: - if invalid: log "discarding invalid token" - elif token.expired: log "discarding expired token" + if invalid: + log "discarding invalid token" + failedBearerTokens[token.bearerTok] = getTime() + elif token.expired: + log "discarding expired token" let idx = tokenPool.find(token) if idx > -1: tokenPool.delete(idx) From ec46eaa4004e50aaaf25d99e188e47b4623c4fc3 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:23:28 +0200 Subject: [PATCH 08/12] Update tokens.nim --- src/tokens.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokens.nim b/src/tokens.nim index 37ca69450..b75fc05c9 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -114,10 +114,10 @@ proc release*(token: Token; used=false; invalid=false) = if token.isNil: return if invalid or token.expired: if invalid: - log "discarding invalid token" + log "discarding invalid token " & token.bearerTok failedBearerTokens[token.bearerTok] = getTime() elif token.expired: - log "discarding expired token" + log "discarding expired token " & token.bearerTok let idx = tokenPool.find(token) if idx > -1: tokenPool.delete(idx) From b94c35325efa0041a2a2e08bc586a00dd9e56b5f Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:45:02 +0200 Subject: [PATCH 09/12] Update tokens.nim --- src/tokens.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokens.nim b/src/tokens.nim index b75fc05c9..e9e86681d 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -65,12 +65,12 @@ proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") proc fetchToken(): Future[Token] {.async.} = - let eligibleBearerTokens = bearerTokens + var eligibleBearerTokens = bearerTokens .filter(proc (x: string): bool = not failedBearerTokens.hasKey(x) or getTime() - failedBearerTokens[x] >= failDelay) if len(eligibleBearerTokens) == 0: echo "[tokens] all bearer tokens failed recently" - raise rateLimitError() + eligibleBearerTokens = bearerTokens.toSeq() let auth = sample(eligibleBearerTokens) log "using token " & auth From 6f588e3446ac0a011c7b6fea82e58250d1360b23 Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Tue, 4 Jul 2023 19:01:37 +0200 Subject: [PATCH 10/12] Update consts.nim --- src/consts.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/consts.nim b/src/consts.nim index a5b0cfa86..6bfdce563 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -17,6 +17,10 @@ const "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF", # macos "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd", + # iphone + "Bearer AAAAAAAAAAAAAAAAAAAAAAj4AQAAAAAAPraK64zCZ9CSzdLesbE7LB%2Bw4uE%3DVJQREvQNCZJNiz3rHO7lOXlkVOQkzzdsgu6wWgcazdMUaGoUGm", + # ipad + "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", ] api = parseUri("https://api.twitter.com") From 56eceffa7b67de805dd77f33bb8649c3f0bb369d Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Fri, 7 Jul 2023 22:13:10 +0200 Subject: [PATCH 11/12] Update consts.nim --- src/consts.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/consts.nim b/src/consts.nim index 6bfdce563..3e8af820f 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -19,8 +19,8 @@ const "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd", # iphone "Bearer AAAAAAAAAAAAAAAAAAAAAAj4AQAAAAAAPraK64zCZ9CSzdLesbE7LB%2Bw4uE%3DVJQREvQNCZJNiz3rHO7lOXlkVOQkzzdsgu6wWgcazdMUaGoUGm", - # ipad - "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", + # ipad -- temporarily disabled because tweet search returns data in a different format, making nitter return empty results + # "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", ] api = parseUri("https://api.twitter.com") From 31f964145d2e5a27030739dbab5104770490f29b Mon Sep 17 00:00:00 2001 From: drpepper66666 <138406307+drpepper66666@users.noreply.github.com> Date: Fri, 7 Jul 2023 22:21:30 +0200 Subject: [PATCH 12/12] Update consts.nim --- src/consts.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/consts.nim b/src/consts.nim index 3e8af820f..80d20d6c2 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -19,8 +19,8 @@ const "Bearer AAAAAAAAAAAAAAAAAAAAAIWCCAAAAAAA2C25AxqI%2BYCS7pdfJKRH8Xh19zA%3D8vpDZzPHaEJhd20MKVWp3UR38YoPpuTX7UD2cVYo3YNikubuxd", # iphone "Bearer AAAAAAAAAAAAAAAAAAAAAAj4AQAAAAAAPraK64zCZ9CSzdLesbE7LB%2Bw4uE%3DVJQREvQNCZJNiz3rHO7lOXlkVOQkzzdsgu6wWgcazdMUaGoUGm", - # ipad -- temporarily disabled because tweet search returns data in a different format, making nitter return empty results - # "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", + # ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint? + "Bearer AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR", ] api = parseUri("https://api.twitter.com")