mirrors/nitter
1
0
Fork 0
mirror of https://github.com/zedeus/nitter.git synced 2024-12-22 23:25:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Temporary (?) fix for false rate limits

Browse source
This commit is contained in:
Zed 2021-01-18 07:47:51 +01:00
parent 51b1567af6
commit 67e15bb492
2 changed files with 24 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/apiutils.nim
View file

@ -37,6 +37,7 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
var token = await getToken() var token = await getToken()
if token.tok.len == 0: if token.tok.len == 0:
release(token, true)
raise rateLimitError() raise rateLimitError()
let headers = genHeaders(token) let headers = genHeaders(token)
@ -61,6 +62,9 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
token.lastUse = getTime() token.lastUse = getTime()
else: else:
echo "fetch error: ", result.getError echo "fetch error: ", result.getError
except Exception: release(token, true)
echo "error: ", url raise rateLimitError()
except Exception as e:
echo "error: ", e.msg, ", url: ", url
release(token, true)
raise rateLimitError() raise rateLimitError()

35
src/tokens.nim
View file

@ -2,11 +2,16 @@ import asyncdispatch, httpclient, times, sequtils, json, math, random
import strutils, strformat import strutils, strformat
import types, agents, consts, http_pool import types, agents, consts, http_pool
const
expirationTime = 3.hours
maxLastUse = 1.hours
resetPeriod = 15.minutes
failDelay = initDuration(minutes=30)
var var
clientPool {.threadvar.}: HttpPool clientPool {.threadvar.}: HttpPool
tokenPool {.threadvar.}: seq[Token] tokenPool {.threadvar.}: seq[Token]
lastFailed: Time lastFailed: Time
minFail = initDuration(minutes=30)
proc getPoolInfo*: string = proc getPoolInfo*: string =
if tokenPool.len == 0: return "token pool empty" if tokenPool.len == 0: return "token pool empty"
@ -18,7 +23,7 @@ proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited with " & getPoolInfo()) newException(RateLimitError, "rate limited with " & getPoolInfo())
proc fetchToken(): Future[Token] {.async.} = proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < minFail: if getTime() - lastFailed < failDelay:
raise rateLimitError() raise rateLimitError()
let headers = newHttpHeaders({ let headers = newHttpHeaders({
@ -38,39 +43,36 @@ proc fetchToken(): Future[Token] {.async.} =
tok = parseJson(resp)["guest_token"].getStr tok = parseJson(resp)["guest_token"].getStr
let time = getTime() let time = getTime()
result = Token(tok: tok, remaining: 187, reset: time + 15.minutes, result = Token(tok: tok, remaining: 187, reset: time + resetPeriod,
init: time, lastUse: time) init: time, lastUse: time)
except Exception as e: except Exception as e:
lastFailed = getTime() lastFailed = getTime()
echo "fetching token failed: ", e.msg echo "fetching token failed: ", e.msg
proc expired(token: Token): bool {.inline.} = template expired(token: Token): untyped =
const
expirationTime = 2.hours
maxLastUse = 1.hours
let time = getTime() let time = getTime()
result = token.init < time - expirationTime or token.init < time - expirationTime or
token.lastUse < time - maxLastUse token.lastUse < time - maxLastUse
proc isLimited(token: Token): bool {.inline.} = template isLimited(token: Token): untyped =
token == nil or (token.remaining <= 1 and token.reset > getTime()) or token == nil or (token.remaining <= 1 and token.reset > getTime()) or
token.expired token.expired
proc release*(token: Token) = proc release*(token: Token; invalid=false) =
if token != nil and token.expired: if token != nil and (invalid or token.expired):
tokenPool.delete(tokenPool.find(token)) let idx = tokenPool.find(token)
if idx > -1: tokenPool.delete(idx)
proc getToken*(): Future[Token] {.async.} = proc getToken*(): Future[Token] {.async.} =
for i in 0 ..< tokenPool.len: for i in 0 ..< tokenPool.len:
if not result.isLimited: break if not result.isLimited: break
result.release() release(result)
result = tokenPool.sample() result = tokenPool.sample()
if result.isLimited: if result.isLimited:
result.release() release(result)
result = await fetchToken() result = await fetchToken()
tokenPool.add result tokenPool.add result
echo getPoolInfo()
if result == nil: if result == nil:
raise rateLimitError() raise rateLimitError()
@ -90,7 +92,6 @@ proc poolTokens*(amount: int) {.async.} =
if newToken != nil: if newToken != nil:
tokenPool.add newToken tokenPool.add newToken
echo getPoolInfo()
proc initTokenPool*(cfg: Config) {.async.} = proc initTokenPool*(cfg: Config) {.async.} =
clientPool = HttpPool() clientPool = HttpPool()