Refactor fetch helper, fix list pagination

This commit is contained in:
Zed 2020-06-15 16:40:27 +02:00
parent 7643293f6b
commit 6167e7dc49
3 changed files with 20 additions and 29 deletions

View file

@ -1,6 +1,8 @@
import httpclient, asyncdispatch, options, times, strutils, uri import httpclient, asyncdispatch, options, times, strutils, uri
import packedjson import packedjson
import types, agents, tokens, consts, parserutils import types, tokens, consts, parserutils
const rl = "x-rate-limit-"
proc genParams*(pars: openarray[(string, string)] = @[]; proc genParams*(pars: openarray[(string, string)] = @[];
cursor=""): seq[(string, string)] = cursor=""): seq[(string, string)] =
@ -10,55 +12,43 @@ proc genParams*(pars: openarray[(string, string)] = @[];
if cursor.len > 0: if cursor.len > 0:
result &= ("cursor", cursor) result &= ("cursor", cursor)
proc genHeaders*(token: Token): HttpHeaders = proc genHeaders*(token: Token = nil): HttpHeaders =
result = newHttpHeaders({ result = newHttpHeaders({
"DNT": "1",
"authorization": auth, "authorization": auth,
"content-type": "application/json", "content-type": "application/json",
"user-agent": getAgent(),
"x-guest-token": if token == nil: "" else: token.tok, "x-guest-token": if token == nil: "" else: token.tok,
"x-twitter-active-user": "yes", "x-twitter-active-user": "yes",
"authority": "api.twitter.com", "authority": "api.twitter.com",
"accept-language": "en-US,en;q=0.9", "accept-language": "en-US,en;q=0.9",
"accept": "*/*", "accept": "*/*",
"DNT": "1"
}) })
proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} = proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
var var
token = await getToken() token = if oldApi: nil else: await getToken()
keepToken = true client = newAsyncHttpClient(headers=genHeaders(token))
proxy: Proxy = when defined(proxy): newProxy(prox) else: nil
client = newAsyncHttpClient(proxy=proxy, headers=genHeaders(token))
try: try:
let let
resp = await client.get($url) resp = await client.get($url)
body = await resp.body body = await resp.body
const rl = "x-rate-limit-" if not body.startsWith('{'):
echo resp.status, ": ", body
result = newJNull()
else:
result = parseJson(body)
if not oldApi and resp.headers.hasKey(rl & "limit"): if not oldApi and resp.headers.hasKey(rl & "limit"):
token.remaining = parseInt(resp.headers[rl & "remaining"]) token.remaining = parseInt(resp.headers[rl & "remaining"])
token.reset = fromUnix(parseInt(resp.headers[rl & "reset"])) token.reset = fromUnix(parseInt(resp.headers[rl & "reset"]))
if resp.status != $Http200: if result.getError notin {invalidToken, forbidden, badToken}:
if "Bad guest token" in body: token.release()
keepToken = false except Exception:
return newJNull()
elif not body.startsWith('{'):
echo resp.status, " ", body
return newJNull()
result = parseJson(body)
if result{"errors"}.notNull and result.getError == forbidden:
keepToken = false
echo "bad token"
except:
echo "error: ", url echo "error: ", url
result = newJNull() result = newJNull()
finally: finally:
if keepToken:
token.release()
try: client.close() try: client.close()
except: discard except: discard

View file

@ -35,7 +35,7 @@ proc createListRouter*(cfg: Config) =
let let
prefs = cookiePrefs() prefs = cookiePrefs()
list = await getCachedList(@"name", @"list") list = await getCachedList(@"name", @"list")
members = await getListMembers(list) members = await getListMembers(list, getCursor())
respList(list, members, renderTimelineUsers(members, prefs, request.path)) respList(list, members, renderTimelineUsers(members, prefs, request.path))
get "/i/lists/@id/?": get "/i/lists/@id/?":

View file

@ -9,7 +9,6 @@ type
remaining*: int remaining*: int
reset*: Time reset*: Time
init*: Time init*: Time
# agent*: string
Error* = enum Error* = enum
null = 0 null = 0
@ -18,9 +17,11 @@ type
doesntExist = 34 doesntExist = 34
userNotFound = 50 userNotFound = 50
suspended = 63 suspended = 63
rateLimited = 88
invalidToken = 89 invalidToken = 89
listIdOrSlug = 112 listIdOrSlug = 112
forbidden = 200 forbidden = 200
badToken = 239
noCsrf = 353 noCsrf = 353
Profile* = object Profile* = object