From ff61d97a1dc37ffc4f65a0dbfabf54952a83e01f Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 20 Jan 2023 04:54:19 +0100 Subject: [PATCH] Optimize profile fetching and caching --- src/api.nim | 31 +++++++++++++--------------- src/consts.nim | 3 ++- src/experimental/parser/graphql.nim | 5 +++++ src/experimental/types/graphuser.nim | 2 ++ src/redis_cache.nim | 11 +++++----- src/tokens.nim | 2 +- src/types.nim | 1 + 7 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/api.nim b/src/api.nim index 392795c..dd94b69 100644 --- a/src/api.nim +++ b/src/api.nim @@ -4,11 +4,22 @@ import packedjson import types, query, formatters, consts, apiutils, parser import experimental/parser as newParser -proc getGraphUser*(id: string): Future[User] {.async.} = +proc getGraphUser*(username: string): Future[User] {.async.} = + if username.len == 0: return + let + variables = """{ + "screen_name": "$1", + "withSafetyModeUserFields": false, + "withSuperFollowsUserFields": false + }""" % [username] + js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName) + result = parseGraphUser(js) + +proc getGraphUserById*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let - variables = %*{"userId": id, "withSuperFollowsUserFields": true} - js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId) + variables = """{"userId": "$1", "withSuperFollowsUserFields": true}""" % [id] + js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId) result = parseGraphUser(js) proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = @@ -47,20 +58,6 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} = url = listTimeline ? ps result = parseTimeline(await fetch(url, Api.timeline), after) -proc getUser*(username: string): Future[User] {.async.} = - if username.len == 0: return - let - ps = genParams({"screen_name": username}) - json = await fetchRaw(userShow ? ps, Api.userShow) - result = parseUser(json, username) - -proc getUserById*(userId: string): Future[User] {.async.} = - if userId.len == 0: return - let - ps = genParams({"user_id": userId}) - json = await fetchRaw(userShow ? ps, Api.userShow) - result = parseUser(json) - proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} = if id.len == 0: return let diff --git a/src/consts.nim b/src/consts.nim index 3687a54..2d3ea56 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -19,7 +19,8 @@ const tweet* = timelineApi / "conversation" graphql = api / "graphql" - graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" + graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" + graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug" graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers" diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index b00ab24..4431db3 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,9 +1,14 @@ +import options import jsony import user, ../types/[graphuser, graphlistmembers] from ../../types import User, Result, Query, QueryKind proc parseGraphUser*(json: string): User = let raw = json.fromJson(GraphUser) + + if raw.data.user.result.reason.get("") == "Suspended": + return User(suspended: true) + result = toUser raw.data.user.result.legacy result.id = raw.data.user.result.restId diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index dded4eb..e13383a 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,3 +1,4 @@ +import options import user type @@ -10,3 +11,4 @@ type UserResult = object legacy*: RawUser restId*: string + reason*: Option[string] diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 469157a..b91d153 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} = pool.withAcquire(r): result = await r.hGet(name.uidKey, name) if result == redisNil: - let user = await getUser(username) + let user = await getGraphUser(username) if user.suspended: return "suspended" else: - await cacheUserId(name, user.id) + await all(cacheUserId(name, user.id), cache(user)) return user.id proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} = @@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} = if prof != redisNil: prof.deserialize(User) elif fetch: - let userId = await getUserId(username) - result = await getGraphUser(userId) + result = await getGraphUser(username) await cache(result) proc getCachedUsername*(userId: string): Future[string] {.async.} = @@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} = if username != redisNil: result = username else: - let user = await getUserById(userId) + let user = await getGraphUserById(userId) result = user.username await setEx(key, baseCacheTime, result) + if result.len > 0 and user.id.len > 0: + await all(cacheUserId(result, user.id), cache(user)) proc getCachedTweet*(id: int64): Future[Tweet] {.async.} = if id == 0: return diff --git a/src/tokens.nim b/src/tokens.nim index 4bbcd81..e3b916a 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,7 +41,7 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500 + of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500 of Api.timeline: 187 else: 180 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index 061ec8a..07f9bf7 100644 --- a/src/types.nim +++ b/src/types.nim @@ -17,6 +17,7 @@ type listBySlug listMembers userRestId + userScreenName status RateLimit* = object