From 624394430c0989d18c279153006c6a7e48f4dd03 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 8 Aug 2023 02:09:56 +0200 Subject: [PATCH] Use legacy timeline/user endpoint for Tweets tab --- src/api.nim | 10 ++++++++ src/apiutils.nim | 2 +- src/consts.nim | 23 +++++++----------- src/parser.nim | 52 +++++++++++++++++++++++++++++++++++++++-- src/parserutils.nim | 6 +++++ src/routes/timeline.nim | 17 +------------- src/tokens.nim | 8 +++---- src/types.nim | 1 + 8 files changed, 81 insertions(+), 38 deletions(-) diff --git a/src/api.nim b/src/api.nim index c7dc0e0..c313aa2 100644 --- a/src/api.nim +++ b/src/api.nim @@ -40,6 +40,16 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi # url = oldUserTweets / (id & ".json") ? ps # result = parseTimeline(await fetch(url, Api.timeline), after) +proc getUserTimeline*(id: string; after=""): Future[Profile] {.async.} = + var ps = genParams({"id": id}) + if after.len > 0: + ps.add ("down_cursor", after) + + let + url = legacyUserTweets ? ps + js = await fetch(url, Api.userTimeline) + result = parseUserTimeline(js, after) + proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let diff --git a/src/apiutils.nim b/src/apiutils.nim index c0c01d4..1da971a 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -16,8 +16,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; for p in pars: result &= p if ext: - result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified") result &= ("include_ext_alt_text", "1") + result &= ("include_ext_media_stats", "1") result &= ("include_ext_media_availability", "1") if count.len > 0: result &= ("count", count) diff --git a/src/consts.nim b/src/consts.nim index 80a098f..a25f6ea 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,6 +7,7 @@ const api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") + legacyUserTweets* = api / "1.1/timeline/user.json" photoRail* = api / "1.1/statuses/media_timeline.json" userSearch* = api / "1.1/users/search.json" tweetSearch* = api / "1.1/search/universal.json" @@ -28,28 +29,20 @@ const graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" timelineParams* = { - "cards_platform": "Web-13", - "tweet_mode": "extended", - "ui_lang": "en-US", - "send_error_codes": "1", - "simple_quoted_tweet": "1", - "skip_status": "1", - "include_blocked_by": "0", - "include_blocking": "0", - "include_can_dm": "0", "include_can_media_tag": "1", "include_cards": "1", - "include_composer_source": "0", "include_entities": "1", - "include_ext_is_blue_verified": "1", - "include_ext_media_color": "0", - "include_followed_by": "0", - "include_mute_edge": "0", "include_profile_interstitial_type": "0", "include_quote_count": "1", "include_reply_count": "1", "include_user_entities": "1", - "include_want_retweets": "0", + "include_ext_reply_count": "1", + "include_ext_is_blue_verified": "1", + "include_ext_media_color": "0", + "cards_platform": "Web-13", + "tweet_mode": "extended", + "send_error_codes": "1", + "simple_quoted_tweet": "1" }.toSeq gqlFeatures* = """{ diff --git a/src/parser.nim b/src/parser.nim index 991ca6e..c7d8bd1 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, times, math +import strutils, options, times, math, tables import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard @@ -81,7 +81,7 @@ proc parseGif(js: JsonNode): Gif = proc parseVideo(js: JsonNode): Video = result = Video( thumb: js{"media_url_https"}.getImageStr, - views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), + views: getVideoViewCount(js), available: true, title: js{"ext_alt_text"}.getStr, durationMs: js{"video_info", "duration_millis"}.getInt @@ -313,6 +313,54 @@ proc parseTweetSearch*(js: JsonNode; after=""): Timeline = if result.content.len > 0: result.bottom = $(result.content[^1][0].id - 1) +proc parseUserTimelineTweet(tweet: JsonNode; users: TableRef[string, User]): Tweet = + result = parseTweet(tweet, tweet{"card"}) + + if result.isNil or not result.available: + return + + with user, tweet{"user"}: + let userId = user{"id_str"}.getStr + if user{"ext_is_blue_verified"}.getBool(false): + users[userId].verified = users[userId].verified or true + result.user = users[userId] + +proc parseUserTimeline*(js: JsonNode; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) + + if js.kind == JNull or "response" notin js or "twitter_objects" notin js: + return + + var users = newTable[string, User]() + for userId, user in js{"twitter_objects", "users"}: + users[userId] = parseUser(user) + + for entity in js{"response", "timeline"}: + let + tweetId = entity{"tweet", "id"}.getId + isPinned = entity{"tweet", "is_pinned"}.getBool(false) + + with tweet, js{"twitter_objects", "tweets", $tweetId}: + var parsed = parseUserTimelineTweet(tweet, users) + + if not parsed.isNil and parsed.available: + if parsed.quote.isSome: + parsed.quote = some parseUserTimelineTweet(tweet{"quoted_status"}, users) + + if parsed.retweet.isSome: + let retweet = parseUserTimelineTweet(tweet{"retweeted_status"}, users) + if retweet.quote.isSome: + retweet.quote = some parseUserTimelineTweet(tweet{"retweeted_status", "quoted_status"}, users) + parsed.retweet = some retweet + + if isPinned: + parsed.pinned = true + result.pinned = some parsed + else: + result.tweets.content.add parsed + + result.tweets.bottom = js{"response", "cursor", "bottom"}.getStr + # proc finalizeTweet(global: GlobalObjects; id: string): Tweet = # let intId = if id.len > 0: parseBiggestInt(id) else: 0 # result = global.tweets.getOrDefault(id, Tweet(id: intId)) diff --git a/src/parserutils.nim b/src/parserutils.nim index f28bd52..c65052e 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -148,6 +148,12 @@ proc getMp4Resolution*(url: string): int = # cannot determine resolution (e.g. m3u8/non-mp4 video) return 0 +proc getVideoViewCount*(js: JsonNode): string = + with stats, js{"ext_media_stats"}: + return stats{"view_count"}.getStr($stats{"viewCount"}.getInt) + + return $js{"mediaStats", "viewCount"}.getInt(0) + proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 82dc45b..8d02b68 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -53,7 +53,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result = case query.kind - # of posts: await getTimeline(userId, after) + of posts: await getUserTimeline(userId, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) else: Profile(tweets: await getTweetSearch(query, after)) @@ -63,21 +63,6 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result.tweets.query = query - if result.user.protected or result.user.suspended: - return - - if query.kind == posts: - if result.user.verified: - for chain in result.tweets.content: - if chain[0].user.id == result.user.id: - chain[0].user.verified = true - if not skipPinned and result.user.pinnedTweet > 0 and after.len == 0: - let tweet = await getCachedTweet(result.user.pinnedTweet) - if not tweet.isNil: - tweet.pinned = true - tweet.user = result.user - result.pinned = some tweet - proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: diff --git a/src/tokens.nim b/src/tokens.nim index b69786e..decf228 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -44,10 +44,10 @@ proc getPoolJson*(): JsonNode = of Api.search: 100000 of Api.photoRail: 180 of Api.timeline: 187 - of Api.userTweets: 300 + of Api.userTweets, Api.userTimeline: 300 of Api.userTweetsAndReplies, Api.userRestId, - Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500 - of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 + Api.userScreenName, Api.tweetDetail, Api.tweetResult, + Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining @@ -161,6 +161,6 @@ proc initTokenPool*(cfg: Config) {.async.} = enableLogging = cfg.enableDebug while true: - if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens: + if tokenPool.countIt(not it.isLimited(Api.userTimeline)) < cfg.minTokens: await poolTokens(min(4, cfg.minTokens - tokenPool.len)) await sleepAsync(2000) diff --git a/src/types.nim b/src/types.nim index 5db9ec3..1a47d25 100644 --- a/src/types.nim +++ b/src/types.nim @@ -18,6 +18,7 @@ type tweetDetail tweetResult timeline + userTimeline photoRail search userSearch