diff --git a/src/api.nim b/src/api.nim index b23aa87..d99eb3d 100644 --- a/src/api.nim +++ b/src/api.nim @@ -7,20 +7,20 @@ import experimental/parser as newParser proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return let - variables = %*{"screen_name": username} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"screen_name": "$1"}""" % username + params = {"variables": variables, "features": gqlFeatures} js = await fetchRaw(graphUser ? params, Api.userScreenName) result = parseGraphUser(js) proc getGraphUserById*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let - variables = %*{"userId": id} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"rest_id": "$1"}""" % id + params = {"variables": variables, "features": gqlFeatures} js = await fetchRaw(graphUserById ? params, Api.userRestId) result = parseGraphUser(js) -proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} = +proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" @@ -40,7 +40,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = variables = listTweetsVariables % [id, cursor] params = {"variables": variables, "features": gqlFeatures} js = await fetch(graphListTweets ? params, Api.listTweets) - result = parseGraphTimeline(js, "list", after) + result = parseGraphTimeline(js, "list", after).tweets proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let @@ -50,8 +50,8 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphList*(id: string): Future[List] {.async.} = let - variables = %*{"listId": id} - params = {"variables": $variables, "features": gqlFeatures} + variables = """{"listId": "$1"}""" % id + params = {"variables": variables, "features": gqlFeatures} result = parseGraphList(await fetch(graphListById ? params, Api.list)) proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} = @@ -72,7 +72,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return let - variables = tweetResultVariables % id + variables = """{"rest_id": "$1"}""" % id params = {"variables": variables, "features": gqlFeatures} js = await fetch(graphTweetResult ? params, Api.tweetResult) result = parseGraphTweetResult(js) @@ -95,10 +95,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = if after.len > 0: result.replies = await getReplies(id, after) -proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} = +proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = let q = genQueryParam(query) if q.len == 0 or q == emptyQuery: - return Result[Tweet](query: query, beginning: true) + return Profile(tweets: Timeline(query: query, beginning: true)) var variables = %*{ @@ -112,8 +112,8 @@ proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} = if after.len > 0: variables["cursor"] = % after let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch(await fetch(url, Api.search), after) - result.query = query + result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after)) + result.tweets.query = query proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = if query.text.len == 0: diff --git a/src/consts.nim b/src/consts.nim index f22581f..184f9da 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,7 +2,7 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" api = parseUri("https://api.twitter.com") activate* = $(api / "1.1/guest/activate.json") @@ -11,18 +11,18 @@ const userSearch* = api / "1.1/users/search.json" graphql = api / "graphql" - graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName" - graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId" - graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets" - graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies" - graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia" - graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail" - graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId" + graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" + graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" + graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2" + graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2" + graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" + graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2" + graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" - graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline" + graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" timelineParams* = { "include_profile_interstitial_type": "0", @@ -49,10 +49,13 @@ const }.toSeq gqlFeatures* = """{ + "android_graphql_skip_api_media_color_palette": false, "blue_business_profile_image_shape_enabled": false, + "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": false, "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, + "hidden_profile_likes_enabled": false, "highlights_tweets_tab_ui_enabled": false, "interactive_text_enabled": false, "longform_notetweets_consumption_enabled": true, @@ -64,15 +67,25 @@ const "responsive_web_graphql_exclude_directive_enabled": true, "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, "responsive_web_graphql_timeline_navigation_enabled": false, + "responsive_web_media_download_video_enabled": false, "responsive_web_text_conversations_enabled": false, + "responsive_web_twitter_article_tweet_consumption_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, "rweb_lists_timeline_redesign_enabled": true, "spaces_2022_h2_clipping": true, "spaces_2022_h2_spaces_communities": true, "standardized_nudges_misinfo": false, + "subscriptions_verification_info_enabled": true, + "subscriptions_verification_info_reason_enabled": true, + "subscriptions_verification_info_verified_since_enabled": true, + "super_follow_badge_privacy_enabled": false, + "super_follow_exclusive_tweet_notifications_enabled": false, + "super_follow_tweet_api_enabled": false, + "super_follow_user_api_enabled": false, "tweet_awards_web_tipping_enabled": false, "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false, "tweetypie_unmention_optimization_enabled": false, + "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, "verified_phone_label_enabled": false, "vibe_api_enabled": false, "view_counts_everywhere_api_enabled": false @@ -81,41 +94,15 @@ const tweetVariables* = """{ "focalTweetId": "$1", $2 - "withBirdwatchNotes": false, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false -}""" - - tweetResultVariables* = """{ - "tweetId": "$1", - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false, - "withCommunity": false + "includeHasBirdwatchNotes": false }""" userTweetsVariables* = """{ - "userId": "$1", $2 - "count": 20, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false, - "withV2Timeline": true + "rest_id": "$1", $2 + "count": 20 }""" listTweetsVariables* = """{ - "listId": "$1", $2 - "count": 20, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false + "rest_id": "$1", $2 + "count": 20 }""" diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 0f08c4f..b9da7c4 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -9,12 +9,12 @@ proc parseGraphUser*(json: string): User = let raw = json.fromJson(GraphUser) - if raw.data.user.result.reason.get("") == "Suspended": + if raw.data.userResult.result.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = toUser raw.data.user.result.legacy - result.id = raw.data.user.result.restId - result.verified = result.verified or raw.data.user.result.isBlueVerified + result = toUser raw.data.userResult.result.legacy + result.id = raw.data.userResult.result.restId + result.verified = result.verified or raw.data.userResult.result.isBlueVerified proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index 478e7f3..c30eed9 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -3,7 +3,7 @@ import user type GraphUser* = object - data*: tuple[user: UserData] + data*: tuple[userResult: UserData] UserData* = object result*: UserResult @@ -12,4 +12,4 @@ type legacy*: RawUser restId*: string isBlueVerified*: bool - reason*: Option[string] + unavailableReason*: Option[string] diff --git a/src/parser.nim b/src/parser.nim index 38dbb24..7b178f3 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -29,7 +29,7 @@ proc parseUser(js: JsonNode; id=""): User = result.expandUserEntities(js) proc parseGraphUser(js: JsonNode): User = - let user = ? js{"user_results", "result"} + let user = ? js{"user_result", "result"} result = parseUser(user{"legacy"}) if "is_blue_verified" in user: @@ -262,6 +262,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.gif = some(parseGif(m)) else: discard + with url, m{"url"}: + if result.text.endsWith(url.getStr): + result.text.removeSuffix(url.getStr) + result.text = result.text.strip() + with jsWithheld, js{"withheld_in_countries"}: let withheldInCountries: seq[string] = if jsWithheld.kind != JArray: @[] @@ -294,16 +299,6 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet = else: result.retweet = some Tweet() -proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = - let pin = js{"pinEntry", "entry", "entryId"}.getStr - if pin.len == 0: return - - let id = pin.getId - if id notin global.tweets: return - - global.tweets[id].pinned = true - return finalizeTweet(global, id) - proc parseGlobalObjects(js: JsonNode): GlobalObjects = result = GlobalObjects() let @@ -314,7 +309,7 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects = result.users[k] = parseUser(v, k) for k, v in tweets: - var tweet = parseTweet(v, v{"card"}) + var tweet = parseTweet(v, v{"tweet_card"}) if tweet.user.id in result.users: tweet.user = result.users[tweet.user.id] result.tweets[k] = tweet @@ -324,11 +319,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod return for i in js: - when T is Tweet: - if res.beginning and i{"pinEntry"}.notNull: - with pin, parsePin(i, global): - res.content.add pin - with r, i{"replaceEntry", "entry"}: if "top" in r{"entryId"}.getStr: res.top = r.getCursor @@ -369,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = proc parsePhotoRail*(js: JsonNode): PhotoRail = for tweet in js: let - t = parseTweet(tweet, js{"card"}) + t = parseTweet(tweet, js{"tweet_card"}) url = if t.photos.len > 0: t.photos[0] elif t.video.isSome: get(t.video).thumb elif t.gif.isSome: get(t.gif).thumb @@ -387,13 +377,17 @@ proc parseGraphTweet(js: JsonNode): Tweet = of "TweetUnavailable": return Tweet() of "TweetTombstone": - return Tweet(text: js{"tombstone", "text"}.getTombstone) + with text, js{"tombstone", "richText"}: + return Tweet(text: text.getTombstone) + with text, js{"tombstone", "text"}: + return Tweet(text: text.getTombstone) + return Tweet() of "TweetPreviewDisplay": return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.") of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}) - var jsCard = copy(js{"card", "legacy"}) + var jsCard = copy(js{"tweet_card", "legacy"}) if jsCard.kind != JNull: var values = newJObject() for val in jsCard["binding_values"]: @@ -401,6 +395,7 @@ proc parseGraphTweet(js: JsonNode): Tweet = jsCard["binding_values"] = values result = parseTweet(js{"legacy"}, jsCard) + result.id = js{"rest_id"}.getId result.user = parseGraphUser(js{"core"}) with noteTweet, js{"note_tweet", "note_tweet_results", "result"}: @@ -414,32 +409,31 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = for t in js{"content", "items"}: let entryId = t{"entryId"}.getStr if "cursor-showmore" in entryId: - let cursor = t{"item", "itemContent", "value"} + let cursor = t{"item", "content", "value"} result.thread.cursor = cursor.getStr result.thread.hasMore = true elif "tweet" in entryId: - let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"}) + let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"}) result.thread.content.add tweet - if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread": + if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread": result.self = true proc parseGraphTweetResult*(js: JsonNode): Tweet = - with tweet, js{"data", "tweetResult", "result"}: + with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet) proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) - let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"} + let instructions = ? js{"data", "timeline_response", "instructions"} if instructions.len == 0: return for e in instructions[0]{"entries"}: let entryId = e{"entryId"}.getStr - # echo entryId if entryId.startsWith("tweet"): - with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + with tweetResult, e{"content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult) if not tweet.available: @@ -454,7 +448,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = let tweet = Tweet( id: parseBiggestInt(id), available: false, - text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone + text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone ) if id == tweetId: @@ -468,34 +462,42 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = else: result.replies.content.add thread elif entryId.startsWith("cursor-bottom"): - result.replies.bottom = e{"content", "itemContent", "value"}.getStr + result.replies.bottom = e{"content", "content", "value"}.getStr -proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) let instructions = - if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"} - else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"} + if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"} + else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} if instructions.len == 0: return for i in instructions: - if i{"type"}.getStr == "TimelineAddEntries": + if i{"__typename"}.getStr == "TimelineAddEntries": for e in i{"entries"}: let entryId = e{"entryId"}.getStr if entryId.startsWith("tweet"): - with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + with tweetResult, e{"content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult) if not tweet.available: tweet.id = parseBiggestInt(entryId.getId()) - result.content.add tweet - elif entryId.startsWith("profile-conversation") or entryId.startsWith("homeConversation"): + result.tweets.content.add tweet + elif "-conversation-" in entryId or entryId.startsWith("homeConversation"): let (thread, self) = parseGraphThread(e) - for tweet in thread.content: - result.content.add tweet + result.tweets.content.add thread elif entryId.startsWith("cursor-bottom"): - result.bottom = e{"content", "value"}.getStr + result.tweets.bottom = e{"content", "value"}.getStr + if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": + with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}: + let tweet = parseGraphTweet(tweetResult) + tweet.pinned = true + if not tweet.available and tweet.tombstone.len == 0: + let entryId = i{"entry", "entryId"}.getEntryId + if entryId.len > 0: + tweet.id = parseBiggestInt(entryId) + result.pinned = some tweet proc parseGraphSearch*(js: JsonNode; after=""): Timeline = result = Timeline(beginning: after.len == 0) diff --git a/src/routes/embed.nim b/src/routes/embed.nim index baaec68..994364b 100644 --- a/src/routes/embed.nim +++ b/src/routes/embed.nim @@ -10,22 +10,22 @@ export api, embed, vdom, tweet, general, router_utils proc createEmbedRouter*(cfg: Config) = router embed: get "/i/videos/tweet/@id": - let convo = await getTweet(@"id") - if convo == nil or convo.tweet == nil or convo.tweet.video.isNone: + let tweet = await getGraphTweetResult(@"id") + if tweet == nil or tweet.video.isNone: resp Http404 - resp renderVideoEmbed(convo.tweet, cfg, request) + resp renderVideoEmbed(tweet, cfg, request) get "/@user/status/@id/embed": let - convo = await getTweet(@"id") + tweet = await getGraphTweetResult(@"id") prefs = cookiePrefs() path = getPath() - if convo == nil or convo.tweet == nil: + if tweet == nil: resp Http404 - resp renderTweetEmbed(convo.tweet, path, prefs, cfg, request) + resp renderTweetEmbed(tweet, path, prefs, cfg, request) get "/embed/Tweet.html": let id = @"id" diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 1323ed3..8eec399 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -27,14 +27,12 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. else: var q = query q.fromUser = names - profile = Profile( - tweets: await getGraphSearch(q, after), - # this is kinda dumb - user: User( - username: name, - fullname: names.join(" | "), - userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" - ) + profile = await getGraphSearch(q, after) + # this is kinda dumb + profile.user = User( + username: name, + fullname: names.join(" | "), + userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" ) if profile.user.suspended: @@ -61,29 +59,29 @@ template respRss*(rss, page) = proc createRssRouter*(cfg: Config) = router rss: - get "/search/rss": - cond cfg.enableRss - if @"q".len > 200: - resp Http400, showError("Search input too long.", cfg) + # get "/search/rss": + # cond cfg.enableRss + # if @"q".len > 200: + # resp Http400, showError("Search input too long.", cfg) - let query = initQuery(params(request)) - if query.kind != tweets: - resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) + # let query = initQuery(params(request)) + # if query.kind != tweets: + # resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) - let - cursor = getCursor() - key = redisKey("search", $hash(genQueryUrl(query)), cursor) + # let + # cursor = getCursor() + # key = redisKey("search", $hash(genQueryUrl(query)), cursor) - var rss = await getCachedRss(key) - if rss.cursor.len > 0: - respRss(rss, "Search") + # var rss = await getCachedRss(key) + # if rss.cursor.len > 0: + # respRss(rss, "Search") - let tweets = await getGraphSearch(query, cursor) - rss.cursor = tweets.bottom - rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) + # let tweets = await getGraphSearch(query, cursor) + # rss.cursor = tweets.bottom + # rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) - await cacheRss(key, rss) - respRss(rss, "Search") + # await cacheRss(key, rss) + # respRss(rss, "Search") get "/@name/rss": cond cfg.enableRss @@ -112,7 +110,7 @@ proc createRssRouter*(cfg: Config) = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) - of "search": initQuery(params(request), name=name) + # of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) let searchKey = if tab != "search": "" diff --git a/src/routes/search.nim b/src/routes/search.nim index 02c14e3..ed2c397 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -34,11 +34,15 @@ proc createSearchRouter*(cfg: Config) = users = Result[User](beginning: true, query: query) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title) of tweets: - let - tweets = await getGraphSearch(query, getCursor()) - rss = "/search/rss?" & genQueryUrl(query) - resp renderMain(renderTweetSearch(tweets, prefs, getPath()), - request, cfg, prefs, title, rss=rss) + # let + # tweets = await getGraphSearch(query, getCursor()) + # rss = "/search/rss?" & genQueryUrl(query) + # resp renderMain(renderTweetSearch(tweets, prefs, getPath()), + # request, cfg, prefs, title, rss=rss) + var fakeTimeline = Timeline(beginning: true) + fakeTimeline.content.add Tweet(tombstone: "Tweet search is unavailable for now") + + resp renderMain(renderTweetSearch(fakeTimeline, prefs, getPath()), request, cfg, prefs, title) else: resp Http404, showError("Invalid search", cfg) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 331b8ae..4ac60d2 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -45,34 +45,24 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; after.setLen 0 let - timeline = - case query.kind - of posts: getGraphUserTweets(userId, TimelineKind.tweets, after) - of replies: getGraphUserTweets(userId, TimelineKind.replies, after) - of media: getGraphUserTweets(userId, TimelineKind.media, after) - else: getGraphSearch(query, after) - rail = skipIf(skipRail or query.kind == media, @[]): getCachedPhotoRail(name) - user = await getCachedUser(name) + user = getCachedUser(name) - var pinned: Option[Tweet] - if not skipPinned and user.pinnedTweet > 0 and - after.len == 0 and query.kind in {posts, replies}: - let tweet = await getCachedTweet(user.pinnedTweet) - if not tweet.isNil: - tweet.pinned = true - tweet.user = user - pinned = some tweet + result = + case query.kind + of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) + of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) + of media: await getGraphUserTweets(userId, TimelineKind.media, after) + else: Profile(tweets: Timeline(beginning: true, content: @[Chain(content: + @[Tweet(tombstone: "Tweet search is unavailable for now")] + )])) + # else: await getGraphSearch(query, after) - result = Profile( - user: user, - pinned: pinned, - tweets: await timeline, - photoRail: await rail - ) + result.user = await user + result.photoRail = await rail if result.user.protected or result.user.suspended: return @@ -83,8 +73,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = if query.fromUser.len != 1: let - timeline = await getGraphSearch(query, after) - html = renderTweetSearch(timeline, prefs, getPath()) + # timeline = await getGraphSearch(query, after) + timeline = Profile(tweets: Timeline(beginning: true, content: @[Chain(content: + @[Tweet(tombstone: "This features is unavailable for now")] + )])) + html = renderTweetSearch(timeline.tweets, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) @@ -138,7 +131,7 @@ proc createTimelineRouter*(cfg: Config) = # used for the infinite scroll feature if @"scroll".len > 0: if query.fromUser.len != 1: - var timeline = await getGraphSearch(query, after) + var timeline = (await getGraphSearch(query, after)).tweets if timeline.content.len == 0: resp Http404 timeline.beginning = true resp $renderTweetSearch(timeline, prefs, getPath()) diff --git a/src/sass/tweet/thread.scss b/src/sass/tweet/thread.scss index 5fbad21..19fb3e0 100644 --- a/src/sass/tweet/thread.scss +++ b/src/sass/tweet/thread.scss @@ -110,3 +110,29 @@ margin-left: 58px; padding: 7px 0; } + +.timeline-item.thread.more-replies-thread { + padding: 0 0.75em; + + &::before { + top: 40px; + margin-bottom: 31px; + } + + .more-replies { + display: flex; + padding-top: unset !important; + margin-top: 8px; + + &::before { + display: inline-block; + position: relative; + top: -1px; + line-height: 0.4em; + } + + .more-replies-text { + display: inline; + } + } +} diff --git a/src/tokens.nim b/src/tokens.nim index 6ef81f5..6643de3 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,11 +41,10 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.timeline: 187 - of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, - Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, - Api.tweetDetail, Api.tweetResult, Api.search: 500 + of Api.timeline: 180 + of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId, + Api.userScreenName, Api.tweetDetail, Api.tweetResult, Api.search: 500 + of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 reqs = maxReqs - token.apis[api].remaining diff --git a/src/types.nim b/src/types.nim index 4dca5f0..e7f3303 100644 --- a/src/types.nim +++ b/src/types.nim @@ -222,7 +222,7 @@ type after*: Chain replies*: Result[Chain] - Timeline* = Result[Tweet] + Timeline* = Result[Chain] Profile* = object user*: User @@ -274,3 +274,6 @@ type proc contains*(thread: Chain; tweet: Tweet): bool = thread.content.anyIt(it.id == tweet.id) + +proc add*(timeline: var seq[Chain]; tweet: Tweet) = + timeline.add Chain(content: @[tweet]) diff --git a/src/views/rss.nimf b/src/views/rss.nimf index 96f6466..ce2518a 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -56,24 +56,29 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} #end if #end proc # -#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string = +#proc renderRssTweets(tweets: seq[Chain]; cfg: Config; userId=""): string = #let urlPrefix = getUrlPrefix(cfg) #var links: seq[string] -#for t in tweets: -# let retweet = if t.retweet.isSome: t.user.username else: "" -# let tweet = if retweet.len > 0: t.retweet.get else: t -# let link = getLink(tweet) -# if link in links: continue -# end if -# links.add link - - ${getTitle(tweet, retweet)} - @${tweet.user.username} - - ${getRfc822Time(tweet)} - ${urlPrefix & link} - ${urlPrefix & link} - +#for c in tweets: +# for t in c.content: +# if userId.len > 0 and t.user.id != userId: continue +# end if +# +# let retweet = if t.retweet.isSome: t.user.username else: "" +# let tweet = if retweet.len > 0: t.retweet.get else: t +# let link = getLink(tweet) +# if link in links: continue +# end if +# links.add link + + ${getTitle(tweet, retweet)} + @${tweet.user.username} + + ${getRfc822Time(tweet)} + ${urlPrefix & link} + ${urlPrefix & link} + +# end for #end for #end proc # @@ -102,13 +107,13 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} 128 #if profile.tweets.content.len > 0: -${renderRssTweets(profile.tweets.content, cfg)} +${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)} #end if #end proc # -#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string = +#proc renderListRss*(tweets: seq[Chain]; list: List; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}" #result = "" @@ -125,7 +130,7 @@ ${renderRssTweets(tweets, cfg)} #end proc # -#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string = +#proc renderSearchRss*(tweets: seq[Chain]; name, param: string; cfg: Config): string = #let link = &"{getUrlPrefix(cfg)}/search" #let escName = xmltree.escape(name) #result = "" diff --git a/src/views/search.nim b/src/views/search.nim index 72c59f5..401e6da 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -88,7 +88,7 @@ proc renderSearchPanel*(query: Query): VNode = span(class="search-title"): text "Near" genInput("near", "", query.near, "Location...", autofocus=false) -proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; +proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string; pinned=none(Tweet)): VNode = let query = results.query buildHtml(tdiv(class="timeline-container")): diff --git a/src/views/timeline.nim b/src/views/timeline.nim index 54cad7a..8ae888e 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, strformat, sequtils, algorithm, uri, options +import strutils, strformat, algorithm, uri, options import karax/[karaxdsl, vdom] import ".."/[types, query, formatters] @@ -43,20 +43,18 @@ proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="thread-line")): let sortedThread = thread.sortedByIt(it.id) for i, tweet in sortedThread: + # thread has a gap, display "more replies" link + if i > 0 and tweet.replyId != sortedThread[i - 1].id: + tdiv(class="timeline-item thread more-replies-thread"): + tdiv(class="more-replies"): + a(class="more-replies-text", href=getLink(tweet)): + text "more replies" + let show = i == thread.high and sortedThread[0].id != tweet.threadId let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: "" renderTweet(tweet, prefs, path, class=(header & "thread"), index=i, last=(i == thread.high), showThread=show) -proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] = - result = @[it] - if it.retweet.isSome or it.replyId in threads: return - for t in tweets: - if t.id == result[0].replyId: - result.insert t - elif t.replyId == result[0].id: - result.add t - proc renderUser(user: User; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-item")): a(class="tweet-link", href=("/" & user.username)) @@ -89,7 +87,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode = else: renderNoMore() -proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; +proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; pinned=none(Tweet)): VNode = buildHtml(tdiv(class="timeline")): if not results.beginning: @@ -105,26 +103,26 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; else: renderNoneFound() else: - var - threads: seq[int64] - retweets: seq[int64] + var retweets: seq[int64] - for tweet in results.content: - let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 + for thread in results.content: + if thread.content.len == 1: + let + tweet = thread.content[0] + retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0 - if tweet.id in threads or rt in retweets or tweet.id in retweets or - tweet.pinned and prefs.hidePins: continue + if retweetId in retweets or tweet.id in retweets or + tweet.pinned and prefs.hidePins: + continue - let thread = results.content.threadFilter(threads, tweet) - if thread.len < 2: var hasThread = tweet.hasThread - if rt != 0: - retweets &= rt + if retweetId != 0 and tweet.retweet.isSome: + retweets &= retweetId hasThread = get(tweet.retweet).hasThread renderTweet(tweet, prefs, path, showThread=hasThread) else: - renderThread(thread, prefs, path) - threads &= thread.mapIt(it.id) + renderThread(thread.content, prefs, path) - renderMore(results.query, results.bottom) + if results.bottom.len > 0: + renderMore(results.query, results.bottom) renderToTop() diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 3338b71..f47ae9a 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -14,15 +14,14 @@ proc renderMiniAvatar(user: User; prefs: Prefs): VNode = buildHtml(): img(class=(prefs.getAvatarClass & " mini"), src=url) -proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode = +proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = buildHtml(tdiv): - if retweet.len > 0: - tdiv(class="retweet-header"): - span: icon "retweet", retweet & " retweeted" - - if tweet.pinned: + if pinned: tdiv(class="pinned"): span: icon "pin", "Pinned Tweet" + elif retweet.len > 0: + tdiv(class="retweet-header"): + span: icon "retweet", retweet & " retweeted" tdiv(class="tweet-header"): a(class="tweet-avatar", href=("/" & tweet.user.username)): @@ -290,7 +289,10 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; if tweet.quote.isSome: renderQuote(tweet.quote.get(), prefs, path) - let fullTweet = tweet + let + fullTweet = tweet + pinned = tweet.pinned + var retweet: string var tweet = fullTweet if tweet.retweet.isSome: @@ -303,7 +305,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; tdiv(class="tweet-body"): var views = "" - renderHeader(tweet, retweet, prefs) + renderHeader(tweet, retweet, pinned, prefs) if not afterTweet and index == 0 and tweet.reply.len > 0 and (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username): diff --git a/tests/test_card.py b/tests/test_card.py index 696b9d5..f84ddca 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -16,7 +16,12 @@ card = [ ['FluentAI/status/1116417904831029248', 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com', True] + 'theverge.com', True], + + ['nim_lang/status/1082989146040340480', + 'Nim in 2018: A short recap', + 'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.', + 'nim-lang.org', True] ] no_thumb = [ @@ -33,12 +38,7 @@ no_thumb = [ ['voidtarget/status/1133028231672582145', 'sinkingsugar/nimqt-example', 'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.', - 'github.com'], - - ['nim_lang/status/1082989146040340480', - 'Nim in 2018: A short recap', - 'Posted by u/miran1 - 36 votes and 46 comments', - 'reddit.com'] + 'github.com'] ] playable = [ @@ -53,17 +53,6 @@ playable = [ 'youtube.com'] ] -# promo = [ - # ['BangOlufsen/status/1145698701517754368', - # 'Upgrade your journey', '', - # 'www.bang-olufsen.com'], - - # ['BangOlufsen/status/1154934429900406784', - # 'Learn more about Beosound Shape', '', - # 'www.bang-olufsen.com'] -# ] - - class CardTest(BaseTestCase): @parameterized.expand(card) def test_card(self, tweet, title, description, destination, large): @@ -98,13 +87,3 @@ class CardTest(BaseTestCase): self.assert_element_visible('.card-overlay') if len(description) > 0: self.assert_text(description, c.description) - - # @parameterized.expand(promo) - # def test_card_promo(self, tweet, title, description, destination): - # self.open_nitter(tweet) - # c = Card(Conversation.main + " ") - # self.assert_text(title, c.title) - # self.assert_text(destination, c.destination) - # self.assert_element_visible('.video-overlay') - # if len(description) > 0: - # self.assert_text(description, c.description) diff --git a/tests/test_profile.py b/tests/test_profile.py index f9b5047..4c75ad2 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -66,8 +66,8 @@ class ProfileTest(BaseTestCase): self.assert_text(f'User "{username}" not found') def test_suspended(self): - self.open_nitter('user') - self.assert_text('User "user" has been suspended') + self.open_nitter('suspendme') + self.assert_text('User "suspendme" has been suspended') @parameterized.expand(banner_image) def test_banner_image(self, username, url): diff --git a/tests/test_search.py b/tests/test_search.py index 80ee36a..62c4640 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -2,8 +2,8 @@ from base import BaseTestCase from parameterized import parameterized -class SearchTest(BaseTestCase): - @parameterized.expand([['@mobile_test'], ['@mobile_test_2']]) - def test_username_search(self, username): - self.search_username(username) - self.assert_text(f'{username}') +#class SearchTest(BaseTestCase): + #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']]) + #def test_username_search(self, username): + #self.search_username(username) + #self.assert_text(f'{username}') diff --git a/tests/test_tweet.py b/tests/test_tweet.py index 9209e70..e4231a4 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -74,9 +74,9 @@ retweet = [ [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] ] -reply = [ - ['mobile_test/with_replies', 15] -] +# reply = [ +# ['mobile_test/with_replies', 15] +# ] class TweetTest(BaseTestCase): @@ -137,8 +137,8 @@ class TweetTest(BaseTestCase): self.open_nitter(tweet) self.assert_text('Tweet not found', '.error-panel') - @parameterized.expand(reply) - def test_thread(self, tweet, num): - self.open_nitter(tweet) - thread = self.find_element(f'.timeline > div:nth-child({num})') - self.assertIn(thread.get_attribute('class'), 'thread-line') + # @parameterized.expand(reply) + # def test_thread(self, tweet, num): + # self.open_nitter(tweet) + # thread = self.find_element(f'.timeline > div:nth-child({num})') + # self.assertIn(thread.get_attribute('class'), 'thread-line')