Rearchitect profile, support pins, Profile -> User

This commit is contained in:
Zed 2022-01-23 07:04:50 +01:00
parent 79b98a8081
commit 51ae076ea0
23 changed files with 374 additions and 285 deletions

View file

@ -1,9 +1,16 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, uri, strutils import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
import packedjson import packedjson
import types, query, formatters, consts, apiutils, parser import types, query, formatters, consts, apiutils, parser
import experimental/parser/user import experimental/parser/user
proc getGraphUser*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId)
result = parseGraphUser(js, id)
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let let
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false} variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
@ -16,6 +23,22 @@ proc getGraphList*(id: string): Future[List] {.async.} =
url = graphList ? {"variables": $variables} url = graphList ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.list)) result = parseGraphList(await fetch(url, Api.list))
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
let
variables = %*{
"listId": list.id,
"cursor": after,
"withSuperFollowsUserFields": false,
"withBirdwatchPivots": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false
}
url = graphListMembers ? {"variables": $variables}
result = parseGraphListMembers(await fetch(url, Api.listMembers), after)
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} = proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return if id.len == 0: return
let let
@ -23,44 +46,42 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
url = listTimeline ? ps url = listTimeline ? ps
result = parseTimeline(await fetch(url, Api.timeline), after) result = parseTimeline(await fetch(url, Api.timeline), after)
proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} = proc getUser*(username: string): Future[User] {.async.} =
if list.id.len == 0: return if username.len == 0: return
let
ps = genParams({"list_id": list.id}, after)
url = listMembers ? ps
result = parseListMembers(await fetch(url, Api.listMembers), after)
proc getProfile*(username: string): Future[Profile] {.async.} =
let let
ps = genParams({"screen_name": username}) ps = genParams({"screen_name": username})
json = await fetchRaw(userShow ? ps, Api.userShow) json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json, username) result = parseUser(json, username)
proc getProfileById*(userId: string): Future[Profile] {.async.} = proc getUserById*(userId: string): Future[User] {.async.} =
if userId.len == 0: return
let let
ps = genParams({"user_id": userId}) ps = genParams({"user_id": userId})
json = await fetchRaw(userShow ? ps, Api.userShow) json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json) result = parseUser(json)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} = proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
if id.len == 0: return
let let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = timeline / (id & ".json") ? ps url = timeline / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after) result = parseTimeline(await fetch(url, Api.timeline), after)
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} = proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after) let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url, Api.timeline), after) result = parseTimeline(await fetch(url, Api.timeline), after)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let let
ps = genParams({"screen_name": name, "trim_user": "true"}, ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false) count="18", ext=false)
url = photoRail ? ps url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.photoRail)) result = parsePhotoRail(await fetch(url, Api.timeline))
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
when T is Profile: when T is User:
const const
searchMode = ("result_filter", "user") searchMode = ("result_filter", "user")
parse = parseUsers parse = parseUsers

View file

@ -2,12 +2,11 @@
import uri, sequtils import uri, sequtils
const const
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
api = parseUri("https://api.twitter.com") api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json") activate* = $(api / "1.1/guest/activate.json")
listMembers* = api / "1.1/lists/members.json"
userShow* = api / "1.1/users/show.json" userShow* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json" photoRail* = api / "1.1/statuses/media_timeline.json"
search* = api / "2/search/adaptive.json" search* = api / "2/search/adaptive.json"
@ -19,8 +18,10 @@ const
tweet* = timelineApi / "conversation" tweet* = timelineApi / "conversation"
graphql = api / "graphql" graphql = api / "graphql"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug" graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
timelineParams* = { timelineParams* = {
"include_profile_interstitial_type": "0", "include_profile_interstitial_type": "0",

View file

@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils]
import jsony import jsony
import utils, slices import utils, slices
import ../types/user as userType import ../types/user as userType
from ../../types import Profile, Error from ../../types import User, Error
let let
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})" unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@ -11,13 +11,13 @@ let
htRegex = re"(^|[^\w-_./?])([#$])([\w_]+)" htRegex = re"(^|[^\w-_./?])([#$])([\w_]+)"
htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>" htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
proc expandProfileEntities(profile: var Profile; user: User) = proc expandUserEntities(user: var User; raw: RawUser) =
let let
orig = profile.bio.toRunes orig = user.bio.toRunes
ent = user.entities ent = raw.entities
if ent.url.urls.len > 0: if ent.url.urls.len > 0:
profile.website = ent.url.urls[0].expandedUrl user.website = ent.url.urls[0].expandedUrl
var replacements = newSeq[ReplaceSlice]() var replacements = newSeq[ReplaceSlice]()
@ -27,26 +27,26 @@ proc expandProfileEntities(profile: var Profile; user: User) =
replacements.dedupSlices replacements.dedupSlices
replacements.sort(cmp) replacements.sort(cmp)
profile.bio = orig.replacedWith(replacements, 0 .. orig.len) user.bio = orig.replacedWith(replacements, 0 .. orig.len)
.replacef(unRegex, unReplace) .replacef(unRegex, unReplace)
.replacef(htRegex, htReplace) .replacef(htRegex, htReplace)
proc getBanner(user: User): string = proc getBanner(user: RawUser): string =
if user.profileBannerUrl.len > 0: if user.profileBannerUrl.len > 0:
return user.profileBannerUrl & "/1500x500" return user.profileBannerUrl & "/1500x500"
if user.profileLinkColor.len > 0: if user.profileLinkColor.len > 0:
return '#' & user.profileLinkColor return '#' & user.profileLinkColor
proc parseUser*(json: string; username=""): Profile = proc parseUser*(json: string; username=""): User =
handleErrors: handleErrors:
case error.code case error.code
of suspended: return Profile(username: username, suspended: true) of suspended: return User(username: username, suspended: true)
of userNotFound: return of userNotFound: return
else: echo "[error - parseUser]: ", error else: echo "[error - parseUser]: ", error
let user = json.fromJson(User) let user = json.fromJson(RawUser)
result = Profile( result = User(
id: user.idStr, id: user.idStr,
username: user.screenName, username: user.screenName,
fullname: user.name, fullname: user.name,
@ -64,4 +64,4 @@ proc parseUser*(json: string; username=""): Profile =
userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "") userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
) )
result.expandProfileEntities(user) result.expandUserEntities(user)

View file

@ -1,7 +1,7 @@
import common import common
type type
User* = object RawUser* = object
idStr*: string idStr*: string
name*: string name*: string
screenName*: string screenName*: string

View file

@ -97,29 +97,29 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
proc getUserPic*(userPic: string; style=""): string = proc getUserPic*(userPic: string; style=""): string =
userPic.replacef(userPicRegex, "$2").replacef(extRegex, style & "$1") userPic.replacef(userPicRegex, "$2").replacef(extRegex, style & "$1")
proc getUserPic*(profile: Profile; style=""): string = proc getUserPic*(user: User; style=""): string =
getUserPic(profile.userPic, style) getUserPic(user.userPic, style)
proc getVideoEmbed*(cfg: Config; id: int64): string = proc getVideoEmbed*(cfg: Config; id: int64): string =
&"{getUrlPrefix(cfg)}/i/videos/{id}" &"{getUrlPrefix(cfg)}/i/videos/{id}"
proc pageTitle*(profile: Profile): string = proc pageTitle*(user: User): string =
&"{profile.fullname} (@{profile.username})" &"{user.fullname} (@{user.username})"
proc pageTitle*(tweet: Tweet): string = proc pageTitle*(tweet: Tweet): string =
&"{pageTitle(tweet.profile)}: \"{stripHtml(tweet.text)}\"" &"{pageTitle(tweet.user)}: \"{stripHtml(tweet.text)}\""
proc pageDesc*(profile: Profile): string = proc pageDesc*(user: User): string =
if profile.bio.len > 0: if user.bio.len > 0:
stripHtml(profile.bio) stripHtml(user.bio)
else: else:
"The latest tweets from " & profile.fullname "The latest tweets from " & user.fullname
proc getJoinDate*(profile: Profile): string = proc getJoinDate*(user: User): string =
profile.joinDate.format("'Joined' MMMM YYYY") user.joinDate.format("'Joined' MMMM YYYY")
proc getJoinDateFull*(profile: Profile): string = proc getJoinDateFull*(user: User): string =
profile.joinDate.format("h:mm tt - d MMM YYYY") user.joinDate.format("h:mm tt - d MMM YYYY")
proc getTime*(tweet: Tweet): string = proc getTime*(tweet: Tweet): string =
tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'") tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'")
@ -146,7 +146,7 @@ proc getShortTime*(tweet: Tweet): string =
proc getLink*(tweet: Tweet; focus=true): string = proc getLink*(tweet: Tweet; focus=true): string =
if tweet.id == 0: return if tweet.id == 0: return
var username = tweet.profile.username var username = tweet.user.username
if username.len == 0: if username.len == 0:
username = "i" username = "i"
result = &"/{username}/status/{tweet.id}" result = &"/{username}/status/{tweet.id}"
@ -175,7 +175,7 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string =
if username.len > 0: if username.len > 0:
result = result.replace("/" & username, "") result = result.replace("/" & username, "")
proc getLocation*(u: Profile | Tweet): (string, string) = proc getLocation*(u: User | Tweet): (string, string) =
if "://" in u.location: return (u.location, "") if "://" in u.location: return (u.location, "")
let loc = u.location.split(":") let loc = u.location.split(":")
let url = if loc.len > 1: "/search?q=place:" & loc[1] else: "" let url = if loc.len > 1: "/search?q=place:" & loc[1] else: ""

View file

@ -4,9 +4,9 @@ import packedjson, packedjson/deserialiser
import types, parserutils, utils import types, parserutils, utils
import experimental/parser/unifiedcard import experimental/parser/unifiedcard
proc parseProfile(js: JsonNode; id=""): Profile = proc parseUser(js: JsonNode; id=""): User =
if js.isNull: return if js.isNull: return
result = Profile( result = User(
id: if id.len > 0: id else: js{"id_str"}.getStr, id: if id.len > 0: id else: js{"id_str"}.getStr,
username: js{"screen_name"}.getStr, username: js{"screen_name"}.getStr,
fullname: js{"name"}.getStr, fullname: js{"name"}.getStr,
@ -24,7 +24,17 @@ proc parseProfile(js: JsonNode; id=""): Profile =
joinDate: js{"created_at"}.getTime joinDate: js{"created_at"}.getTime
) )
result.expandProfileEntities(js) result.expandUserEntities(js)
proc parseGraphUser*(js: JsonNode; id: string): User =
if js.isNull: return
with user, js{"data", "user", "result", "legacy"}:
result = parseUser(user, id)
with pinned, user{"pinned_tweet_ids_str"}:
if pinned.kind == JArray and pinned.len > 0:
result.pinnedTweet = parseBiggestInt(pinned[0].getStr)
proc parseGraphList*(js: JsonNode): List = proc parseGraphList*(js: JsonNode): List =
if js.isNull: return if js.isNull: return
@ -45,21 +55,30 @@ proc parseGraphList*(js: JsonNode): List =
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
) )
proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] = proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] =
result = Result[Profile]( result = Result[User](
beginning: cursor.len == 0, beginning: cursor.len == 0,
query: Query(kind: userList) query: Query(kind: userList)
) )
if js.isNull: return if js.isNull: return
result.top = js{"previous_cursor_str"}.getStr # result.top = js{"previous_cursor_str"}.getStr
result.bottom = js{"next_cursor_str"}.getStr # result.bottom = js{"next_cursor_str"}.getStr
if result.bottom.len == 1: # if result.bottom.len == 1:
result.bottom.setLen 0 # result.bottom.setLen 0
let root = js{"data", "list", "members_timeline", "timeline", "instructions"}
for instruction in root:
if instruction{"type"}.getStr == "TimelineAddEntries":
for entry in instruction{"entries"}:
let content = entry{"content"}
if content{"entryType"}.getStr == "TimelineTimelineItem":
with legacy, content{"itemContent", "user_results", "result", "legacy"}:
result.content.add parseUser(legacy)
elif content{"cursorType"}.getStr == "Bottom":
result.bottom = content{"value"}.getStr
for u in js{"users"}:
result.content.add parseProfile(u)
proc parsePoll(js: JsonNode): Poll = proc parsePoll(js: JsonNode): Poll =
let vals = js{"binding_values"} let vals = js{"binding_values"}
@ -206,7 +225,7 @@ proc parseTweet(js: JsonNode): Tweet =
time: js{"created_at"}.getTime, time: js{"created_at"}.getTime,
hasThread: js{"self_thread"}.notNull, hasThread: js{"self_thread"}.notNull,
available: true, available: true,
profile: Profile(id: js{"user_id_str"}.getStr), user: User(id: js{"user_id_str"}.getStr),
stats: TweetStats( stats: TweetStats(
replies: js{"reply_count"}.getInt, replies: js{"reply_count"}.getInt,
retweets: js{"retweet_count"}.getInt, retweets: js{"retweet_count"}.getInt,
@ -244,7 +263,7 @@ proc parseTweet(js: JsonNode): Tweet =
of "video": of "video":
result.video = some(parseVideo(m)) result.video = some(parseVideo(m))
with user, m{"additional_media_info", "source_user"}: with user, m{"additional_media_info", "source_user"}:
result.attribution = some(parseProfile(user)) result.attribution = some(parseUser(user))
of "animated_gif": of "animated_gif":
result.gif = some(parseGif(m)) result.gif = some(parseGif(m))
else: discard else: discard
@ -298,36 +317,32 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
users = ? js{"globalObjects", "users"} users = ? js{"globalObjects", "users"}
for k, v in users: for k, v in users:
result.users[k] = parseProfile(v, k) result.users[k] = parseUser(v, k)
for k, v in tweets: for k, v in tweets:
var tweet = parseTweet(v) var tweet = parseTweet(v)
if tweet.profile.id in result.users: if tweet.user.id in result.users:
tweet.profile = result.users[tweet.profile.id] tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet result.tweets[k] = tweet
proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] = proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
result.thread = Chain() result.thread = Chain()
for t in js{"content", "timelineModule", "items"}:
let content = t{"item", "content"} let thread = js{"content", "item", "content", "conversationThread"}
if "Self" in content{"tweet", "displayType"}.getStr: with cursor, thread{"showMoreCursor"}:
result.thread.cursor = cursor{"value"}.getStr
result.thread.hasMore = true
for t in thread{"conversationComponents"}:
let content = t{"conversationTweetComponent", "tweet"}
if content{"displayType"}.getStr == "SelfThread":
result.self = true result.self = true
let entry = t{"entryId"}.getStr var tweet = finalizeTweet(global, content{"id"}.getStr)
if "show_more" in entry: if not tweet.available:
let tweet.tombstone = getTombstone(content{"tombstone"})
cursor = content{"timelineCursor"} result.thread.content.add tweet
more = cursor{"displayTreatment", "actionText"}.getStr
result.thread.cursor = cursor{"value"}.getStr
if more.len > 0 and more[0].isDigit():
result.thread.more = parseInt(more[0 ..< more.find(" ")])
else:
result.thread.more = -1
else:
var tweet = finalizeTweet(global, t.getEntryId)
if not tweet.available:
tweet.tombstone = getTombstone(content{"tombstone"})
result.thread.content.add tweet
proc parseConversation*(js: JsonNode; tweetId: string): Conversation = proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true)) result = Conversation(replies: Result[Chain](beginning: true))
@ -373,8 +388,8 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod
elif "bottom" in r{"entryId"}.getStr: elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor res.bottom = r.getCursor
proc parseUsers*(js: JsonNode; after=""): Result[Profile] = proc parseUsers*(js: JsonNode; after=""): Result[User] =
result = Result[Profile](beginning: after.len == 0) result = Result[User](beginning: after.len == 0)
let global = parseGlobalObjects(? js) let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"} let instructions = ? js{"timeline", "instructions"}
@ -404,7 +419,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
for e in instructions[0]{"addEntries", "entries"}: for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr let entry = e{"entryId"}.getStr
if "tweet" in entry or "sq-I-t" in entry or "tombstone" in entry: if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId) let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue if not tweet.available: continue
result.content.add tweet result.content.add tweet
@ -412,6 +427,12 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
result.top = e.getCursor result.top = e.getCursor
elif "cursor-bottom" in entry: elif "cursor-bottom" in entry:
result.bottom = e.getCursor result.bottom = e.getCursor
elif entry.startsWith("sq-C"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.bottom = cursor{"value"}.getStr
else:
result.top = cursor{"value"}.getStr
proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js: for tweet in js:

View file

@ -119,6 +119,16 @@ proc getBanner*(js: JsonNode): string =
if color.len > 0: if color.len > 0:
return '#' & color return '#' & color
# use primary color from profile picture color histogram
with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
if p.len > 0:
let pal = p[0]{"rgb"}
result = "#"
result.add toHex(pal{"red"}.getInt, 2)
result.add toHex(pal{"green"}.getInt, 2)
result.add toHex(pal{"blue"}.getInt, 2)
return
proc getTombstone*(js: JsonNode): string = proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more") result.removeSuffix(" Learn more")
@ -184,13 +194,13 @@ proc deduplicate(s: var seq[ReplaceSlice]) =
proc cmp(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b) proc cmp(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
proc expandProfileEntities*(profile: var Profile; js: JsonNode) = proc expandUserEntities*(user: var User; js: JsonNode) =
let let
orig = profile.bio.toRunes orig = user.bio.toRunes
ent = ? js{"entities"} ent = ? js{"entities"}
with urls, ent{"url", "urls"}: with urls, ent{"url", "urls"}:
profile.website = urls[0]{"expanded_url"}.getStr user.website = urls[0]{"expanded_url"}.getStr
var replacements = newSeq[ReplaceSlice]() var replacements = newSeq[ReplaceSlice]()
@ -201,9 +211,9 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
replacements.deduplicate replacements.deduplicate
replacements.sort(cmp) replacements.sort(cmp)
profile.bio = orig.replacedWith(replacements, 0 .. orig.len) user.bio = orig.replacedWith(replacements, 0 .. orig.len)
profile.bio = profile.bio.replacef(unRegex, unReplace) user.bio = user.bio.replacef(unRegex, unReplace)
.replacef(htRegex, htReplace) .replacef(htRegex, htReplace)
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let let

View file

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, times, strutils, tables, hashes import asyncdispatch, times, strformat, strutils, tables, hashes
import redis, redpool, flatty, supersnappy import redis, redpool, flatty, supersnappy
import types, api import types, api
@ -51,9 +51,10 @@ proc initRedisPool*(cfg: Config) {.async.} =
await migrate("userBuckets", "p:*") await migrate("userBuckets", "p:*")
await migrate("profileDates", "p:*") await migrate("profileDates", "p:*")
await migrate("profileStats", "p:*") await migrate("profileStats", "p:*")
await migrate("userType", "p:*")
pool.withAcquire(r): pool.withAcquire(r):
# optimize memory usage for profile ID buckets # optimize memory usage for user ID buckets
await r.configSet("hash-max-ziplist-entries", "1000") await r.configSet("hash-max-ziplist-entries", "1000")
except OSError: except OSError:
@ -61,9 +62,10 @@ proc initRedisPool*(cfg: Config) {.async.} =
stdout.flushFile stdout.flushFile
quit(1) quit(1)
template pidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000) template uidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
template profileKey(name: string): string = "p:" & name template userKey(name: string): string = "p:" & name
template listKey(l: List): string = "l:" & l.id template listKey(l: List): string = "l:" & l.id
template tweetKey(id: int64): string = "t:" & $id
proc get(query: string): Future[string] {.async.} = proc get(query: string): Future[string] {.async.} =
pool.withAcquire(r): pool.withAcquire(r):
@ -73,25 +75,29 @@ proc setEx(key: string; time: int; data: string) {.async.} =
pool.withAcquire(r): pool.withAcquire(r):
dawait r.setEx(key, time, data) dawait r.setEx(key, time, data)
proc cacheUserId(username, id: string) {.async.} =
if username.len == 0 or id.len == 0: return
let name = toLower(username)
pool.withAcquire(r):
dawait r.hSet(name.uidKey, name, id)
proc cache*(data: List) {.async.} = proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data))) await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
proc cache*(data: PhotoRail; name: string) {.async.} = proc cache*(data: PhotoRail; name: string) {.async.} =
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data))) await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
proc cache*(data: Profile) {.async.} = proc cache*(data: User) {.async.} =
if data.username.len == 0: return if data.username.len == 0: return
let name = toLower(data.username) let name = toLower(data.username)
await cacheUserId(name, data.id)
pool.withAcquire(r): pool.withAcquire(r):
dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data))) dawait r.setEx(name.userKey, baseCacheTime, compress(toFlatty(data)))
if data.id.len > 0:
dawait r.hSet(name.pidKey, name, data.id)
proc cacheProfileId(username, id: string) {.async.} = proc cache*(data: Tweet) {.async.} =
if username.len == 0 or id.len == 0: return if data.isNil or data.id == 0: return
let name = toLower(username)
pool.withAcquire(r): pool.withAcquire(r):
dawait r.hSet(name.pidKey, name, id) dawait r.setEx(data.id.tweetKey, baseCacheTime, compress(toFlatty(data)))
proc cacheRss*(query: string; rss: Rss) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} =
let key = "rss:" & query let key = "rss:" & query
@ -100,24 +106,34 @@ proc cacheRss*(query: string; rss: Rss) {.async.} =
dawait r.hSet(key, "min", rss.cursor) dawait r.hSet(key, "min", rss.cursor)
dawait r.expire(key, rssCacheTime) dawait r.expire(key, rssCacheTime)
proc getProfileId*(username: string): Future[string] {.async.} = template deserialize(data, T) =
try:
result = fromFlatty(uncompress(data), T)
except:
echo "Decompression failed($#): '$#'" % [astToStr(T), data]
proc getUserId*(username: string): Future[string] {.async.} =
let name = toLower(username) let name = toLower(username)
pool.withAcquire(r): pool.withAcquire(r):
result = await r.hGet(name.pidKey, name) result = await r.hGet(name.uidKey, name)
if result == redisNil: if result == redisNil:
result.setLen(0) let user = await getUser(username)
if user.suspended:
return "suspended"
else:
await cacheUserId(name, user.id)
return user.id
proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.} = proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
let prof = await get("p:" & toLower(username)) let prof = await get("p:" & toLower(username))
if prof != redisNil: if prof != redisNil:
result = fromFlatty(uncompress(prof), Profile) prof.deserialize(User)
elif fetch: elif fetch:
result = await getProfile(username) let userId = await getUserId(username)
await cacheProfileId(result.username, result.id) result = await getGraphUser(userId)
if result.suspended: await cache(result)
await cache(result)
proc getCachedProfileUsername*(userId: string): Future[string] {.async.} = proc getCachedUsername*(userId: string): Future[string] {.async.} =
let let
key = "i:" & userId key = "i:" & userId
username = await get(key) username = await get(key)
@ -125,15 +141,26 @@ proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
if username != redisNil: if username != redisNil:
result = username result = username
else: else:
let profile = await getProfileById(userId) let user = await getUserById(userId)
result = profile.username result = user.username
await setEx(key, baseCacheTime, result) await setEx(key, baseCacheTime, result)
proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
if id == 0: return
let tweet = await get(id.tweetKey)
if tweet != redisNil:
tweet.deserialize(Tweet)
else:
let conv = await getTweet($id)
if not conv.isNil:
result = conv.tweet
await cache(result)
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} = proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return if name.len == 0: return
let rail = await get("pr:" & toLower(name)) let rail = await get("pr:" & toLower(name))
if rail != redisNil: if rail != redisNil:
result = fromFlatty(uncompress(rail), PhotoRail) rail.deserialize(PhotoRail)
else: else:
result = await getPhotoRail(name) result = await getPhotoRail(name)
await cache(result, name) await cache(result, name)
@ -143,7 +170,7 @@ proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
else: await get("l:" & id) else: await get("l:" & id)
if list != redisNil: if list != redisNil:
result = fromFlatty(uncompress(list), List) list.deserialize(List)
else: else:
if id.len > 0: if id.len > 0:
result = await getGraphList(id) result = await getGraphList(id)

View file

@ -47,5 +47,5 @@ proc createListRouter*(cfg: Config) =
prefs = cookiePrefs() prefs = cookiePrefs()
list = await getCachedList(id=(@"id")) list = await getCachedList(id=(@"id"))
title = "@" & list.username & "/" & list.name title = "@" & list.username & "/" & list.name
members = await getListMembers(list, getCursor()) members = await getGraphListMembers(list, getCursor())
respList(list, members, title, renderTimelineUsers(members, prefs, request.path)) respList(list, members, title, renderTimelineUsers(members, prefs, request.path))

View file

@ -12,32 +12,32 @@ export times, hashes, supersnappy
proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} = proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
var profile: Profile var profile: Profile
var timeline: Timeline
let let
name = req.params.getOrDefault("name") name = req.params.getOrDefault("name")
after = getCursor(req) after = getCursor(req)
names = getNames(name) names = getNames(name)
if names.len == 1: if names.len == 1:
(profile, timeline) = await fetchTimeline(after, query, skipRail=true) profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
else: else:
var q = query var q = query
q.fromUser = names q.fromUser = names
timeline = await getSearch[Tweet](q, after)
# this is kinda dumb
profile = Profile( profile = Profile(
username: name, tweets: await getSearch[Tweet](q, after),
fullname: names.join(" | "), # this is kinda dumb
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png" user: User(
username: name,
fullname: names.join(" | "),
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
)
) )
if profile.suspended: if profile.user.suspended:
return Rss(feed: profile.username, cursor: "suspended") return Rss(feed: profile.user.username, cursor: "suspended")
if profile.fullname.len > 0: if profile.user.fullname.len > 0:
let rss = compress renderTimelineRss(timeline, profile, cfg, let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1))
multi=(names.len > 1)) return Rss(feed: rss, cursor: profile.tweets.bottom)
return Rss(feed: rss, cursor: timeline.bottom)
template respRss*(rss, page) = template respRss*(rss, page) =
if rss.cursor.len == 0: if rss.cursor.len == 0:

View file

@ -25,7 +25,7 @@ proc createSearchRouter*(cfg: Config) =
of users: of users:
if "," in @"q": if "," in @"q":
redirect("/" & @"q") redirect("/" & @"q")
let users = await getSearch[Profile](query, getCursor()) let users = await getSearch[User](query, getCursor())
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs) resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
of tweets: of tweets:
let let

View file

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils, sequtils, uri, options import asyncdispatch, strutils, sequtils, uri, options, sugar
import jester, karax/vdom import jester, karax/vdom
@ -7,7 +7,7 @@ import router_utils
import ".."/[types, formatters, api] import ".."/[types, formatters, api]
import ../views/[general, status] import ../views/[general, status]
export uri, sequtils, options export uri, sequtils, options, sugar
export router_utils export router_utils
export api, formatters export api, formatters
export status export status
@ -16,6 +16,7 @@ proc createStatusRouter*(cfg: Config) =
router status: router status:
get "/@name/status/@id/?": get "/@name/status/@id/?":
cond '.' notin @"name" cond '.' notin @"name"
cond not @"id".any(c => not c.isDigit)
let prefs = cookiePrefs() let prefs = cookiePrefs()
# used for the infinite scroll feature # used for the infinite scroll feature
@ -37,7 +38,7 @@ proc createStatusRouter*(cfg: Config) =
let let
title = pageTitle(conv.tweet) title = pageTitle(conv.tweet)
ogTitle = pageTitle(conv.tweet.profile) ogTitle = pageTitle(conv.tweet.user)
desc = conv.tweet.text desc = conv.tweet.text
var var

View file

@ -19,62 +19,57 @@ proc getQuery*(request: Request; tab, name: string): Query =
of "search": initQuery(params(request), name=name) of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name]) else: Query(fromUser: @[name])
proc fetchTimeline*(after: string; query: Query; skipRail=false): proc fetchProfile*(after: string; query: Query; skipRail=false;
Future[(Profile, Timeline, PhotoRail)] {.async.} = skipPinned=false): Future[Profile] {.async.} =
let name = query.fromUser[0] let name = query.fromUser[0]
var let userId = await getUserId(name)
profile: Profile if userId.len == 0:
profileId = await getProfileId(name) return Profile(user: User(username: name))
fetched = false elif userId == "suspended":
return Profile(user: User(username: name, suspended: true))
if profileId.len == 0:
profile = await getCachedProfile(name)
profileId = profile.id
fetched = true
if profile.protected or profile.suspended:
return (profile, Timeline(), @[])
elif profileId.len == 0:
return (Profile(username: name), Timeline(), @[])
var rail: Future[PhotoRail] var rail: Future[PhotoRail]
if skipRail or profile.protected or query.kind == media: if skipRail or result.user.protected or query.kind == media:
rail = newFuture[PhotoRail]() rail = newFuture[PhotoRail]()
rail.complete(@[]) rail.complete(@[])
else: else:
rail = getCachedPhotoRail(name) rail = getCachedPhotoRail(name)
# var timeline = # temporary fix to prevent errors from people browsing
# case query.kind # timelines during/immediately after deployment
# of posts: await getTimeline(profileId, after) var after = after
# of replies: await getTimeline(profileId, after, replies=true) if query.kind in {posts, replies} and after.startsWith("scroll"):
# of media: await getMediaTimeline(profileId, after) after.setLen 0
# else: await getSearch[Tweet](query, after)
var timeline = let timeline =
case query.kind case query.kind
of media: await getMediaTimeline(profileId, after) of posts: getTimeline(userId, after)
else: await getSearch[Tweet](query, after) of replies: getTimeline(userId, after, replies=true)
of media: getMediaTimeline(userId, after)
else: getSearch[Tweet](query, after)
timeline.query = query let user = await getCachedUser(name)
var found = false var pinned: Option[Tweet]
for tweet in timeline.content.mitems: if not skipPinned and user.pinnedTweet > 0 and
if tweet.profile.id == profileId or after.len == 0 and query.kind in {posts, replies}:
tweet.profile.username.cmpIgnoreCase(name) == 0: let tweet = await getCachedTweet(user.pinnedTweet)
profile = tweet.profile if not tweet.isNil:
found = true tweet.pinned = true
break pinned = some tweet
if profile.username.len == 0: result = Profile(
profile = await getCachedProfile(name) user: user,
fetched = true pinned: pinned,
tweets: await timeline,
photoRail: await rail
)
if fetched and not found: if result.user.protected or result.user.suspended:
await cache(profile) return
return (profile, timeline, await rail) result.tweets.query = query
proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} = rss, after: string): Future[string] {.async.} =
@ -84,15 +79,18 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
html = renderTweetSearch(timeline, prefs, getPath()) html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss) return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
var (p, t, r) = await fetchTimeline(after, query) var profile = await fetchProfile(after, query)
template u: untyped = profile.user
if p.suspended: return showError(getSuspended(p.username), cfg) if u.suspended:
if p.id.len == 0: return return showError(getSuspended(u.username), cfg)
let pHtml = renderProfile(p, t, r, prefs, getPath()) if profile.user.id.len == 0: return
result = renderMain(pHtml, request, cfg, prefs, pageTitle(p), pageDesc(p),
rss=rss, images = @[p.getUserPic("_400x400")], let pHtml = renderProfile(profile, prefs, getPath())
banner=p.banner) result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u),
rss=rss, images = @[u.getUserPic("_400x400")],
banner=u.banner)
template respTimeline*(timeline: typed) = template respTimeline*(timeline: typed) =
let t = timeline let t = timeline
@ -102,7 +100,7 @@ template respTimeline*(timeline: typed) =
template respUserId*() = template respUserId*() =
cond @"user_id".len > 0 cond @"user_id".len > 0
let username = await getCachedProfileUsername(@"user_id") let username = await getCachedUsername(@"user_id")
if username.len > 0: if username.len > 0:
redirect("/" & username) redirect("/" & username)
else: else:
@ -137,10 +135,10 @@ proc createTimelineRouter*(cfg: Config) =
timeline.beginning = true timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath()) resp $renderTweetSearch(timeline, prefs, getPath())
else: else:
var (_, timeline, _) = await fetchTimeline(after, query, skipRail=true) var profile = await fetchProfile(after, query, skipRail=true)
if timeline.content.len == 0: resp Http404 if profile.tweets.content.len == 0: resp Http404
timeline.beginning = true profile.tweets.beginning = true
resp $renderTimelineTweets(timeline, prefs, getPath()) resp $renderTimelineTweets(profile.tweets, prefs, getPath())
let rss = let rss =
if @"tab".len == 0: if @"tab".len == 0:

View file

@ -37,7 +37,7 @@ proc getPoolJson*(): JsonNode =
let let
maxReqs = maxReqs =
case api case api
of Api.listBySlug, Api.list: 500 of Api.listBySlug, Api.list, Api.userRestId: 500
of Api.timeline: 187 of Api.timeline: 187
else: 180 else: 180
reqs = maxReqs - token.apis[api].remaining reqs = maxReqs - token.apis[api].remaining

View file

@ -10,13 +10,13 @@ type
Api* {.pure.} = enum Api* {.pure.} = enum
userShow userShow
photoRail
timeline timeline
search search
tweet tweet
list list
listBySlug listBySlug
listMembers listMembers
userRestId
RateLimit* = object RateLimit* = object
remaining*: int remaining*: int
@ -44,7 +44,7 @@ type
badToken = 239 badToken = 239
noCsrf = 353 noCsrf = 353
Profile* = object User* = object
id*: string id*: string
username*: string username*: string
fullname*: string fullname*: string
@ -53,6 +53,7 @@ type
bio*: string bio*: string
userPic*: string userPic*: string
banner*: string banner*: string
pinnedTweet*: int64
following*: int following*: int
followers*: int followers*: int
tweets*: int tweets*: int
@ -162,7 +163,7 @@ type
id*: int64 id*: int64
threadId*: int64 threadId*: int64
replyId*: int64 replyId*: int64
profile*: Profile user*: User
text*: string text*: string
time*: DateTime time*: DateTime
reply*: seq[string] reply*: seq[string]
@ -173,8 +174,8 @@ type
location*: string location*: string
stats*: TweetStats stats*: TweetStats
retweet*: Option[Tweet] retweet*: Option[Tweet]
attribution*: Option[Profile] attribution*: Option[User]
mediaTags*: seq[Profile] mediaTags*: seq[User]
quote*: Option[Tweet] quote*: Option[Tweet]
card*: Option[Card] card*: Option[Card]
poll*: Option[Poll] poll*: Option[Poll]
@ -190,7 +191,7 @@ type
Chain* = object Chain* = object
content*: seq[Tweet] content*: seq[Tweet]
more*: int64 hasMore*: bool
cursor*: string cursor*: string
Conversation* = ref object Conversation* = ref object
@ -201,6 +202,12 @@ type
Timeline* = Result[Tweet] Timeline* = Result[Tweet]
Profile* = object
user*: User
photoRail*: PhotoRail
pinned*: Option[Tweet]
tweets*: Timeline
List* = object List* = object
id*: string id*: string
name*: string name*: string
@ -212,7 +219,7 @@ type
GlobalObjects* = ref object GlobalObjects* = ref object
tweets*: Table[string, Tweet] tweets*: Table[string, Tweet]
users*: Table[string, Profile] users*: Table[string, User]
Config* = ref object Config* = ref object
address*: string address*: string

View file

@ -12,32 +12,32 @@ proc renderStat(num: int; class: string; text=""): VNode =
span(class="profile-stat-num"): span(class="profile-stat-num"):
text insertSep($num, ',') text insertSep($num, ',')
proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode = proc renderUserCard*(user: User; prefs: Prefs): VNode =
buildHtml(tdiv(class="profile-card")): buildHtml(tdiv(class="profile-card")):
tdiv(class="profile-card-info"): tdiv(class="profile-card-info"):
let let
url = getPicUrl(profile.getUserPic()) url = getPicUrl(user.getUserPic())
size = size =
if prefs.autoplayGifs and profile.userPic.endsWith("gif"): "" if prefs.autoplayGifs and user.userPic.endsWith("gif"): ""
else: "_400x400" else: "_400x400"
a(class="profile-card-avatar", href=url, target="_blank"): a(class="profile-card-avatar", href=url, target="_blank"):
genImg(profile.getUserPic(size)) genImg(user.getUserPic(size))
tdiv(class="profile-card-tabs-name"): tdiv(class="profile-card-tabs-name"):
linkUser(profile, class="profile-card-fullname") linkUser(user, class="profile-card-fullname")
linkUser(profile, class="profile-card-username") linkUser(user, class="profile-card-username")
tdiv(class="profile-card-extra"): tdiv(class="profile-card-extra"):
if profile.bio.len > 0: if user.bio.len > 0:
tdiv(class="profile-bio"): tdiv(class="profile-bio"):
p(dir="auto"): p(dir="auto"):
verbatim replaceUrls(profile.bio, prefs) verbatim replaceUrls(user.bio, prefs)
if profile.location.len > 0: if user.location.len > 0:
tdiv(class="profile-location"): tdiv(class="profile-location"):
span: icon "location" span: icon "location"
let (place, url) = getLocation(profile) let (place, url) = getLocation(user)
if url.len > 1: if url.len > 1:
a(href=url): text place a(href=url): text place
elif "://" in place: elif "://" in place:
@ -45,29 +45,29 @@ proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
else: else:
span: text place span: text place
if profile.website.len > 0: if user.website.len > 0:
tdiv(class="profile-website"): tdiv(class="profile-website"):
span: span:
let url = replaceUrls(profile.website, prefs) let url = replaceUrls(user.website, prefs)
icon "link" icon "link"
a(href=url): text shortLink(url) a(href=url): text shortLink(url)
tdiv(class="profile-joindate"): tdiv(class="profile-joindate"):
span(title=getJoinDateFull(profile)): span(title=getJoinDateFull(user)):
icon "calendar", getJoinDate(profile) icon "calendar", getJoinDate(user)
tdiv(class="profile-card-extra-links"): tdiv(class="profile-card-extra-links"):
ul(class="profile-statlist"): ul(class="profile-statlist"):
renderStat(profile.tweets, "posts", text="Tweets") renderStat(user.tweets, "posts", text="Tweets")
renderStat(profile.following, "following") renderStat(user.following, "following")
renderStat(profile.followers, "followers") renderStat(user.followers, "followers")
renderStat(profile.likes, "likes") renderStat(user.likes, "likes")
proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode = proc renderPhotoRail(profile: Profile): VNode =
let count = insertSep($profile.media, ',') let count = insertSep($profile.user.media, ',')
buildHtml(tdiv(class="photo-rail-card")): buildHtml(tdiv(class="photo-rail-card")):
tdiv(class="photo-rail-header"): tdiv(class="photo-rail-header"):
a(href=(&"/{profile.username}/media")): a(href=(&"/{profile.user.username}/media")):
icon "picture", count & " Photos and videos" icon "picture", count & " Photos and videos"
input(id="photo-rail-grid-toggle", `type`="checkbox") input(id="photo-rail-grid-toggle", `type`="checkbox")
@ -76,18 +76,19 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
icon "down" icon "down"
tdiv(class="photo-rail-grid"): tdiv(class="photo-rail-grid"):
for i, photo in photoRail: for i, photo in profile.photoRail:
if i == 16: break if i == 16: break
a(href=(&"/{profile.username}/status/{photo.tweetId}#m")): a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
genImg(photo.url & (if "format" in photo.url: "" else: ":thumb")) genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
proc renderBanner(banner: string): VNode = proc renderBanner(banner: string): VNode =
buildHtml(): buildHtml():
if banner.startsWith('#'): if banner.len == 0:
a()
elif banner.startsWith('#'):
a(style={backgroundColor: banner}) a(style={backgroundColor: banner})
else: else:
a(href=getPicUrl(banner), target="_blank"): a(href=getPicUrl(banner), target="_blank"): genImg(banner)
genImg(banner)
proc renderProtected(username: string): VNode = proc renderProtected(username: string): VNode =
buildHtml(tdiv(class="timeline-container")): buildHtml(tdiv(class="timeline-container")):
@ -95,22 +96,21 @@ proc renderProtected(username: string): VNode =
h2: text "This account's tweets are protected." h2: text "This account's tweets are protected."
p: text &"Only confirmed followers have access to @{username}'s tweets." p: text &"Only confirmed followers have access to @{username}'s tweets."
proc renderProfile*(profile: Profile; timeline: var Timeline; proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
photoRail: PhotoRail; prefs: Prefs; path: string): VNode = profile.tweets.query.fromUser = @[profile.user.username]
timeline.query.fromUser = @[profile.username]
buildHtml(tdiv(class="profile-tabs")): buildHtml(tdiv(class="profile-tabs")):
if not prefs.hideBanner: if not prefs.hideBanner:
tdiv(class="profile-banner"): tdiv(class="profile-banner"):
if profile.banner.len > 0: renderBanner(profile.user.banner)
renderBanner(profile.banner)
let sticky = if prefs.stickyProfile: " sticky" else: "" let sticky = if prefs.stickyProfile: " sticky" else: ""
tdiv(class=(&"profile-tab{sticky}")): tdiv(class=(&"profile-tab{sticky}")):
renderProfileCard(profile, prefs) renderUserCard(profile.user, prefs)
if photoRail.len > 0: if profile.photoRail.len > 0:
renderPhotoRail(profile, photoRail) renderPhotoRail(profile)
if profile.protected: if profile.user.protected:
renderProtected(profile.username) renderProtected(profile.user.username)
else: else:
renderTweetSearch(timeline, prefs, path) renderTweetSearch(profile.tweets, prefs, path, profile.pinned)

View file

@ -15,18 +15,18 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
if text.len > 0: if text.len > 0:
text " " & text text " " & text
proc linkUser*(profile: Profile, class=""): VNode = proc linkUser*(user: User, class=""): VNode =
let let
isName = "username" notin class isName = "username" notin class
href = "/" & profile.username href = "/" & user.username
nameText = if isName: profile.fullname nameText = if isName: user.fullname
else: "@" & profile.username else: "@" & user.username
buildHtml(a(href=href, class=class, title=nameText)): buildHtml(a(href=href, class=class, title=nameText)):
text nameText text nameText
if isName and profile.verified: if isName and user.verified:
icon "ok", class="verified-icon", title="Verified account" icon "ok", class="verified-icon", title="Verified account"
if isName and profile.protected: if isName and user.protected:
text " " text " "
icon "lock", title="Protected account" icon "lock", title="Protected account"

View file

@ -60,7 +60,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
#let urlPrefix = getUrlPrefix(cfg) #let urlPrefix = getUrlPrefix(cfg)
#var links: seq[string] #var links: seq[string]
#for t in tweets: #for t in tweets:
# let retweet = if t.retweet.isSome: t.profile.username else: "" # let retweet = if t.retweet.isSome: t.user.username else: ""
# let tweet = if retweet.len > 0: t.retweet.get else: t # let tweet = if retweet.len > 0: t.retweet.get else: t
# let link = getLink(tweet) # let link = getLink(tweet)
# if link in links: continue # if link in links: continue
@ -68,7 +68,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
# links.add link # links.add link
<item> <item>
<title>${getTitle(tweet, retweet)}</title> <title>${getTitle(tweet, retweet)}</title>
<dc:creator>@${tweet.profile.username}</dc:creator> <dc:creator>@${tweet.user.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description> <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate> <pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>${urlPrefix & link}</guid> <guid>${urlPrefix & link}</guid>
@ -77,32 +77,32 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
#end for #end for
#end proc #end proc
# #
#proc renderTimelineRss*(timeline: Timeline; profile: Profile; cfg: Config; multi=false): string = #proc renderTimelineRss*(profile: Profile; cfg: Config; multi=false): string =
#let urlPrefix = getUrlPrefix(cfg) #let urlPrefix = getUrlPrefix(cfg)
#result = "" #result = ""
#let user = (if multi: "" else: "@") & profile.username #let handle = (if multi: "" else: "@") & profile.user.username
#var title = profile.fullname #var title = profile.user.fullname
#if not multi: title &= " / " & user #if not multi: title &= " / " & handle
#end if #end if
#title = xmltree.escape(title).sanitizeXml #title = xmltree.escape(title).sanitizeXml
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0"> <rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
<channel> <channel>
<atom:link href="${urlPrefix}/${profile.username}/rss" rel="self" type="application/rss+xml" /> <atom:link href="${urlPrefix}/${profile.user.username}/rss" rel="self" type="application/rss+xml" />
<title>${title}</title> <title>${title}</title>
<link>${urlPrefix}/${profile.username}</link> <link>${urlPrefix}/${profile.user.username}</link>
<description>${getDescription(user, cfg)}</description> <description>${getDescription(handle, cfg)}</description>
<language>en-us</language> <language>en-us</language>
<ttl>40</ttl> <ttl>40</ttl>
<image> <image>
<title>${title}</title> <title>${title}</title>
<link>${urlPrefix}/${profile.username}</link> <link>${urlPrefix}/${profile.user.username}</link>
<url>${urlPrefix}${getPicUrl(profile.getUserPic(style="_400x400"))}</url> <url>${urlPrefix}${getPicUrl(profile.user.getUserPic(style="_400x400"))}</url>
<width>128</width> <width>128</width>
<height>128</height> <height>128</height>
</image> </image>
#if timeline.content.len > 0: #if profile.tweets.content.len > 0:
${renderRssTweets(timeline.content, cfg)} ${renderRssTweets(profile.tweets.content, cfg)}
#end if #end if
</channel> </channel>
</rss> </rss>

View file

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, sequtils, unicode, tables import strutils, strformat, sequtils, unicode, tables, options
import karax/[karaxdsl, vdom] import karax/[karaxdsl, vdom]
import renderutils, timeline import renderutils, timeline
@ -88,7 +88,8 @@ proc renderSearchPanel*(query: Query): VNode =
span(class="search-title"): text "Near" span(class="search-title"): text "Near"
genInput("near", "", query.near, placeholder="Location...") genInput("near", "", query.near, placeholder="Location...")
proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string): VNode = proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
let query = results.query let query = results.query
buildHtml(tdiv(class="timeline-container")): buildHtml(tdiv(class="timeline-container")):
if query.fromUser.len > 1: if query.fromUser.len > 1:
@ -105,9 +106,9 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string): VNo
if query.fromUser.len == 0: if query.fromUser.len == 0:
renderSearchTabs(query) renderSearchTabs(query)
renderTimelineTweets(results, prefs, path) renderTimelineTweets(results, prefs, path, pinned)
proc renderUserSearch*(results: Result[Profile]; prefs: Prefs): VNode = proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")): buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"): tdiv(class="timeline-header"):
form(`method`="get", action="/search", class="search-field"): form(`method`="get", action="/search", class="search-field"):

View file

@ -10,24 +10,22 @@ proc renderEarlier(thread: Chain): VNode =
text "earlier replies" text "earlier replies"
proc renderMoreReplies(thread: Chain): VNode = proc renderMoreReplies(thread: Chain): VNode =
let num = if thread.more != -1: $thread.more & " " else: ""
let reply = if thread.more == 1: "reply" else: "replies"
let link = getLink(thread.content[^1]) let link = getLink(thread.content[^1])
buildHtml(tdiv(class="timeline-item more-replies")): buildHtml(tdiv(class="timeline-item more-replies")):
if thread.content[^1].available: if thread.content[^1].available:
a(class="more-replies-text", href=link): a(class="more-replies-text", href=link):
text $num & "more " & reply text "more replies"
else: else:
a(class="more-replies-text"): a(class="more-replies-text"):
text $num & "more " & reply text "more replies"
proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode = proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
buildHtml(tdiv(class="reply thread thread-line")): buildHtml(tdiv(class="reply thread thread-line")):
for i, tweet in thread.content: for i, tweet in thread.content:
let last = (i == thread.content.high and thread.more == 0) let last = (i == thread.content.high and not thread.hasMore)
renderTweet(tweet, prefs, path, index=i, last=last) renderTweet(tweet, prefs, path, index=i, last=last)
if thread.more != 0: if thread.hasMore:
renderMoreReplies(thread) renderMoreReplies(thread)
proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode = proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
@ -60,12 +58,12 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
tdiv(class="after-tweet thread-line"): tdiv(class="after-tweet thread-line"):
let let
total = conv.after.content.high total = conv.after.content.high
more = conv.after.more hasMore = conv.after.hasMore
for i, tweet in conv.after.content: for i, tweet in conv.after.content:
renderTweet(tweet, prefs, path, index=i, renderTweet(tweet, prefs, path, index=i,
last=(i == total and more == 0), afterTweet=true) last=(i == total and not hasMore), afterTweet=true)
if more != 0: if hasMore:
renderMoreReplies(conv.after) renderMoreReplies(conv.after)
if not prefs.hideReplies: if not prefs.hideReplies:

View file

@ -57,7 +57,7 @@ proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet
elif t.replyId == result[0].id: elif t.replyId == result[0].id:
result.add t result.add t
proc renderUser(user: Profile; prefs: Prefs): VNode = proc renderUser(user: User; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-item")): buildHtml(tdiv(class="timeline-item")):
a(class="tweet-link", href=("/" & user.username)) a(class="tweet-link", href=("/" & user.username))
tdiv(class="tweet-body profile-result"): tdiv(class="tweet-body profile-result"):
@ -73,7 +73,7 @@ proc renderUser(user: Profile; prefs: Prefs): VNode =
tdiv(class="tweet-content media-body", dir="auto"): tdiv(class="tweet-content media-body", dir="auto"):
verbatim replaceUrls(user.bio, prefs) verbatim replaceUrls(user.bio, prefs)
proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNode = proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
buildHtml(tdiv(class="timeline")): buildHtml(tdiv(class="timeline")):
if not results.beginning: if not results.beginning:
renderNewer(results.query, path) renderNewer(results.query, path)
@ -89,11 +89,16 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
else: else:
renderNoMore() renderNoMore()
proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode = proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
buildHtml(tdiv(class="timeline")): buildHtml(tdiv(class="timeline")):
if not results.beginning: if not results.beginning:
renderNewer(results.query, parseUri(path).path) renderNewer(results.query, parseUri(path).path)
if pinned.isSome:
let tweet = get pinned
renderTweet(tweet, prefs, path, showThread=tweet.hasThread)
if results.content.len == 0: if results.content.len == 0:
if not results.beginning: if not results.beginning:
renderNoMore() renderNoMore()

View file

@ -13,8 +13,8 @@ proc getSmallPic(url: string): string =
result &= ":small" result &= ":small"
result = getPicUrl(result) result = getPicUrl(result)
proc renderMiniAvatar(profile: Profile; prefs: Prefs): VNode = proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
let url = getPicUrl(profile.getUserPic("_mini")) let url = getPicUrl(user.getUserPic("_mini"))
buildHtml(): buildHtml():
img(class=(prefs.getAvatarClass & " mini"), src=url) img(class=(prefs.getAvatarClass & " mini"), src=url)
@ -29,16 +29,16 @@ proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
span: icon "pin", "Pinned Tweet" span: icon "pin", "Pinned Tweet"
tdiv(class="tweet-header"): tdiv(class="tweet-header"):
a(class="tweet-avatar", href=("/" & tweet.profile.username)): a(class="tweet-avatar", href=("/" & tweet.user.username)):
var size = "_bigger" var size = "_bigger"
if not prefs.autoplayGifs and tweet.profile.userPic.endsWith("gif"): if not prefs.autoplayGifs and tweet.user.userPic.endsWith("gif"):
size = "_400x400" size = "_400x400"
genImg(tweet.profile.getUserPic(size), class=prefs.getAvatarClass) genImg(tweet.user.getUserPic(size), class=prefs.getAvatarClass)
tdiv(class="tweet-name-row"): tdiv(class="tweet-name-row"):
tdiv(class="fullname-and-username"): tdiv(class="fullname-and-username"):
linkUser(tweet.profile, class="fullname") linkUser(tweet.user, class="fullname")
linkUser(tweet.profile, class="username") linkUser(tweet.user, class="username")
span(class="tweet-date"): span(class="tweet-date"):
a(href=getLink(tweet), title=tweet.getTime): a(href=getLink(tweet), title=tweet.getTime):
@ -203,14 +203,14 @@ proc renderReply(tweet: Tweet): VNode =
if i > 0: text " " if i > 0: text " "
a(href=("/" & u)): text "@" & u a(href=("/" & u)): text "@" & u
proc renderAttribution(profile: Profile; prefs: Prefs): VNode = proc renderAttribution(user: User; prefs: Prefs): VNode =
buildHtml(a(class="attribution", href=("/" & profile.username))): buildHtml(a(class="attribution", href=("/" & user.username))):
renderMiniAvatar(profile, prefs) renderMiniAvatar(user, prefs)
strong: text profile.fullname strong: text user.fullname
if profile.verified: if user.verified:
icon "ok", class="verified-icon", title="Verified account" icon "ok", class="verified-icon", title="Verified account"
proc renderMediaTags(tags: seq[Profile]): VNode = proc renderMediaTags(tags: seq[User]): VNode =
buildHtml(tdiv(class="media-tag-block")): buildHtml(tdiv(class="media-tag-block")):
icon "user" icon "user"
for i, p in tags: for i, p in tags:
@ -244,9 +244,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
tdiv(class="tweet-name-row"): tdiv(class="tweet-name-row"):
tdiv(class="fullname-and-username"): tdiv(class="fullname-and-username"):
renderMiniAvatar(quote.profile, prefs) renderMiniAvatar(quote.user, prefs)
linkUser(quote.profile, class="fullname") linkUser(quote.user, class="fullname")
linkUser(quote.profile, class="username") linkUser(quote.user, class="username")
span(class="tweet-date"): span(class="tweet-date"):
a(href=getLink(quote), title=quote.getTime): a(href=getLink(quote), title=quote.getTime):
@ -301,7 +301,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
var tweet = fullTweet var tweet = fullTweet
if tweet.retweet.isSome: if tweet.retweet.isSome:
tweet = tweet.retweet.get tweet = tweet.retweet.get
retweet = fullTweet.profile.fullname retweet = fullTweet.user.fullname
buildHtml(tdiv(class=("timeline-item " & divClass))): buildHtml(tdiv(class=("timeline-item " & divClass))):
if not mainTweet: if not mainTweet:
@ -312,7 +312,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderHeader(tweet, retweet, prefs) renderHeader(tweet, retweet, prefs)
if not afterTweet and index == 0 and tweet.reply.len > 0 and if not afterTweet and index == 0 and tweet.reply.len > 0 and
(tweet.reply.len > 1 or tweet.reply[0] != tweet.profile.username): (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
renderReply(tweet) renderReply(tweet)
var tweetClass = "tweet-content media-body" var tweetClass = "tweet-content media-body"

View file

@ -18,9 +18,8 @@ protected = [
invalid = [['thisprofiledoesntexist'], ['%']] invalid = [['thisprofiledoesntexist'], ['%']]
banner_color = [ banner_color = [
['TheTwoffice', '29, 161, 242'], ['nim_lang', '22, 25, 32'],
['profiletest', '80, 176, 58'], ['rustlang', '35, 31, 32']
['nim_lang', '24, 26, 36']
] ]
banner_image = [ banner_image = [