Add list support

src/api.nim

@@ -1,2 +1,2 @@
-import api/[profile, timeline, tweet, search, media]
-export profile, timeline, tweet, search, media
+import api/[profile, timeline, tweet, search, media, list]
+export profile, timeline, tweet, search, media, list

src/api/consts.nim

@@ -11,6 +11,8 @@ const
 
   timelineUrl* = "i/profiles/show/$1/timeline/tweets"
   timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
+  listUrl* = "$1/lists/$2/timeline"
+  listMembersUrl* = "$1/lists/$2/members"
   profilePopupUrl* = "i/profiles/popup"
   profileIntentUrl* = "intent/user"
   searchUrl* = "i/search/timeline"

src/api/list.nim

@@ -0,0 +1,83 @@
+import httpclient, asyncdispatch, htmlparser, strformat
+import sequtils, strutils, json, uri
+
+import ".."/[types, parser, parserutils, query]
+import utils, consts, timeline, search
+
+proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
+  let url = base / (listUrl % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $url,
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "Accept-Language": lang
+  })
+
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "reset_error_state": "false"
+  })
+
+  if after.len > 0:
+    params.add {"max_position": after}
+
+  let json = await fetchJson(url ? params, headers)
+  result = await finishTimeline(json, Query(), after, agent)
+  if result.content.len > 0:
+    result.minId = result.content[^1].id
+
+proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
+  let url = base / (listMembersUrl % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": htmlAccept,
+    "Referer": $(base / &"{username}/lists/{list}/members"),
+    "User-Agent": agent,
+    "Accept-Language": lang
+  })
+
+  let html = await fetchHtml(url, headers)
+
+  result = Result[Profile](
+    minId: html.selectAttr(".stream-container", "data-min-position"),
+    hasMore: html.select(".has-more-items") != nil,
+    beginning: true,
+    query: Query(kind: users),
+    content: html.selectAll(".account").map(parseListProfile)
+  )
+
+proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
+  let url = base / ((listMembersUrl & "/timeline") % [username, list])
+
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / &"{username}/lists/{list}/members"),
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "X-Push-With": "XMLHttpRequest",
+    "Accept-Language": lang
+  })
+
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "reset_error_state": "false"
+  })
+
+  if after.len > 0:
+    params.add {"max_position": after}
+
+  let json = await fetchJson(url ? params, headers)
+
+  result = getResult[Profile](json, Query(kind: users), after)
+  if json == nil or not json.hasKey("items_html"): return
+
+  let html = json["items_html"].to(string)
+  result.hasMore = html != "\n"
+  for p in parseHtml(html).selectAll(".account"):
+    result.content.add parseListProfile(p)

src/api/search.nim

@@ -7,7 +7,7 @@ import utils, consts, timeline
 proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
   if json == nil: return Result[T](beginning: true, query: query)
   Result[T](
-    hasMore: json["has_more_items"].to(bool),
+    hasMore: json.getOrDefault("has_more_items").getBool(false),
     maxId: json.getOrDefault("max_position").getStr(""),
     minId: json.getOrDefault("min_position").getStr("").cleanPos(),
     query: query,
@@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
 
 proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
   let
-    kind = if query.kind == users: "users" else: "tweets"
+    kind = if query.kind == userSearch: "users" else: "tweets"
     pos = when T is Tweet: genPos(after) else: after
 
     param = genQueryParam(query)
@@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
     return Result[T](query: query, beginning: true)
 
   let json = await fetchJson(base / searchUrl ? params, headers)
-  if json == nil: return Result[T](query: query, beginning: true)
 
   result = getResult[T](json, query, after)
-  if not json.hasKey("items_html"): return
+  if json == nil or not json.hasKey("items_html"): return
 
   when T is Tweet:
     result = await finishTimeline(json, query, after, agent)

src/api/timeline.nim

@@ -1,4 +1,4 @@
-import httpclient, asyncdispatch, htmlparser
+import httpclient, asyncdispatch, htmlparser, strformat
 import sequtils, strutils, json, xmltree, uri
 
 import ".."/[types, parser, parserutils, formatters, query]

src/nitter.nim

@@ -5,13 +5,14 @@ import jester
 
 import types, config, prefs
 import views/[general, about]
-import routes/[preferences, timeline, status, media, search, rss]
+import routes/[preferences, timeline, status, media, search, rss, list]
 
 const configPath {.strdefine.} = "./nitter.conf"
 let cfg = getConfig(configPath)
 
 createPrefRouter(cfg)
 createTimelineRouter(cfg)
+createListRouter(cfg)
 createStatusRouter(cfg)
 createSearchRouter(cfg)
 createMediaRouter(cfg)
@@ -24,15 +25,16 @@ settings:
 
 routes:
   get "/":
-    resp renderMain(renderSearch(), Prefs(), cfg.title)
+    resp renderMain(renderSearch(), request, cfg.title)
 
   get "/about":
-    resp renderMain(renderAbout(), Prefs(), cfg.title)
+    resp renderMain(renderAbout(), request, cfg.title)
 
   extend preferences, ""
   extend rss, ""
   extend search, ""
   extend timeline, ""
+  extend list, ""
   extend status, ""
   extend media, ""
 

src/parser.nim

@@ -39,6 +39,16 @@ proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
 
   result.getPopupStats(profile)
 
+proc parseListProfile*(profile: XmlNode): Profile =
+  result = Profile(
+    fullname:  profile.getName(".fullname"),
+    username:  profile.getUsername(".username"),
+    bio:       profile.getBio(".bio"),
+    userpic:   profile.getAvatar(".avatar"),
+    verified:  isVerified(profile),
+    protected: isProtected(profile),
+  )
+
 proc parseIntentProfile*(profile: XmlNode): Profile =
   result = Profile(
     fullname:  profile.getName("a.fn.url.alternate-context"),

src/query.nim

@@ -58,7 +58,7 @@ proc genQueryParam*(query: Query): string =
   var filters: seq[string]
   var param: string
 
-  if query.kind == users:
+  if query.kind == userSearch:
     return query.text
 
   for i, user in query.fromUser:
@@ -84,7 +84,7 @@ proc genQueryParam*(query: Query): string =
     result &= " " & query.text
 
 proc genQueryUrl*(query: Query): string =
-  if query.kind notin {custom, users}: return
+  if query.kind notin {custom, userSearch}: return
 
   var params = @[&"kind={query.kind}"]
   if query.text.len > 0:

src/routes/list.nim

@@ -0,0 +1,34 @@
+import strutils
+
+import jester
+
+import router_utils
+import ".."/[query, types, api, agents]
+import ../views/[general, timeline, list]
+
+template respList*(list, timeline: typed) =
+  if list.minId.len == 0:
+    resp Http404, showError("List \"" & @"list" & "\" not found", cfg.title)
+  let html = renderList(timeline, list.query, @"name", @"list")
+  let rss = "/$1/lists/$2/rss" % [@"name", @"list"]
+  resp renderMain(html, request, cfg.title, rss=rss)
+
+proc createListRouter*(cfg: Config) =
+  router list:
+    get "/@name/lists/@list":
+      cond '.' notin @"name"
+      let
+        list = await getListTimeline(@"name", @"list", getAgent(), @"after")
+        tweets = renderTimelineTweets(list, cookiePrefs(), request.path)
+      respList list, tweets
+
+    get "/@name/lists/@list/members":
+      cond '.' notin @"name"
+      let list =
+        if @"after".len == 0:
+          await getListMembers(@"name", @"list", getAgent())
+        else:
+          await getListMembersSearch(@"name", @"list", getAgent(), @"after")
+
+      let users = renderTimelineUsers(list, cookiePrefs(), request.path)
+      respList list, users

src/routes/media.nim

@@ -3,7 +3,7 @@ import asyncfile, uri, strutils, httpclient, os
 import jester, regex
 
 import router_utils
-import ".."/[types, formatters, prefs]
+import ".."/[types, formatters]
 import ../views/general
 
 export asyncfile, httpclient, os, strutils

src/routes/preferences.nim

@@ -3,7 +3,7 @@ import strutils, uri
 import jester
 
 import router_utils
-import ".."/[prefs, types]
+import ".."/[types]
 import ../views/[general, preferences]
 
 export preferences

src/routes/router_utils.nim

@@ -1,5 +1,5 @@
-import ../utils
-export utils
+import ../utils, ../prefs
+export utils, prefs
 
 template cookiePrefs*(): untyped {.dirty.} =
   getPrefs(request.cookies.getOrDefault("preferences"))

src/routes/rss.nim

@@ -34,3 +34,8 @@ proc createRssRouter*(cfg: Config) =
     get "/@name/search/rss":
       cond '.' notin @"name"
       respRss(await showRss(@"name", initQuery(params(request), name=(@"name"))))
+
+    get "/@name/lists/@list/rss":
+      cond '.' notin @"name"
+      let list = await getListTimeline(@"name", @"list", getAgent(), "")
+      respRss(renderListRss(list.content, @"name", @"list"))

src/routes/search.nim

@@ -3,7 +3,7 @@ import strutils, sequtils, uri
 import jester
 
 import router_utils
-import ".."/[query, types, api, agents, prefs]
+import ".."/[query, types, api, agents]
 import ../views/[general, search]
 
 export search
@@ -18,7 +18,7 @@ proc createSearchRouter*(cfg: Config) =
       let query = initQuery(params(request))
 
       case query.kind
-      of users:
+      of userSearch:
         if "," in @"text":
           redirect("/" & @"text")
         let users = await getSearch[Profile](query, @"after", getAgent())

src/routes/status.nim

@@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
 import jester
 
 import router_utils
-import ".."/[api, prefs, types, formatters, agents]
+import ".."/[api, types, formatters, agents]
 import ../views/[general, status]
 
 export uri, sequtils

src/routes/timeline.nim

@@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
 import jester
 
 import router_utils
-import ".."/[api, prefs, types, cache, formatters, agents, query]
+import ".."/[api, types, cache, formatters, agents, query]
 import ../views/[general, profile, timeline, status, search]
 
 export uri, sequtils

src/sass/tweet/_base.scss

@@ -85,8 +85,11 @@
 .replying-to {
     color: $fg_dark;
     margin: -2px 0 4px;
+
+    a {
         pointer-events: all;
     }
+}
 
 .retweet, .pinned, .tweet-stats {
     align-content: center;

src/types.nim

@@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
 
 type
   QueryKind* = enum
-    posts, replies, media, users, custom
+    posts, replies, media, users, userSearch, custom
 
   Query* = object
     kind*: QueryKind

src/utils.nim

@@ -41,7 +41,7 @@ proc cleanFilename*(filename: string): string =
   filename.replace(reg, "_")
 
 proc filterParams*(params: Table): seq[(string, string)] =
-  let filter = ["name", "id"]
+  let filter = ["name", "id", "list"]
   toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0)
 
 proc isTwitterUrl*(url: string): bool =

src/views/general.nim

@@ -71,5 +71,5 @@ proc renderError*(error: string): VNode =
     tdiv(class="error-panel"):
       span: text error
 
-proc showError*(error, title: string): string =
-  renderMain(renderError(error), Request(), title, "Error")
+template showError*(error, title: string): string =
+  renderMain(renderError(error), request, title, "Error")

src/views/list.nim

@@ -0,0 +1,20 @@
+import strformat
+import karax/[karaxdsl, vdom]
+
+import renderutils
+import ".."/[types]
+
+proc renderListTabs*(query: Query; path: string): VNode =
+  buildHtml(ul(class="tab")):
+    li(class=query.getTabClass(posts)):
+      a(href=(path)): text "Tweets"
+    li(class=query.getTabClass(users)):
+      a(href=(path & "/members")): text "Members"
+
+proc renderList*(body: VNode; query: Query; name, list: string): VNode =
+  buildHtml(tdiv(class="timeline-container")):
+    tdiv(class="timeline-header"):
+      text &"\"{list}\" by @{name}"
+
+    renderListTabs(query, &"/{name}/lists/{list}")
+    body

src/views/renderutils.nim

@@ -30,10 +30,6 @@ proc linkUser*(profile: Profile, class=""): VNode =
       text " "
       icon "lock-circled", title="Protected account"
 
-proc genImg*(url: string; class=""): VNode =
-  buildHtml():
-    img(src=getPicUrl(url), class=class, alt="Image")
-
 proc linkText*(text: string; class=""): VNode =
   let url = if "http" notin text: "http://" & text else: text
   buildHtml():
@@ -91,3 +87,12 @@ proc genDate*(pref, state: string): VNode =
     else:
       verbatim &"<input name={pref} type=\"date\"/>"
     icon "calendar"
+
+proc genImg*(url: string; class=""): VNode =
+  buildHtml():
+    img(src=getPicUrl(url), class=class, alt="Image")
+
+proc getTabClass*(query: Query; tab: QueryKind): string =
+  result = "tab-item"
+  if query.kind == tab:
+    result &= " active"

src/views/rss.nimf

@@ -71,3 +71,29 @@
   </channel>
 </rss>
 #end proc
+#
+#proc renderListRss*(tweets: seq[Tweet]; name, list: string): string =
+#let prefs = Prefs(replaceTwitter: hostname)
+#result = ""
+<?xml version="1.0" encoding="UTF-8"?>
+<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
+  <channel>
+    <atom:link href="https://${hostname}/${name}/lists/${list}/rss" rel="self" type="application/rss+xml" />
+    <title>${list} / @${name}</title>
+    <link>https://${hostname}/${name}/lists/${list}</link>
+    <description>Twitter feed for: ${list} by @${name}. Generated by ${hostname}</description>
+    <language>en-us</language>
+    <ttl>40</ttl>
+    #for tweet in tweets:
+    <item>
+      <title>${getTitle(tweet, prefs)}</title>
+      <dc:creator>@${tweet.profile.username}</dc:creator>
+      <description><![CDATA[${renderRssTweet(tweet, prefs).strip(chars={'\n'})}]]></description>
+      <pubDate>${getRfc822Time(tweet)}</pubDate>
+      <guid>https://${hostname}${getLink(tweet)}</guid>
+      <link>https://${hostname}${getLink(tweet)}</link>
+    </item>
+    #end for
+  </channel>
+</rss>
+#end proc

src/views/search.nim

@@ -23,15 +23,10 @@ proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="search-bar"):
       form(`method`="get", action="/search"):
-        hiddenField("kind", "users")
+        hiddenField("kind", "userSearch")
         input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
         button(`type`="submit"): icon "search"
 
-proc getTabClass(query: Query; tab: QueryKind): string =
-  result = "tab-item"
-  if query.kind == tab:
-    result &= " active"
-
 proc renderProfileTabs*(query: Query; username: string): VNode =
   let link = "/" & username
   buildHtml(ul(class="tab")):
@@ -50,8 +45,8 @@ proc renderSearchTabs*(query: Query): VNode =
     li(class=query.getTabClass(custom)):
       q.kind = custom
       a(href=("?" & genQueryUrl(q))): text "Tweets"
-    li(class=query.getTabClass(users)):
-      q.kind = users
+    li(class=query.getTabClass(userSearch)):
+      q.kind = userSearch
       a(href=("?" & genQueryUrl(q))): text "Users"
 
 proc isPanelOpen(q: Query): bool =
@@ -114,7 +109,7 @@ proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-container")):
     tdiv(class="timeline-header"):
       form(`method`="get", action="/search", class="search-field"):
-        hiddenField("kind", "users")
+        hiddenField("kind", "userSearch")
         genInput("text", "", users.query.text, "Enter username...", class="pref-inline")
         button(`type`="submit"): icon "search"
 

src/views/timeline.nim

@@ -64,6 +64,7 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
     if results.content.len > 0:
       for user in results.content:
         renderUser(user, prefs)
+      if results.minId != "0":
         renderMore(results.query, results.minId)
     elif results.beginning:
       renderNoneFound()