Optimize RSS requests by disabling media fetching

src/api/list.nim

@@ -4,7 +4,7 @@ import sequtils, strutils, json, uri
 import ".."/[types, parser, parserutils, query]
 import utils, consts, timeline, search
 
-proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
+proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} =
   let url = base / (listUrl % [username, list])
 
   var params = toSeq({
@@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
     params.add {"max_position": after}
 
   let json = await fetchJson(url ? params, genHeaders(agent, url))
-  result = await finishTimeline(json, Query(), after, agent)
+  result = await finishTimeline(json, Query(), after, agent, media)
   if result.content.len == 0:
     return
 

src/api/search.nim

@@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
     beginning: after.len == 0
   )
 
-proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
+proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} =
   let
     kind = if query.kind == users: "users" else: "tweets"
 
@@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
   if json == nil or not json.hasKey("items_html"): return
 
   when T is Tweet:
-    result = await finishTimeline(json, query, after, agent)
+    result = await finishTimeline(json, query, after, agent, media)
   elif T is Profile:
     let html = json["items_html"].to(string)
     result.hasMore = html != "\n"

src/api/timeline.nim

@@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
             getCards(thread, agent),
             getPolls(thread, agent))
 
-proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
+proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
+                     media=true): Future[Timeline] {.async.} =
   result = getResult[Tweet](json, query, after)
   if json == nil: return
 
@@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
   let html = parseHtml(json["items_html"].to(string))
   let thread = parseChain(html)
 
-  await getMedia(thread, agent)
+  if media: await getMedia(thread, agent)
   result.content = thread.content
 
-proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
+proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
   var params = toSeq({
     "include_available_features": "1",
     "include_entities": "1",
@@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
   let headers = genHeaders(agent, base / username, xml=true)
   let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
 
-  result = await finishTimeline(json, Query(), after, agent)
+  result = await finishTimeline(json, Query(), after, agent, media)
 
-proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
+proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
   var url = base / username
   if after.len > 0:
     url = url ? {"max_position": after}
@@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
     timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
     profile = parseTimelineProfile(html)
 
-  await getMedia(timeline, agent)
+  if media: await getMedia(timeline, agent)
   result = (profile, timeline)

src/routes/rss.nim

@@ -9,7 +9,9 @@ import ../views/general
 include "../views/rss.nimf"
 
 proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
-  let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query)
+  let (profile, timeline, _) =
+    await fetchSingleTimeline(name, "", getAgent(), query, media=false)
+
   if timeline != nil:
     return renderTimelineRss(timeline, profile, hostname)
 
@@ -28,7 +30,7 @@ proc createRssRouter*(cfg: Config) =
       if query.kind != tweets:
         resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
 
-      let tweets = await getSearch[Tweet](query, "", getAgent())
+      let tweets = await getSearch[Tweet](query, "", getAgent(), media=false)
       respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname))
 
     get "/@name/rss":
@@ -49,5 +51,5 @@ proc createRssRouter*(cfg: Config) =
 
     get "/@name/lists/@list/rss":
       cond '.' notin @"name"
-      let list = await getListTimeline(@"name", @"list", getAgent(), "")
+      let list = await getListTimeline(@"name", @"list", getAgent(), "", media=false)
       respRss(renderListRss(list.content, @"name", @"list", cfg.hostname))

src/routes/timeline.nim

@@ -13,8 +13,8 @@ export profile, timeline, status
 
 type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
 
-proc fetchSingleTimeline*(name, after, agent: string;
+proc fetchSingleTimeline*(name, after, agent: string; query: Query;
-                          query: Query): Future[ProfileTimeline] {.async.} =
+                          media=true): Future[ProfileTimeline] {.async.} =
   let railFut = getPhotoRail(name, agent)
 
   var timeline: Timeline
@@ -26,12 +26,12 @@ proc fetchSingleTimeline*(name, after, agent: string;
 
   if query.kind == posts:
     if cachedProfile.isSome:
-      timeline = await getTimeline(name, after, agent)
+      timeline = await getTimeline(name, after, agent, media)
     else:
-      (profile, timeline) = await getProfileAndTimeline(name, agent, after)
+      (profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
       cache(profile)
   else:
-    var timelineFut = getSearch[Tweet](query, after, agent)
+    var timelineFut = getSearch[Tweet](query, after, agent, media)
     if cachedProfile.isNone:
       profile = await getCachedProfile(name, agent)
     timeline = await timelineFut