Merge pull request #3 from zedeus/card-support

Card support

README.md

@@ -27,17 +27,15 @@ is on implementing missing features.
 
 - Search (images/videos, hashtags, etc.)
 - Custom timeline filter
-- Media-only/gallery view
 - Nitter link previews
 - Server configuration
-- Caching (waiting for [moigagoo/norm#19](https://github.com/moigagoo/norm/pull/19))
-- Twitter "Cards" (link previews)
+- More caching (waiting for [moigagoo/norm#19](https://github.com/moigagoo/norm/pull/19))
 - Simple account system with customizable feed
-- Emoji support (WIP, needs font)
 - Video support with hls.js
 - Json API endpoints
 - Themes
 - Nitter logo
+- Emoji support (WIP, uses native font for now)
 
 ## Why?
 

public/style.css

@@ -237,7 +237,7 @@ nav {
 
 .gallery-row .attachment:last-child, .gallery-row .attachments:last-child, .video-container {
     margin: 0;
-    max-height: 500px;
+    max-height: 530px;
 }
 
 .attachments .attachment {
@@ -419,7 +419,6 @@ video {
 .profile-banner-color {
     width: 100%;
     padding-bottom: 25%;
-    margin-bottom: 8px;
 }
 
 .profile-card {
@@ -882,6 +881,136 @@ video {
     margin: 0;
 }
 
+.card {
+    margin: 5px 0;
+}
+
+.card-container {
+    border-radius: 10px;
+    border-width: 1px;
+    border-style: solid;
+    border-color: #404040;
+    background-color: #121212;
+    overflow: hidden;
+    color: inherit;
+    display: flex;
+    text-decoration: none !important;
+}
+
+.card-container:hover {
+    border-color: #808080;
+}
+
+.card-container .attachments {
+    margin: 0;
+    border-radius: 0;
+}
+
+.large .card-container {
+    display: block;
+}
+
+.card-content {
+    padding: 0.5em;
+}
+
+.card-title {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    font-weight: bold;
+    font-size: 1.15em;
+}
+
+.card-description {
+    margin: 0.3em 0;
+}
+
+.card-destination {
+    color: hsla(240,1%,73%,.9);
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    display: block;
+}
+
+.card-image-container {
+    width: 98px;
+    flex-shrink: 0;
+    position: relative;
+    overflow: hidden;
+}
+
+.large .card-image-container {
+    width: unset;
+}
+
+.card-image-container:before {
+    content: "";
+    display: block;
+    padding-top: 100%;
+}
+
+.large .card-image-container:before {
+    display: none;
+}
+
+.card-image {
+    position: absolute;
+    top: 0;
+    left: 0;
+    bottom: 0;
+    right: 0;
+    background-color: white;
+}
+
+.large .card-image {
+    position: unset;
+    border-style: solid;
+    border-color: #404040;
+    border-width: 0;
+    border-bottom-width: 1px;
+}
+
+.card-image img {
+    width: 100%;
+    height: 100%;
+    display: block;
+    object-fit: cover;
+}
+
+.card-overlay {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    opacity: 0.8;
+}
+
+.card-overlay-circle {
+    border-radius: 50%;
+    background-color: #404040;
+    width: 40px;
+    height: 40px;
+    align-items: center;
+    display: flex;
+    border-width: 5px;
+    border-color: #d8574d;
+    border-style: solid;
+}
+
+.card-overlay-triangle {
+    width: 0;
+    height: 0;
+    border-style: solid;
+    border-width: 12px 0 12px 17px;
+    border-color: transparent transparent transparent #d8574d;
+    margin-left: 14px;
+}
+
 .poll-meter {
     overflow: hidden;
     position: relative;

src/api.nim

@@ -106,7 +106,11 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
     await getVideo(tweet, guestToken)
     return
 
+  if tweet.card.isNone:
     tweet.video = some(parseVideo(json))
+  else:
+    get(tweet.card).video = some(parseVideo(json))
+    tweet.video = none(Video)
   tokenUses.inc
 
 proc getVideos*(thread: Thread; token="") {.async.} =
@@ -163,6 +167,36 @@ proc getConversationPolls*(convo: Conversation) {.async.} =
   futs.add convo.replies.map(getPolls)
   await all(futs)
 
+proc getCard*(tweet: Tweet) {.async.} =
+  if tweet.card.isNone(): return
+
+  let headers = newHttpHeaders({
+    "Accept": cardAccept,
+    "Referer": $(base / getLink(tweet)),
+    "User-Agent": agent,
+    "Authority": "twitter.com",
+    "Accept-Language": lang,
+  })
+
+  let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
+  let html = await fetchHtml(base / query, headers)
+  if html == nil: return
+
+  parseCard(get(tweet.card), html)
+
+proc getCards*(thread: Thread) {.async.} =
+  if thread == nil: return
+  var cards = thread.tweets.filterIt(it.card.isSome)
+  await all(cards.map(getCard))
+
+proc getConversationCards*(convo: Conversation) {.async.} =
+  var futs: seq[Future[void]]
+  futs.add getCard(convo.tweet)
+  futs.add getCards(convo.before)
+  futs.add getCards(convo.after)
+  futs.add convo.replies.map(getCards)
+  await all(futs)
+
 proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} =
   let headers = newHttpHeaders({
     "Accept": jsonAccept,
@@ -234,9 +268,12 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
 
   result = parseConversation(html)
 
-  let vidsFut = getConversationVideos(result)
-  let pollFut = getConversationPolls(result)
-  await all(vidsFut, pollFut)
+  let
+    vidsFut = getConversationVideos(result)
+    pollFut = getConversationPolls(result)
+    cardFut = getConversationCards(result)
+
+  await all(vidsFut, pollFut, cardFut)
 
 proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} =
   if json == nil: return Timeline()
@@ -257,8 +294,9 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
     thread = parseThread(html)
     vidsFut = getVideos(thread)
     pollFut = getPolls(thread)
+    cardFut = getCards(thread)
 
-  await all(vidsFut, pollFut)
+  await all(vidsFut, pollFut, cardFut)
   result.tweets = thread.tweets
 
 proc getTimeline*(username, after: string): Future[Timeline] {.async.} =

src/formatters.nim

@@ -10,7 +10,6 @@ const
   emailRegex = re"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
   usernameRegex = re"(^|[^A-z0-9_?])@([A-z0-9_]+)"
   picRegex = re"pic.twitter.com/[^ ]+"
-  cardRegex = re"(https?://)?cards.twitter.com/[^ ]+"
   ellipsisRegex = re" ?…"
   nbsp = $Rune(0x000A0)
 
@@ -60,7 +59,6 @@ proc linkifyText*(text: string): string =
 proc stripTwitterUrls*(text: string): string =
   result = text
   result = result.replace(picRegex, "")
-  result = result.replace(cardRegex, "")
   result = result.replace(ellipsisRegex, "")
 
 proc getUserpic*(userpic: string; style=""): string =

src/parser.nim

@@ -75,7 +75,7 @@ proc parseTweet*(node: XmlNode): Tweet =
   )
 
   result.getTweetMedia(tweet)
-  result.getTweetCards(tweet)
+  result.getTweetCard(tweet)
 
   let by = tweet.selectText(".js-retweet-text > a > b")
   if by.len > 0:
@@ -178,3 +178,20 @@ proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] =
       tweetId: img.attr("data-tweet-id"),
       color: img.attr("background-color").replace("style: ", "")
     )
+
+proc parseCard*(card: var Card; node: XmlNode) =
+  card.title = node.selectText("h2.TwitterCard-title")
+  card.text = node.selectText("p.tcu-resetMargin")
+  card.dest = node.selectText("span.SummaryCard-destination")
+
+  if card.url.len == 0:
+    card.url = node.select("a").attr("href")
+
+  let image = node.select(".tcu-imageWrapper img")
+  if image != nil:
+    # workaround for issue 11713
+    card.image = some(image.attr("data-src").replace("gname", "g&name"))
+
+  if card.kind == liveEvent:
+    card.text = card.title
+    card.title = node.selectText(".TwitterCard-attribution--category")

src/parserutils.nim

@@ -1,4 +1,4 @@
-import xmltree, htmlparser, strtabs, strformat, times
+import xmltree, htmlparser, strtabs, strformat, strutils, times
 import regex
 
 import types, formatters, api
@@ -167,10 +167,36 @@ proc getQuoteMedia*(quote: var Quote; node: XmlNode) =
   elif gifBadge != nil:
     quote.badge = "GIF"
 
-proc getTweetCards*(tweet: Tweet; node: XmlNode) =
+proc getTweetCard*(tweet: Tweet; node: XmlNode) =
   if node.attr("data-has-cards") == "false": return
-  if "poll" in node.attr("data-card2-type"):
+  var cardType = node.attr("data-card2-type")
+
+  if ":" in cardType:
+    cardType = cardType.split(":")[^1]
+
+  if "poll" in cardType:
     tweet.poll = some(Poll())
+    return
+
+  let cardDiv = node.select(".card2 > .js-macaw-cards-iframe-container")
+  if cardDiv == nil: return
+
+  var card = Card(
+    id: tweet.id,
+    query: cardDiv.attr("data-src")
+  )
+
+  try:
+    card.kind = parseEnum[CardKind](cardType)
+  except ValueError:
+    card.kind = summary
+
+  let cardUrl = cardDiv.attr("data-card-url")
+  for n in node.selectAll(".tweet-text a"):
+    if n.attr("href") == cardUrl:
+      card.url = n.attr("data-expanded-url")
+
+  tweet.card = some(card)
 
 proc getMoreReplies*(node: XmlNode): int =
   let text = node.innerText().strip()

src/search.nim

@@ -21,7 +21,7 @@ const
 proc initQuery*(filters, includes, excludes, separator: string; name=""): Query =
   var sep = separator.strip().toUpper()
   Query(
-    queryType: custom,
+    kind: custom,
     filters: filters.split(",").filterIt(it in validFilters),
     includes: includes.split(",").filterIt(it in validFilters),
     excludes: excludes.split(",").filterIt(it in validFilters),
@@ -31,7 +31,7 @@ proc initQuery*(filters, includes, excludes, separator: string; name=""): Query
 
 proc getMediaQuery*(name: string): Query =
   Query(
-    queryType: media,
+    kind: media,
     filters: @["twimg", "native_video"],
     fromUser: name,
     sep: "OR"
@@ -39,7 +39,7 @@ proc getMediaQuery*(name: string): Query =
 
 proc getReplyQuery*(name: string): Query =
   Query(
-    queryType: replies,
+    kind: replies,
     includes: @["nativeretweets"],
     fromUser: name
   )
@@ -61,8 +61,8 @@ proc genQueryParam*(query: Query): string =
   return strip(param & filters.join(&" {query.sep} "))
 
 proc genQueryUrl*(query: Query): string =
-  result = &"/{query.queryType}?"
-  if query.queryType != custom: return
+  result = &"/{query.kind}?"
+  if query.kind != custom: return
 
   var params: seq[string]
   if query.filters.len > 0:

src/types.nim

@@ -31,11 +31,11 @@ db("cache.db", "", "", ""):
         .}: Time
 
 type
-  QueryType* = enum
+  QueryKind* = enum
     replies, media, custom = "search"
 
   Query* = object
-    queryType*: QueryType
+    kind*: QueryKind
     filters*: seq[string]
     includes*: seq[string]
     excludes*: seq[string]
@@ -70,6 +70,25 @@ type
     status*: string
     leader*: int
 
+  CardKind* = enum
+    summary = "summary"
+    summaryLarge = "summary_large_image"
+    promoWebsite = "promo_website"
+    promoVideo = "promo_video_website"
+    player = "player"
+    liveEvent = "live_event"
+
+  Card* = object
+    kind*: CardKind
+    id*: string
+    query*: string
+    url*: string
+    title*: string
+    dest*: string
+    text*: string
+    image*: Option[string]
+    video*: Option[Video]
+
   Quote* = object
     id*: string
     profile*: Profile
@@ -104,6 +123,7 @@ type
     stats*: TweetStats
     retweet*: Option[Retweet]
     quote*: Option[Quote]
+    card*: Option[Card]
     gif*: Option[Gif]
     video*: Option[Video]
     photos*: seq[string]

src/views/timeline.nim

@@ -14,7 +14,7 @@ proc getTabClass(timeline: Timeline; tab: string): string =
   if timeline.query.isNone:
     if tab == "tweets":
       classes.add "active"
-  elif $timeline.query.get().queryType == tab:
+  elif $timeline.query.get().kind == tab:
     classes.add "active"
 
   return classes.join(" ")

src/views/tweet.nim

@@ -77,6 +77,32 @@ proc renderPoll(poll: Poll): VNode =
     span(class="poll-info"):
       text $poll.votes & " votes • " & poll.status
 
+proc renderCardImage(card: Card): VNode =
+  buildHtml(tdiv(class="card-image-container")):
+    tdiv(class="card-image"):
+      img(src=get(card.image).getSigUrl("pic"))
+      if card.kind == player:
+        tdiv(class="card-overlay"):
+          tdiv(class="card-overlay-circle"):
+            span(class="card-overlay-triangle")
+
+proc renderCard(card: Card): VNode =
+  const largeCards = {summaryLarge, liveEvent, promoWebsite, promoVideo}
+  let large = if card.kind in largeCards: " large" else: ""
+
+  buildHtml(tdiv(class=("card" & large))):
+    a(class="card-container", href=card.url):
+      if card.image.isSome:
+        renderCardImage(card)
+      elif card.video.isSome:
+        renderVideo(get(card.video))
+
+      tdiv(class="card-content-container"):
+        tdiv(class="card-content"):
+          h2(class="card-title"): text card.title
+          p(class="card-description"): text card.text
+          span(class="card-destination"): text card.dest
+
 proc renderStats(stats: TweetStats): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): text "💬 " & $stats.replies
@@ -160,7 +186,9 @@ proc renderTweet*(tweet: Tweet; class=""; index=0; total=(-1); last=false): VNod
         if tweet.quote.isSome:
           renderQuote(tweet.quote.get())
 
-        if tweet.photos.len > 0:
+        if tweet.card.isSome:
+          renderCard(tweet.card.get())
+        elif tweet.photos.len > 0:
           renderAlbum(tweet)
         elif tweet.video.isSome:
           renderVideo(tweet.video.get())