Replace /.tokens with /.health and /.accounts

nitter.example.conf

@@ -23,7 +23,7 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints (/.tokens)
+enableDebug = false  # enable request logs and debug endpoints (/.accounts)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
 tokenCount = 10

src/apiutils.nim

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
 import jsony, packedjson, zippy, oauth1
-import types, tokens, consts, parserutils, http_pool
+import types, auth, consts, parserutils, http_pool
 import experimental/types/common
 
 const
@@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   except OSError as e:
     raise e
   except Exception as e:
-    let id = if account.isNil: "null" else: account.id
+    let id = if account.isNil: "null" else: $account.id
     echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
     raise rateLimitError()
   finally:

src/auth.nim

@@ -1,5 +1,5 @@
 #SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, times, json, random, strutils, tables, sets, os
+import asyncdispatch, times, json, random, strutils, tables, intsets, os
 import types
 import experimental/parser/guestaccount
 
@@ -7,6 +7,21 @@ import experimental/parser/guestaccount
 const
   maxConcurrentReqs = 2
   dayInSeconds = 24 * 60 * 60
+  apiMaxReqs: Table[Api, int] = {
+    Api.search: 50,
+    Api.tweetDetail: 150,
+    Api.photoRail: 180,
+    Api.userTweets: 500,
+    Api.userTweetsAndReplies: 500,
+    Api.userMedia: 500,
+    Api.userRestId: 500,
+    Api.userScreenName: 500,
+    Api.tweetResult: 500,
+    Api.list: 500,
+    Api.listTweets: 500,
+    Api.listMembers: 500,
+    Api.listBySlug: 500
+  }.toTable
 
 var
   accountPool: seq[GuestAccount]
@@ -15,20 +30,64 @@ var
 template log(str: varargs[string, `$`]) =
   if enableLogging: echo "[accounts] ", str.join("")
 
-proc getPoolJson*(): JsonNode =
+proc getAccountPoolHealth*(): JsonNode =
-  var
-    list = newJObject()
-    totalReqs = 0
-    totalPending = 0
-    limited: HashSet[string]
-    reqsPerApi: Table[string, int]
-
   let now = epochTime().int
 
-  for account in accountPool:
+  var
-    totalPending.inc(account.pending)
+    totalReqs = 0
+    limited: IntSet
+    reqsPerApi: Table[string, int]
+    oldest = now
+    newest = 0
+    average = 0
 
-    var includeAccount = false
+  for account in accountPool:
+    # Twitter snowflake conversion
+    let created = ((account.id shr 22) + 1288834974657) div 1000
+
+    if created > newest:
+      newest = created
+    if created < oldest:
+      oldest = created
+    average.inc created
+
+    for api in account.apis.keys:
+      let
+        apiStatus = account.apis[api]
+        reqs = apiMaxReqs[api] - apiStatus.remaining
+
+      reqsPerApi.mgetOrPut($api, 0).inc reqs
+      totalReqs.inc reqs
+
+      if apiStatus.limited:
+        limited.incl account.id
+
+  if accountPool.len > 0:
+    average = average div accountPool.len
+  else:
+    oldest = 0
+    average = 0
+
+  return %*{
+    "accounts": %*{
+      "total": accountPool.len,
+      "active": accountPool.len - limited.card,
+      "limited": limited.card,
+      "oldest": $fromUnix(oldest),
+      "newest": $fromUnix(newest),
+      "average": $fromUnix(average)
+    },
+    "requests": %*{
+      "total": totalReqs,
+      "apis": reqsPerApi
+    }
+  }
+
+proc getAccountPoolDebug*(): JsonNode =
+  let now = epochTime().int
+  var list = newJObject()
+
+  for account in accountPool:
     let accountJson = %*{
       "apis": newJObject(),
       "pending": account.pending,
@@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode =
 
       if apiStatus.limited:
         obj["limited"] = %true
-        limited.incl account.id
 
       accountJson{"apis", $api} = obj
-      includeAccount = true
+      list[$account.id] = accountJson
 
-      let
+  return %list
-        maxReqs =
-          case api
-          of Api.search: 50
-          of Api.tweetDetail: 150
-          of Api.photoRail: 180
-          of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
-             Api.userRestId, Api.userScreenName,
-             Api.tweetResult,
-             Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
-        reqs = maxReqs - apiStatus.remaining
-
-      reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
-      totalReqs.inc(reqs)
-
-    if includeAccount:
-      list[account.id] = accountJson
-
-  return %*{
-    "amount": accountPool.len,
-    "limited": limited.card,
-    "requests": totalReqs,
-    "pending": totalPending,
-    "apis": reqsPerApi,
-    "accounts": list
-  }
 
 proc rateLimitError*(): ref RateLimitError =
   newException(RateLimitError, "rate limited")

src/experimental/parser/guestaccount.nim

@@ -1,3 +1,4 @@
+import std/strutils
 import jsony
 import ../types/guestaccount
 from ../../types import GuestAccount
@@ -5,7 +6,7 @@ from ../../types import GuestAccount
 proc toGuestAccount(account: RawAccount): GuestAccount =
   let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
   result = GuestAccount(
-    id: id,
+    id: parseBiggestInt(id),
     oauthToken: account.oauthToken,
     oauthSecret: account.oauthTokenSecret
   )

src/nitter.nim

@@ -6,7 +6,7 @@ from os import getEnv
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool, tokens
+import types, config, prefs, formatters, redis_cache, http_pool, auth
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,

src/routes/debug.nim

@@ -1,10 +1,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import jester
 import router_utils
-import ".."/[tokens, types]
+import ".."/[auth, types]
 
 proc createDebugRouter*(cfg: Config) =
   router debug:
-    get "/.tokens":
+    get "/.health":
+      respJson getAccountPoolHealth()
+
+    get "/.accounts":
       cond cfg.enableDebug
-      respJson getPoolJson()
+      respJson getAccountPoolDebug()

src/types.nim

@@ -36,7 +36,7 @@ type
     limitedAt*: int
 
   GuestAccount* = ref object
-    id*: string
+    id*: BiggestInt
     oauthToken*: string
     oauthSecret*: string
     pending*: int