mirror of
https://github.com/zedeus/nitter.git
synced 2024-12-23 03:15:40 +00:00
157 lines
4.9 KiB
Nim
157 lines
4.9 KiB
Nim
# SPDX-License-Identifier: AGPL-3.0-only
|
|
import asyncdispatch, times, strutils, tables, hashes
|
|
import redis, redpool, flatty, supersnappy
|
|
|
|
import types, api
|
|
|
|
const
|
|
redisNil = "\0\0"
|
|
baseCacheTime = 60 * 60
|
|
|
|
var
|
|
pool: RedisPool
|
|
rssCacheTime: int
|
|
listCacheTime*: int
|
|
|
|
# flatty can't serialize DateTime, so we need to define this
|
|
proc toFlatty*(s: var string, x: DateTime) =
|
|
s.toFlatty(x.toTime().toUnix())
|
|
|
|
proc fromFlatty*(s: string, i: var int, x: var DateTime) =
|
|
var unix: int64
|
|
s.fromFlatty(i, unix)
|
|
x = fromUnix(unix).utc()
|
|
|
|
proc setCacheTimes*(cfg: Config) =
|
|
rssCacheTime = cfg.rssCacheTime * 60
|
|
listCacheTime = cfg.listCacheTime * 60
|
|
|
|
proc migrate*(key, match: string) {.async.} =
|
|
pool.withAcquire(r):
|
|
let hasKey = await r.get(key)
|
|
if hasKey == redisNil:
|
|
let list = await r.scan(newCursor(0), match, 100000)
|
|
r.startPipelining()
|
|
for item in list:
|
|
discard await r.del(item)
|
|
await r.setk(key, "true")
|
|
discard await r.flushPipeline()
|
|
|
|
proc initRedisPool*(cfg: Config) {.async.} =
|
|
try:
|
|
pool = await newRedisPool(cfg.redisConns, cfg.redisMaxConns,
|
|
host=cfg.redisHost, port=cfg.redisPort,
|
|
password=cfg.redisPassword)
|
|
|
|
await migrate("flatty", "*:*")
|
|
await migrate("snappyRss", "rss:*")
|
|
await migrate("userBuckets", "p:*")
|
|
await migrate("profileDates", "p:*")
|
|
await migrate("profileStats", "p:*")
|
|
|
|
pool.withAcquire(r):
|
|
# optimize memory usage for profile ID buckets
|
|
await r.configSet("hash-max-ziplist-entries", "1000")
|
|
|
|
except OSError:
|
|
stdout.write "Failed to connect to Redis.\n"
|
|
stdout.flushFile
|
|
quit(1)
|
|
|
|
template pidKey(name: string): string = "pid:" & $(hash(name) div 1_000_000)
|
|
template profileKey(name: string): string = "p:" & name
|
|
template listKey(l: List): string = "l:" & l.id
|
|
|
|
proc get(query: string): Future[string] {.async.} =
|
|
pool.withAcquire(r):
|
|
result = await r.get(query)
|
|
|
|
proc setEx(key: string; time: int; data: string) {.async.} =
|
|
pool.withAcquire(r):
|
|
discard await r.setEx(key, time, data)
|
|
|
|
proc cache*(data: List) {.async.} =
|
|
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
|
|
|
|
proc cache*(data: PhotoRail; name: string) {.async.} =
|
|
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
|
|
|
|
proc cache*(data: Profile) {.async.} =
|
|
if data.username.len == 0 or data.id.len == 0: return
|
|
let name = toLower(data.username)
|
|
pool.withAcquire(r):
|
|
r.startPipelining()
|
|
discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
|
|
discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
|
|
discard await r.hSet(name.pidKey, name, data.id)
|
|
discard await r.flushPipeline()
|
|
|
|
proc cacheProfileId*(username, id: string) {.async.} =
|
|
if username.len == 0 or id.len == 0: return
|
|
let name = toLower(username)
|
|
pool.withAcquire(r):
|
|
discard await r.hSet(name.pidKey, name, id)
|
|
|
|
proc cacheRss*(query: string; rss: Rss) {.async.} =
|
|
let key = "rss:" & query
|
|
pool.withAcquire(r):
|
|
r.startPipelining()
|
|
discard await r.hSet(key, "rss", rss.feed)
|
|
discard await r.hSet(key, "min", rss.cursor)
|
|
discard await r.expire(key, rssCacheTime)
|
|
discard await r.flushPipeline()
|
|
|
|
proc getProfileId*(username: string): Future[string] {.async.} =
|
|
let name = toLower(username)
|
|
pool.withAcquire(r):
|
|
result = await r.hGet(name.pidKey, name)
|
|
if result == redisNil:
|
|
result.setLen(0)
|
|
|
|
proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.} =
|
|
let prof = await get("p:" & toLower(username))
|
|
if prof != redisNil:
|
|
result = fromFlatty(uncompress(prof), Profile)
|
|
elif fetch:
|
|
result = await getProfile(username)
|
|
|
|
proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
|
|
let username = await get("i:" & userId)
|
|
if username != redisNil:
|
|
result = username
|
|
else:
|
|
let profile = await getProfileById(userId)
|
|
result = profile.username
|
|
await cache(profile)
|
|
|
|
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
|
if name.len == 0: return
|
|
let rail = await get("pr:" & toLower(name))
|
|
if rail != redisNil:
|
|
result = fromFlatty(uncompress(rail), PhotoRail)
|
|
else:
|
|
result = await getPhotoRail(name)
|
|
await cache(result, name)
|
|
|
|
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
|
|
let list = if id.len == 0: redisNil
|
|
else: await get("l:" & id)
|
|
|
|
if list != redisNil:
|
|
result = fromFlatty(uncompress(list), List)
|
|
else:
|
|
if id.len > 0:
|
|
result = await getGraphList(id)
|
|
else:
|
|
result = await getGraphListBySlug(username, slug)
|
|
await cache(result)
|
|
|
|
proc getCachedRss*(key: string): Future[Rss] {.async.} =
|
|
let k = "rss:" & key
|
|
pool.withAcquire(r):
|
|
result.cursor = await r.hGet(k, "min")
|
|
if result.cursor.len > 2:
|
|
result.feed = await r.hGet(k, "rss")
|
|
else:
|
|
result.cursor.setLen 0
|