bridging emojis from revolt to discord

bridge/src/discord/bridgeEmojis.ts

@@ -1,7 +1,12 @@
 import axios from "axios";
+import { GuildEmoji } from "discord.js";
 import JSON5 from 'json5';
+import { BRIDGED_EMOJIS, logger } from "..";
+import { client } from "./client";
 
 const EMOJI_DICT_URL = 'https://raw.githubusercontent.com/revoltchat/revite/master/src/assets/emojis.ts';
+const EMOJI_URL_BASE = 'https://dl.insrt.uk/projects/revolt/emotes/';
+const EMOJI_SERVERS = process.env.EMOJI_SERVERS?.split(',') || [];
 
 async function fetchEmojiList(): Promise<Record<string, string>> {
     const file: string = (await axios.get(EMOJI_DICT_URL)).data;
@@ -10,3 +15,129 @@ async function fetchEmojiList(): Promise<Record<string, string>> {
 
     return JSON5.parse(file.substring(start, end).trim());
 }
+
+const emojiUpdate = async () => {
+    try {
+        if (!EMOJI_SERVERS.length) return logger.info('$EMOJI_SERVERS not set, not bridging emojis.');
+
+        if (!client.readyAt) await new Promise(r => client.once('ready', r));
+        logger.info('Updating bridged emojis. Due to Discord rate limits, this can take a few hours to complete.');
+
+        const emojis = await fetchEmojiList();
+        logger.info(`Downloaded emoji list: ${Object.keys(emojis).length} emojis.`);
+
+        const servers = await Promise.all(EMOJI_SERVERS.map(id => client.guilds.fetch(id)));
+        await Promise.all(servers.map(server => server.emojis.fetch())); // Make sure all emojis are cached
+
+        const findFreeServer = (animated: boolean) => servers.find(
+            server => server.emojis.cache
+                .filter(e => e.animated == animated)
+                .size < 50
+        );
+
+        // Remove unknown emojis from servers
+        for (const server of servers) {
+            for (const emoji of server.emojis.cache) {
+                const dbEmoji = await BRIDGED_EMOJIS.findOne({
+                    emojiid: emoji[1].id,
+                });
+
+                if (!dbEmoji) {
+                    try {
+                        logger.info('Found unknown emoji; deleting.');
+                        await emoji[1].delete('Unknown emoji');
+                    } catch(e) {
+                        logger.warn('Failed to delete emoji: ' + e);
+                    }
+                }
+            }
+        }
+
+        for (const emoji of Object.entries(emojis)) {
+            const dbEmoji = await BRIDGED_EMOJIS.findOne({
+                $or: [
+                    { name: emoji[0] },
+                    { originalFileUrl: emoji[1] },
+                ],
+            });
+
+            if (!dbEmoji) {
+                // Upload to Discord
+                logger.debug('Uploading emoji: ' + emoji[1]);
+
+                const fileurl = EMOJI_URL_BASE + emoji[1].replace('custom:', '');
+                const server = findFreeServer(emoji[1].endsWith('.gif'));
+
+                if (!server) {
+                    logger.warn('Could not find a server with free emoji slots for ' + emoji[1]);
+                    continue;
+                }
+
+                let e: GuildEmoji;
+                try {
+                    e = await server.emojis.create(fileurl, emoji[0], { reason: 'Bridged Emoji' });
+                } catch(e) {
+                    logger.warn(emoji[0] + ': Failed to upload emoji: ' + e);
+                    continue;
+                }
+
+                await BRIDGED_EMOJIS.insert({
+                    animated: e.animated || false,
+                    emojiid: e.id,
+                    name: emoji[0],
+                    originalFileUrl: fileurl,
+                    server: e.guild.id,
+                });
+            }
+            else {
+                // Double check if emoji exists
+                let exists = false;
+                for (const server of servers) {
+                    if (server.emojis.cache.find(e => e.id == dbEmoji.emojiid)) {
+                        exists = true;
+                        break;
+                    }
+                }
+
+                if (!exists) {
+                    logger.info(`Emoji ${emoji[0]} does not exist; reuploading.`);
+                    await BRIDGED_EMOJIS.remove({ emojiid: dbEmoji.emojiid });
+
+                    const fileurl = EMOJI_URL_BASE + emoji[1].replace('custom:', '');
+                    const server = findFreeServer(emoji[1].endsWith('.gif'));
+
+                    if (!server) {
+                        logger.warn('Could not find a server with free emoji slots for ' + emoji[1]);
+                        continue;
+                    }
+
+                    let e: GuildEmoji;
+                    try {
+                        e = await server.emojis.create(fileurl, emoji[0], { reason: 'Bridged Emoji' });
+                    } catch(e) {
+                        logger.warn(emoji[0] + ': Failed to upload emoji: ' + e);
+                        continue;
+                    }
+
+                    await BRIDGED_EMOJIS.insert({
+                        animated: e.animated || false,
+                        emojiid: e.id,
+                        name: emoji[0],
+                        originalFileUrl: fileurl,
+                        server: e.guild.id,
+                    });
+                }
+            }
+        };
+
+        logger.done('Emoji update finished.');
+    } catch(e) {
+        logger.error('Updating bridged emojis failed');
+        console.error(e);
+    }
+};
+
+emojiUpdate();
+setInterval(emojiUpdate, 1000 * 60 * 60 * 6); // Every 6h
+
+export { fetchEmojiList }

bridge/src/discord/client.ts

@@ -24,5 +24,6 @@ const login = () => new Promise((resolve: (value: Discord.Client) => void) => {
 
 import('./events');
 import('./commands');
+import('./bridgeEmojis');
 
 export { client, login }

bridge/src/index.ts

@@ -7,6 +7,7 @@ import { ICollection } from 'monk';
 import BridgeConfig from './types/BridgeConfig';
 import BridgedMessage from './types/BridgedMessage';
 import BridgeRequest from './types/BridgeRequest';
+import DiscordBridgedEmoji from './types/DiscordBridgedEmoji';
 
 config();
 
@@ -15,6 +16,7 @@ const db = getDb();
 const BRIDGED_MESSAGES: ICollection<BridgedMessage> = db.get('bridged_messages');
 const BRIDGE_CONFIG: ICollection<BridgeConfig> = db.get('bridge_config');
 const BRIDGE_REQUESTS: ICollection<BridgeRequest> = db.get('bridge_requests');
+const BRIDGED_EMOJIS: ICollection<DiscordBridgedEmoji> = db.get('bridged_emojis');
 
 for (const v of [ 'REVOLT_TOKEN', 'DISCORD_TOKEN', 'DB_STRING' ]) {
     if (!process.env[v]) {
@@ -31,4 +33,4 @@ for (const v of [ 'REVOLT_TOKEN', 'DISCORD_TOKEN', 'DB_STRING' ]) {
     ]);
 })();
 
-export { logger, db, BRIDGED_MESSAGES, BRIDGE_CONFIG, BRIDGE_REQUESTS }
+export { logger, db, BRIDGED_MESSAGES, BRIDGE_CONFIG, BRIDGE_REQUESTS, BRIDGED_EMOJIS }

bridge/src/revolt/events.ts

@@ -1,4 +1,4 @@
-import { BRIDGED_MESSAGES, BRIDGE_CONFIG, logger } from "..";
+import { BRIDGED_EMOJIS, BRIDGED_MESSAGES, BRIDGE_CONFIG, logger } from "..";
 import { AUTUMN_URL, client } from "./client";
 import { client as discordClient } from "../discord/client";
 import { MessageEmbed, MessagePayload, TextChannel, WebhookClient, WebhookMessageOptions } from "discord.js";
@@ -7,9 +7,17 @@ import { SendableEmbed } from "revolt-api";
 import { clipText, discordFetchMessage, revoltFetchUser } from "../util";
 import { smartReplace } from "smart-replace";
 import { metrics } from "../metrics";
+import { fetchEmojiList } from "../discord/bridgeEmojis";
 
 const RE_MENTION_USER = /<@[0-9A-HJ-KM-NP-TV-Z]{26}>/g;
 const RE_MENTION_CHANNEL = /<#[0-9A-HJ-KM-NP-TV-Z]{26}>/g;
+const RE_EMOJI = /:[^\s]+/g;
+
+const KNOWN_EMOJI_NAMES: string[] = [];
+
+fetchEmojiList()
+    .then(emojis => Object.keys(emojis).forEach(name => KNOWN_EMOJI_NAMES.push(name)))
+    .catch(e => console.error(e));
 
 client.on('message/delete', async id => {
     try {
@@ -252,7 +260,16 @@ async function renderMessageBody(message: string): Promise<string> {
         return discordChannel ? `<#${discordChannel.id}>` : `#${channel?.name || id}`;
     }, { cacheMatchResults: true, maxMatches: 10 });
 
-    // TODO: fetch emojis and upload them to Discord or smth
+    message = await smartReplace(message, RE_EMOJI, async (match) => {
+        const emojiName = match.replace(/(^:)|(:$)/g, '');
+
+        if (!KNOWN_EMOJI_NAMES.includes(emojiName)) return match;
+
+        const dbEmoji = await BRIDGED_EMOJIS.findOne({ name: emojiName });
+        if (!dbEmoji) return match;
+
+        return `<${dbEmoji.animated ? 'a' : ''}:${emojiName}:${dbEmoji.emojiid}>`;
+    }, { cacheMatchResults: true, maxMatches: 40 });
 
     return message;
 }

bridge/src/types/DiscordBridgedEmoji.ts

@@ -1,5 +1,7 @@
 export default class {
     name: string;
     emojiid: string;
+    server: string;
     animated: boolean;
+    originalFileUrl: string;
 }