From e0bc136a372b06486f338e058e789797d9be31e5 Mon Sep 17 00:00:00 2001 From: janderedev Date: Sun, 1 May 2022 14:34:49 +0200 Subject: [PATCH] bridging emojis from revolt to discord --- bridge/src/discord/bridgeEmojis.ts | 131 ++++++++++++++++++++++++ bridge/src/discord/client.ts | 1 + bridge/src/index.ts | 4 +- bridge/src/revolt/events.ts | 21 +++- bridge/src/types/DiscordBridgedEmoji.ts | 2 + 5 files changed, 156 insertions(+), 3 deletions(-) diff --git a/bridge/src/discord/bridgeEmojis.ts b/bridge/src/discord/bridgeEmojis.ts index 42d95ad..4183f52 100644 --- a/bridge/src/discord/bridgeEmojis.ts +++ b/bridge/src/discord/bridgeEmojis.ts @@ -1,7 +1,12 @@ import axios from "axios"; +import { GuildEmoji } from "discord.js"; import JSON5 from 'json5'; +import { BRIDGED_EMOJIS, logger } from ".."; +import { client } from "./client"; const EMOJI_DICT_URL = 'https://raw.githubusercontent.com/revoltchat/revite/master/src/assets/emojis.ts'; +const EMOJI_URL_BASE = 'https://dl.insrt.uk/projects/revolt/emotes/'; +const EMOJI_SERVERS = process.env.EMOJI_SERVERS?.split(',') || []; async function fetchEmojiList(): Promise> { const file: string = (await axios.get(EMOJI_DICT_URL)).data; @@ -10,3 +15,129 @@ async function fetchEmojiList(): Promise> { return JSON5.parse(file.substring(start, end).trim()); } + +const emojiUpdate = async () => { + try { + if (!EMOJI_SERVERS.length) return logger.info('$EMOJI_SERVERS not set, not bridging emojis.'); + + if (!client.readyAt) await new Promise(r => client.once('ready', r)); + logger.info('Updating bridged emojis. Due to Discord rate limits, this can take a few hours to complete.'); + + const emojis = await fetchEmojiList(); + logger.info(`Downloaded emoji list: ${Object.keys(emojis).length} emojis.`); + + const servers = await Promise.all(EMOJI_SERVERS.map(id => client.guilds.fetch(id))); + await Promise.all(servers.map(server => server.emojis.fetch())); // Make sure all emojis are cached + + const findFreeServer = (animated: boolean) => servers.find( + server => server.emojis.cache + .filter(e => e.animated == animated) + .size < 50 + ); + + // Remove unknown emojis from servers + for (const server of servers) { + for (const emoji of server.emojis.cache) { + const dbEmoji = await BRIDGED_EMOJIS.findOne({ + emojiid: emoji[1].id, + }); + + if (!dbEmoji) { + try { + logger.info('Found unknown emoji; deleting.'); + await emoji[1].delete('Unknown emoji'); + } catch(e) { + logger.warn('Failed to delete emoji: ' + e); + } + } + } + } + + for (const emoji of Object.entries(emojis)) { + const dbEmoji = await BRIDGED_EMOJIS.findOne({ + $or: [ + { name: emoji[0] }, + { originalFileUrl: emoji[1] }, + ], + }); + + if (!dbEmoji) { + // Upload to Discord + logger.debug('Uploading emoji: ' + emoji[1]); + + const fileurl = EMOJI_URL_BASE + emoji[1].replace('custom:', ''); + const server = findFreeServer(emoji[1].endsWith('.gif')); + + if (!server) { + logger.warn('Could not find a server with free emoji slots for ' + emoji[1]); + continue; + } + + let e: GuildEmoji; + try { + e = await server.emojis.create(fileurl, emoji[0], { reason: 'Bridged Emoji' }); + } catch(e) { + logger.warn(emoji[0] + ': Failed to upload emoji: ' + e); + continue; + } + + await BRIDGED_EMOJIS.insert({ + animated: e.animated || false, + emojiid: e.id, + name: emoji[0], + originalFileUrl: fileurl, + server: e.guild.id, + }); + } + else { + // Double check if emoji exists + let exists = false; + for (const server of servers) { + if (server.emojis.cache.find(e => e.id == dbEmoji.emojiid)) { + exists = true; + break; + } + } + + if (!exists) { + logger.info(`Emoji ${emoji[0]} does not exist; reuploading.`); + await BRIDGED_EMOJIS.remove({ emojiid: dbEmoji.emojiid }); + + const fileurl = EMOJI_URL_BASE + emoji[1].replace('custom:', ''); + const server = findFreeServer(emoji[1].endsWith('.gif')); + + if (!server) { + logger.warn('Could not find a server with free emoji slots for ' + emoji[1]); + continue; + } + + let e: GuildEmoji; + try { + e = await server.emojis.create(fileurl, emoji[0], { reason: 'Bridged Emoji' }); + } catch(e) { + logger.warn(emoji[0] + ': Failed to upload emoji: ' + e); + continue; + } + + await BRIDGED_EMOJIS.insert({ + animated: e.animated || false, + emojiid: e.id, + name: emoji[0], + originalFileUrl: fileurl, + server: e.guild.id, + }); + } + } + }; + + logger.done('Emoji update finished.'); + } catch(e) { + logger.error('Updating bridged emojis failed'); + console.error(e); + } +}; + +emojiUpdate(); +setInterval(emojiUpdate, 1000 * 60 * 60 * 6); // Every 6h + +export { fetchEmojiList } diff --git a/bridge/src/discord/client.ts b/bridge/src/discord/client.ts index 8751ed7..2709b1d 100644 --- a/bridge/src/discord/client.ts +++ b/bridge/src/discord/client.ts @@ -24,5 +24,6 @@ const login = () => new Promise((resolve: (value: Discord.Client) => void) => { import('./events'); import('./commands'); +import('./bridgeEmojis'); export { client, login } diff --git a/bridge/src/index.ts b/bridge/src/index.ts index 7edd7f8..b2b5a56 100644 --- a/bridge/src/index.ts +++ b/bridge/src/index.ts @@ -7,6 +7,7 @@ import { ICollection } from 'monk'; import BridgeConfig from './types/BridgeConfig'; import BridgedMessage from './types/BridgedMessage'; import BridgeRequest from './types/BridgeRequest'; +import DiscordBridgedEmoji from './types/DiscordBridgedEmoji'; config(); @@ -15,6 +16,7 @@ const db = getDb(); const BRIDGED_MESSAGES: ICollection = db.get('bridged_messages'); const BRIDGE_CONFIG: ICollection = db.get('bridge_config'); const BRIDGE_REQUESTS: ICollection = db.get('bridge_requests'); +const BRIDGED_EMOJIS: ICollection = db.get('bridged_emojis'); for (const v of [ 'REVOLT_TOKEN', 'DISCORD_TOKEN', 'DB_STRING' ]) { if (!process.env[v]) { @@ -31,4 +33,4 @@ for (const v of [ 'REVOLT_TOKEN', 'DISCORD_TOKEN', 'DB_STRING' ]) { ]); })(); -export { logger, db, BRIDGED_MESSAGES, BRIDGE_CONFIG, BRIDGE_REQUESTS } +export { logger, db, BRIDGED_MESSAGES, BRIDGE_CONFIG, BRIDGE_REQUESTS, BRIDGED_EMOJIS } diff --git a/bridge/src/revolt/events.ts b/bridge/src/revolt/events.ts index 5d30d99..2db9acc 100644 --- a/bridge/src/revolt/events.ts +++ b/bridge/src/revolt/events.ts @@ -1,4 +1,4 @@ -import { BRIDGED_MESSAGES, BRIDGE_CONFIG, logger } from ".."; +import { BRIDGED_EMOJIS, BRIDGED_MESSAGES, BRIDGE_CONFIG, logger } from ".."; import { AUTUMN_URL, client } from "./client"; import { client as discordClient } from "../discord/client"; import { MessageEmbed, MessagePayload, TextChannel, WebhookClient, WebhookMessageOptions } from "discord.js"; @@ -7,9 +7,17 @@ import { SendableEmbed } from "revolt-api"; import { clipText, discordFetchMessage, revoltFetchUser } from "../util"; import { smartReplace } from "smart-replace"; import { metrics } from "../metrics"; +import { fetchEmojiList } from "../discord/bridgeEmojis"; const RE_MENTION_USER = /<@[0-9A-HJ-KM-NP-TV-Z]{26}>/g; const RE_MENTION_CHANNEL = /<#[0-9A-HJ-KM-NP-TV-Z]{26}>/g; +const RE_EMOJI = /:[^\s]+/g; + +const KNOWN_EMOJI_NAMES: string[] = []; + +fetchEmojiList() + .then(emojis => Object.keys(emojis).forEach(name => KNOWN_EMOJI_NAMES.push(name))) + .catch(e => console.error(e)); client.on('message/delete', async id => { try { @@ -252,7 +260,16 @@ async function renderMessageBody(message: string): Promise { return discordChannel ? `<#${discordChannel.id}>` : `#${channel?.name || id}`; }, { cacheMatchResults: true, maxMatches: 10 }); - // TODO: fetch emojis and upload them to Discord or smth + message = await smartReplace(message, RE_EMOJI, async (match) => { + const emojiName = match.replace(/(^:)|(:$)/g, ''); + + if (!KNOWN_EMOJI_NAMES.includes(emojiName)) return match; + + const dbEmoji = await BRIDGED_EMOJIS.findOne({ name: emojiName }); + if (!dbEmoji) return match; + + return `<${dbEmoji.animated ? 'a' : ''}:${emojiName}:${dbEmoji.emojiid}>`; + }, { cacheMatchResults: true, maxMatches: 40 }); return message; } diff --git a/bridge/src/types/DiscordBridgedEmoji.ts b/bridge/src/types/DiscordBridgedEmoji.ts index 02aab63..e80ce2b 100644 --- a/bridge/src/types/DiscordBridgedEmoji.ts +++ b/bridge/src/types/DiscordBridgedEmoji.ts @@ -1,5 +1,7 @@ export default class { name: string; emojiid: string; + server: string; animated: boolean; + originalFileUrl: string; }