From 98be31dd64289c9c07ea4ea6a8ddf6679e029d47 Mon Sep 17 00:00:00 2001 From: janderedev Date: Sun, 8 May 2022 12:26:58 +0200 Subject: [PATCH] various fixes - get rid of userscan - cannot fetch mutual servers from api anymore - minor fixes --- bot/package.json | 2 +- bot/src/bot/commands/admin/botadm.ts | 15 +- bot/src/bot/commands/configuration/botctl.ts | 35 ---- bot/src/bot/modules/api/servers.ts | 9 +- bot/src/bot/modules/command_handler.ts | 1 + bot/src/bot/modules/mod_logs.ts | 4 +- bot/src/bot/modules/user_scan.ts | 193 ------------------- bot/src/bot/util.ts | 9 + bot/src/index.ts | 3 - bot/src/struct/ScannedUser.ts | 14 -- bot/src/struct/ServerConfig.ts | 2 - bot/yarn.lock | 21 +- 12 files changed, 37 insertions(+), 271 deletions(-) delete mode 100644 bot/src/bot/modules/user_scan.ts delete mode 100644 bot/src/struct/ScannedUser.ts diff --git a/bot/package.json b/bot/package.json index 78fa40a..d67c0b3 100644 --- a/bot/package.json +++ b/bot/package.json @@ -13,7 +13,7 @@ "author": "", "license": "ISC", "dependencies": { - "@janderedev/revolt.js": "^6.0.0-rc.24-patch.1", + "@janderedev/revolt.js": "^6.0.0-patch.3", "@types/monk": "^6.0.0", "axios": "^0.22.0", "dayjs": "^1.10.7", diff --git a/bot/src/bot/commands/admin/botadm.ts b/bot/src/bot/commands/admin/botadm.ts index 03eda96..6186cc7 100644 --- a/bot/src/bot/commands/admin/botadm.ts +++ b/bot/src/bot/commands/admin/botadm.ts @@ -5,11 +5,10 @@ import { commands, DEFAULT_PREFIX, ownerIDs } from "../../modules/command_handle import child_process from 'child_process'; import fs from 'fs'; import path from 'path'; -import { wordlist } from "../../modules/user_scan"; import { User } from "@janderedev/revolt.js/dist/maps/Users"; import { adminBotLog } from "../../logging"; import CommandCategory from "../../../struct/commands/CommandCategory"; -import { parseUserOrId } from "../../util"; +import { getMutualServers, parseUserOrId } from "../../util"; const BLACKLIST_BAN_REASON = `This user is globally blacklisted and has been banned automatically. If you wish to opt out of the global blacklist, run '/botctl ignore_blacklist yes'.`; const BLACKLIST_MESSAGE = (username: string) => `\`@${username}\` has been banned automatically. Check the ban reason for more info.`; @@ -76,8 +75,7 @@ export default { + `Heartbeat: \`${client.heartbeat}\`\n` + `Ping: \`${client.websocket.ping ?? 'Unknown'}\`\n` + `### Bot configuration\n` - + `Owners: \`${ownerIDs.length}\` (${ownerIDs.join(', ')})\n` - + `Wordlist loaded: \`${wordlist ? `Yes (${wordlist.length} line${wordlist.length == 1 ? '' : 's'})` : 'No'}\`\n`; + + `Owners: \`${ownerIDs.length}\` (${ownerIDs.join(', ')})\n`; await message.reply(msg, false); break; @@ -166,11 +164,8 @@ export default { const msg = await message.reply(`User update stored.`); let bannedServers = 0; - const mutuals = await target.fetchMutual(); - for (const serverid of mutuals.servers) { - const server = client.servers.get(serverid); - if (!server) continue; - + const mutuals = getMutualServers(target); + for (const server of mutuals) { if (server.havePermission('BanMembers')) { const config = await dbs.SERVERS.findOne({ id: server._id }); if (config?.allowBlacklistedUsers) continue; @@ -188,7 +183,7 @@ export default { } } } catch(e) { - console.error(`Failed to ban in ${serverid}: ${e}`); + console.error(`Failed to ban in ${server._id}: ${e}`); } } } diff --git a/bot/src/bot/commands/configuration/botctl.ts b/bot/src/bot/commands/configuration/botctl.ts index d764eb0..265e351 100644 --- a/bot/src/bot/commands/configuration/botctl.ts +++ b/bot/src/bot/commands/configuration/botctl.ts @@ -1,14 +1,9 @@ -import { FindOneResult } from "monk"; import { dbs } from "../../.."; import CommandCategory from "../../../struct/commands/CommandCategory"; import SimpleCommand from "../../../struct/commands/SimpleCommand"; import MessageCommandContext from "../../../struct/MessageCommandContext"; -import ServerConfig from "../../../struct/ServerConfig"; -import { scanServer } from "../../modules/user_scan"; import { isBotManager, NO_MANAGER_MSG } from "../../util"; -let userscans: string[] = []; - export default { name: 'botctl', aliases: null, @@ -19,35 +14,6 @@ export default { let action = args.shift(); switch(action) { - case 'scan_userlist': - try { - let serverConf: FindOneResult = await dbs.SERVERS.findOne({ id: message.serverContext._id }); - - if (!serverConf?.enableUserScan) return message.reply(`User scanning is not enabled for this server.`); - if (userscans.includes(message.serverContext._id)) return message.reply(`There is already a scan running for this server.`); - userscans.push(message.serverContext._id); - - let msg = await message.reply(`Fetching users...`); - - let counter = 0; - - let onUserScan = async () => { - counter++; - if (counter % 10 == 0) await msg?.edit({ content: `Fetching users... ${counter}` }); - } - - let onDone = async () => { - msg?.edit({ content: `All done! (${counter} users fetched)` }); - userscans = userscans.filter(s => s != message.serverContext._id); - } - - await scanServer(message.serverContext._id, onUserScan, onDone); - } catch(e) { - message.reply(`An error occurred: ${e}`); - userscans = userscans.filter(s => s != message.serverContext._id); - } - break; - case 'ignore_blacklist': try { if (args[0] == 'yes') { @@ -68,7 +34,6 @@ export default { case undefined: case '': message.reply(`### Available subcommands\n` - + `- \`scan_userlist\` - If user scanning is enabled, this will scan the entire user list.\n` + `- \`ignore_blacklist\` - Ignore the bot's global blacklist.`); break default: diff --git a/bot/src/bot/modules/api/servers.ts b/bot/src/bot/modules/api/servers.ts index 6aab4d6..e771a89 100644 --- a/bot/src/bot/modules/api/servers.ts +++ b/bot/src/bot/modules/api/servers.ts @@ -1,6 +1,6 @@ import { User } from '@janderedev/revolt.js/dist/maps/Users'; import { client } from '../../..'; -import { getPermissionLevel, isBotManager } from '../../util'; +import { getMutualServers, getPermissionLevel } from '../../util'; import { wsEvents, WSResponse } from '../api_communication'; type ReqData = { user: string } @@ -15,20 +15,19 @@ wsEvents.on('req:getUserServers', async (data: ReqData, cb: (data: WSResponse) = return; } - const mutuals = await user.fetchMutual(); + const mutuals = getMutualServers(user); type ServerResponse = { id: string, perms: 0|1|2|3, name: string, iconURL?: string, bannerURL?: string } const promises: Promise[] = []; - for (const sid of mutuals.servers) { + for (const server of mutuals) { promises.push(new Promise(async (resolve, reject) => { try { - const server = client.servers.get(sid); if (!server) return reject('Server not found'); const perms = await getPermissionLevel(user, server); resolve({ - id: sid, + id: server._id, perms, name: server.name, bannerURL: server.generateBannerURL(), diff --git a/bot/src/bot/modules/command_handler.ts b/bot/src/bot/modules/command_handler.ts index 8016c97..29f2de3 100644 --- a/bot/src/bot/modules/command_handler.ts +++ b/bot/src/bot/modules/command_handler.ts @@ -138,6 +138,7 @@ let commands: SimpleCommand[]; try { await cmd.run(message, args); } catch(e) { + console.error(e); message.reply(`### An error has occurred:\n\`\`\`js\n${e}\n\`\`\``); } }); diff --git a/bot/src/bot/modules/mod_logs.ts b/bot/src/bot/modules/mod_logs.ts index f1842bf..0789e66 100644 --- a/bot/src/bot/modules/mod_logs.ts +++ b/bot/src/bot/modules/mod_logs.ts @@ -149,11 +149,11 @@ async function logModAction(type: 'warn'|'kick'|'ban'|'votekick', server: Server } -let fetchUsername = async (id: string) => { +let fetchUsername = async (id: string, fallbackText?: string) => { try { let u = client.users.get(id) || await client.users.fetch(id); return `@${u.username}`; - } catch(e) { return 'Unknown user' } + } catch(e) { return fallbackText || 'Unknown user' } } export { fetchUsername, logModAction } diff --git a/bot/src/bot/modules/user_scan.ts b/bot/src/bot/modules/user_scan.ts deleted file mode 100644 index 70bcf5d..0000000 --- a/bot/src/bot/modules/user_scan.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { client, dbs } from "../.."; -import fs from 'fs'; -import { FindOneResult } from "monk"; -import ScannedUser from "../../struct/ScannedUser"; -import { Member } from "@janderedev/revolt.js/dist/maps/Members"; -import ServerConfig from "../../struct/ServerConfig"; -import logger from "../logger"; -import { sendLogMessage } from "../util"; - -let { USERSCAN_WORDLIST_PATH } = process.env; - -let wordlist = USERSCAN_WORDLIST_PATH - ? fs.readFileSync(USERSCAN_WORDLIST_PATH, 'utf8') - .split('\n') - .map(word => minifyText(word)) - .filter(word => word.length > 0) - : null; - -if (wordlist) logger.info("Found word list; user scanning enabled"); - -let serverConfig: Map = new Map(); -let userScanTimeout: Map = new Map(); - -async function scanServer(id: string, userScanned: () => void, done: () => void) { - if (!wordlist) return; - let conf = await dbs.SERVERS.findOne({ id: id }); - serverConfig.set(id, conf as ServerConfig); - if (!conf?.enableUserScan) return; - - try { - logger.debug(`Scanning user list for ${id}`); - - let server = client.servers.get(id) || await client.servers.fetch(id); - let members = await server.fetchMembers(); // This can take multiple seconds, depending on the size of the server - - for (const member of members.members) { - if (!member.user?.bot && member._id.user != client.user?._id) { - userScanned(); - await scanUser(member); - } - } - - done(); - } catch(e) { console.error(e) } -} - -async function scanUser(member: Member) { - if (!wordlist) return; - - try { - let dbEntry: FindOneResult - = await dbs.SCANNED_USERS.findOne({ id: member._id.user, server: member.server?._id }); - let user = member.user || await client.users.fetch(member._id.user); - let profile = await user.fetchProfile(); - let report = false; - - if (dbEntry) { - if (dbEntry.approved) return; - if (dbEntry.lastLog > Date.now() - (1000 * 60 * 60 * 48)) return; - } - - for (const word of wordlist) { - for (const text of [ user?.username, member.nickname, profile.content, user.status?.text ]) { - if (text && minifyText(text).includes(word)) report = true; - } - } - - if (report) { - if (dbEntry) { - await dbs.SCANNED_USERS.update({ _id: dbEntry._id }, { - $set: { - lastLog: Date.now(), - lastLoggedProfile: { - username: user.username, - nickname: member.nickname || undefined, - profile: profile.content || undefined, - status: user.status?.text || undefined, - } - } - }); - } else { - await dbs.SCANNED_USERS.insert({ - approved: false, - id: user._id, - lastLog: Date.now(), - server: member.server!._id, - lastLoggedProfile: { - username: user.username, - nickname: member.nickname, - profile: profile.content, - status: user.status?.text, - } - } as ScannedUser); - } - - await logUser(member, profile); - } - } catch(e) { console.error(e) } -} - - -async function logUser(member: Member, profile: any) { // `Profile` type doesn't seem to be exported by revolt.js - try { - let conf = serverConfig.get(member.server!._id); - if (!conf || !conf.enableUserScan) return; - - logger.debug(`User ${member._id} matched word list; reporting`); - - if (conf.enableUserScan && conf.logs?.userScan) { - let bannerUrl = client.generateFileURL({ - _id: profile.background._id, - tag: profile.background.tag, - content_type: profile.background.content_type, - }, undefined, true); - let embedFields: { title: string, content: string, inline?: boolean }[] = []; - if (member.nickname) embedFields.push({ title: 'Nickname', content: member.nickname || 'None', inline: true }); - if (member.user?.status?.text) embedFields.push({ title: 'Status', content: member.user.status.text || 'None', inline: true }); - embedFields.push({ title: 'Profile', content: ((profile?.content || 'No about me text') as string).substring(0, 1000), inline: true }); - - sendLogMessage(conf.logs.userScan, { - title: 'Potentially suspicious user found', - description: `${member.user?.username ?? 'Unknown user'} | [${member._id.user}](/@${member._id.user}) | [Avatar](<${member.generateAvatarURL()}>)`, - color: '#ff9c11', - fields: embedFields, - image: bannerUrl ? { - type: 'BIG', - url: bannerUrl - } : undefined, - }); - - } - } catch(e) { console.error(e) } -} - -// Removes symbols from a text to make it easier to match against the wordlist -function minifyText(text: string) { - return text - .toLowerCase() - .replace(/\s_./g, ''); -} - -new Promise((res: (value: void) => void) => client.user ? res() : client.once('ready', res)).then(() => { - client.on('packet', async packet => { - if (!wordlist) return; - if (packet.type == 'UserUpdate') { - try { - let user = client.users.get(packet.id); - if (!user || user.bot || user._id == client.user?._id) return; - let mutual = await user.fetchMutual(); - - mutual.servers.forEach(async sid => { - let server = client.servers.get(sid); - if (!server) return; - - let conf = await dbs.SERVERS.findOne({ id: server._id }); - serverConfig.set(server._id, conf as ServerConfig); - - if (conf?.enableUserScan) { - let member = await server.fetchMember(packet.id); - let t = userScanTimeout.get(member._id.user); - if (t && t > (Date.now() - 10000)) return; - userScanTimeout.set(member._id.user, Date.now()); - scanUser(member); - } - }); - } catch(e) { console.error(e) } - } - }); - - client.on('member/join', async (member) => { - if (!wordlist) return; - - try { - let user = member.user || await client.users.fetch(member._id.user); - if (!user || user.bot || user._id == client.user?._id) return; - - let server = member.server || await client.servers.fetch(member._id.server); - if (!server) return; - - let conf: FindOneResult = await dbs.SERVERS.findOne({ id: server._id }); - serverConfig.set(server._id, conf as ServerConfig); - - if (conf?.enableUserScan) { - let t = userScanTimeout.get(member._id.user); - if (t && t > (Date.now() - 10000)) return; - userScanTimeout.set(member._id.user, Date.now()); - scanUser(member); - } - } catch(e) { console.error(e) } - }); -}); - -export { scanServer, USERSCAN_WORDLIST_PATH, wordlist }; diff --git a/bot/src/bot/util.ts b/bot/src/bot/util.ts index 445e7d9..a13c5a9 100644 --- a/bot/src/bot/util.ts +++ b/bot/src/bot/util.ts @@ -346,6 +346,14 @@ function dedupeArray(...arrays: T[][]): T[] { return found; } +function getMutualServers(user: User) { + const servers: Server[] = []; + for (const member of client.members) { + if (member[1]._id.user == user._id && member[1].server) servers.push(member[1].server); + } + return servers; +} + const awaitClient = () => new Promise(async resolve => { if (!client.user) client.once('ready', () => resolve()); else resolve(); @@ -369,6 +377,7 @@ export { embed, dedupeArray, awaitClient, + getMutualServers, EmbedColor, NO_MANAGER_MSG, ULID_REGEX, diff --git a/bot/src/index.ts b/bot/src/index.ts index fc0fd7d..5dacbb1 100644 --- a/bot/src/index.ts +++ b/bot/src/index.ts @@ -10,7 +10,6 @@ import Infraction from './struct/antispam/Infraction'; import PendingLogin from './struct/PendingLogin'; import TempBan from './struct/TempBan'; import { VoteEntry } from './bot/commands/moderation/votekick'; -import ScannedUser from './struct/ScannedUser'; import BridgeRequest from './struct/BridgeRequest'; import BridgeConfig from './struct/BridgeConfig'; import BridgedMessage from './struct/BridgedMessage'; @@ -36,7 +35,6 @@ const dbs = { SESSIONS: db.get('sessions'), TEMPBANS: db.get('tempbans'), VOTEKICKS: db.get('votekicks'), - SCANNED_USERS: db.get('scanned_users'), BRIDGE_CONFIG: db.get('bridge_config'), BRIDGED_MESSAGES: db.get('bridged_messages'), BRIDGE_REQUESTS: db.get('bridge_requests'), @@ -63,7 +61,6 @@ logger.info(`\ import('./bot/modules/mod_logs'); import('./bot/modules/event_handler'); import('./bot/modules/tempbans'); - import('./bot/modules/user_scan'); import('./bot/modules/api_communication'); import('./bot/modules/metrics'); import('./bot/modules/bot_status'); diff --git a/bot/src/struct/ScannedUser.ts b/bot/src/struct/ScannedUser.ts deleted file mode 100644 index d0bef1f..0000000 --- a/bot/src/struct/ScannedUser.ts +++ /dev/null @@ -1,14 +0,0 @@ -class ScannedUser { - id: string; - server: string; - lastLog: number; - approved: boolean = false; - lastLoggedProfile?: { - username: string; - nickname?: string; - status?: string; - profile?: string; - } -} - -export default ScannedUser; diff --git a/bot/src/struct/ServerConfig.ts b/bot/src/struct/ServerConfig.ts index 601fad5..01b890c 100644 --- a/bot/src/struct/ServerConfig.ts +++ b/bot/src/struct/ServerConfig.ts @@ -23,9 +23,7 @@ class ServerConfig { logs?: { messageUpdate?: LogConfig, // Message edited or deleted modAction?: LogConfig, // User warned, kicked or banned - userScan?: LogConfig // User profile matched word list }; - enableUserScan?: boolean; allowBlacklistedUsers?: boolean; // Whether the server explicitly allows users that are globally blacklisted } diff --git a/bot/yarn.lock b/bot/yarn.lock index 643c35c..b8d0645 100644 --- a/bot/yarn.lock +++ b/bot/yarn.lock @@ -47,10 +47,10 @@ axios "^0.26.1" openapi-typescript "^5.2.0" -"@janderedev/revolt.js@^6.0.0-rc.24-patch.1": - version "6.0.0-rc.24-patch.1" - resolved "https://registry.yarnpkg.com/@janderedev/revolt.js/-/revolt.js-6.0.0-rc.24-patch.1.tgz#58a9762fc887db16d34874d1e8a9c032112f6c83" - integrity sha512-+1Q94zNWj+OhRrBa4kVyFaG1U7OC2XwOrXqIk6Jiult8krH2lqKOjSqsxaKKITQLJ+8xIpM2TmMWVoXY52OdWQ== +"@janderedev/revolt.js@^6.0.0-patch.3": + version "6.0.0-patch.3" + resolved "https://registry.yarnpkg.com/@janderedev/revolt.js/-/revolt.js-6.0.0-patch.3.tgz#7d9ad66e5a0d54fb2f5f0f8887cbd1382c69d301" + integrity sha512-aZ1vubm8+l10lTy5HwO3vAc7E2/bm3+hfFhNqU7l+9QKecIAm6f45p7RNC19afSYChIZiyhtGPtWTuVJ9pa0RA== dependencies: "@insertish/exponential-backoff" "3.1.0-patch.2" "@insertish/isomorphic-ws" "^4.0.1" @@ -61,7 +61,7 @@ lodash.isequal "^4.5.0" long "^5.2.0" mobx "^6.3.2" - revolt-api "0.5.3-rc.15" + revolt-api "0.5.3" ulid "^2.3.0" ws "^8.2.2" @@ -600,7 +600,16 @@ require-at@^1.0.6: resolved "https://registry.yarnpkg.com/require-at/-/require-at-1.0.6.tgz#9eb7e3c5e00727f5a4744070a7f560d4de4f6e6a" integrity sha512-7i1auJbMUrXEAZCOQ0VNJgmcT2VOKPRl2YGJwgpHpC9CE91Mv4/4UYIUm4chGJaI381ZDq1JUicFii64Hapd8g== -revolt-api@0.5.3-rc.15, revolt-api@^0.5.3-rc.15: +revolt-api@0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/revolt-api/-/revolt-api-0.5.3.tgz#e0ec2dcf812ea4338247b2eb77d67fc731d71b8a" + integrity sha512-hYdyStQiDZFvD+0dlf6SgQSiOk+JiEmQo0qIQHaqYRtrFN6FQBbGVNaiv7b5LzHHMPq7vks6ZVVA7hSNpcwlkA== + dependencies: + "@insertish/oapi" "0.1.15" + axios "^0.26.1" + lodash.defaultsdeep "^4.6.1" + +revolt-api@^0.5.3-rc.15: version "0.5.3-rc.15" resolved "https://registry.yarnpkg.com/revolt-api/-/revolt-api-0.5.3-rc.15.tgz#abd08dd8109d0ca31be118461eabbeb6c3b7792e" integrity sha512-MYin3U+KoObNkILPf2cz+FPperynExkUu7CjzurMJCRvBncpnhb2czvWDvnhLDKBHlpo8W597xNqzQnaklV4ug==