various fixes

- get rid of userscan - cannot fetch mutual servers from api anymore - minor fixes
2022-05-08 12:26:58 +02:00 · 2022-05-08 12:26:58 +02:00 · 98be31dd64
commit 98be31dd64
parent e27d9f6f2b
12 changed files with 37 additions and 271 deletions
--- a/bot/package.json
+++ b/bot/package.json
@ -13,7 +13,7 @@
  "author": "",
  "license": "ISC",
  "dependencies": {
-    "@janderedev/revolt.js": "^6.0.0-rc.24-patch.1",
+    "@janderedev/revolt.js": "^6.0.0-patch.3",
    "@types/monk": "^6.0.0",
    "axios": "^0.22.0",
    "dayjs": "^1.10.7",
--- a/bot/src/bot/commands/admin/botadm.ts
+++ b/bot/src/bot/commands/admin/botadm.ts
@ -5,11 +5,10 @@ import { commands, DEFAULT_PREFIX, ownerIDs } from "../../modules/command_handle
 import child_process from 'child_process';
 import fs from 'fs';
 import path from 'path';
-import { wordlist } from "../../modules/user_scan";
 import { User } from "@janderedev/revolt.js/dist/maps/Users";
 import { adminBotLog } from "../../logging";
 import CommandCategory from "../../../struct/commands/CommandCategory";
-import { parseUserOrId } from "../../util";
+import { getMutualServers, parseUserOrId } from "../../util";

 const BLACKLIST_BAN_REASON = `This user is globally blacklisted and has been banned automatically. If you wish to opt out of the global blacklist, run '/botctl ignore_blacklist yes'.`;
 const BLACKLIST_MESSAGE = (username: string) => `\`@${username}\` has been banned automatically. Check the ban reason for more info.`;
@ -76,8 +75,7 @@ export default {
                            + `Heartbeat: \`${client.heartbeat}\`\n`
                            + `Ping: \`${client.websocket.ping ?? 'Unknown'}\`\n`
                            + `### Bot configuration\n`
-                            + `Owners: \`${ownerIDs.length}\` (${ownerIDs.join(', ')})\n`
-                            + `Wordlist loaded: \`${wordlist ? `Yes (${wordlist.length} line${wordlist.length == 1 ? '' : 's'})` : 'No'}\`\n`;
+                            + `Owners: \`${ownerIDs.length}\` (${ownerIDs.join(', ')})\n`;

                    await message.reply(msg, false);
                    break;
@ -166,11 +164,8 @@ export default {
                            const msg = await message.reply(`User update stored.`);
                            let bannedServers = 0;

-                            const mutuals = await target.fetchMutual();
-                            for (const serverid of mutuals.servers) {
-                                const server = client.servers.get(serverid);
-                                if (!server) continue;
-
+                            const mutuals = getMutualServers(target);
+                            for (const server of mutuals) {
                                if (server.havePermission('BanMembers')) {
                                    const config = await dbs.SERVERS.findOne({ id: server._id });
                                    if (config?.allowBlacklistedUsers) continue;
@ -188,7 +183,7 @@ export default {
                                            }
                                        }
                                    } catch(e) {
-                                        console.error(`Failed to ban in ${serverid}: ${e}`);
+                                        console.error(`Failed to ban in ${server._id}: ${e}`);
                                    }
                                }
                            }
--- a/bot/src/bot/commands/configuration/botctl.ts
+++ b/bot/src/bot/commands/configuration/botctl.ts
@ -1,14 +1,9 @@
-import { FindOneResult } from "monk";
 import { dbs } from "../../..";
 import CommandCategory from "../../../struct/commands/CommandCategory";
 import SimpleCommand from "../../../struct/commands/SimpleCommand";
 import MessageCommandContext from "../../../struct/MessageCommandContext";
-import ServerConfig from "../../../struct/ServerConfig";
-import { scanServer } from "../../modules/user_scan";
 import { isBotManager, NO_MANAGER_MSG } from "../../util";

-let userscans: string[] = [];
-
 export default {
    name: 'botctl',
    aliases: null,
@ -19,35 +14,6 @@ export default {

        let action = args.shift();
        switch(action) {
-            case 'scan_userlist':
-                try {
-                    let serverConf: FindOneResult<ServerConfig> = await dbs.SERVERS.findOne({ id: message.serverContext._id });
-
-                    if (!serverConf?.enableUserScan) return message.reply(`User scanning is not enabled for this server.`);
-                    if (userscans.includes(message.serverContext._id)) return message.reply(`There is already a scan running for this server.`);
-                    userscans.push(message.serverContext._id);
-
-                    let msg = await message.reply(`Fetching users...`);
-
-                    let counter = 0;
-
-                    let onUserScan = async () => {
-                        counter++;
-                        if (counter % 10 == 0) await msg?.edit({ content: `Fetching users... ${counter}` });
-                    }
-
-                    let onDone = async () => {
-                        msg?.edit({ content: `All done! (${counter} users fetched)` });
-                        userscans = userscans.filter(s => s != message.serverContext._id);
-                    }
-
-                    await scanServer(message.serverContext._id, onUserScan, onDone);
-                } catch(e) {
-                    message.reply(`An error occurred: ${e}`);
-                    userscans = userscans.filter(s => s != message.serverContext._id);
-                }
-            break;
-
            case 'ignore_blacklist':
                try {
                    if (args[0] == 'yes') {
@ -68,7 +34,6 @@ export default {
            case undefined:
            case '':
                message.reply(`### Available subcommands\n`
-                            + `- \`scan_userlist\` - If user scanning is enabled, this will scan the entire user list.\n`
                            + `- \`ignore_blacklist\` - Ignore the bot's global blacklist.`);
            break
            default:
--- a/bot/src/bot/modules/api/servers.ts
+++ b/bot/src/bot/modules/api/servers.ts
@ -1,6 +1,6 @@
 import { User } from '@janderedev/revolt.js/dist/maps/Users';
 import { client } from '../../..';
-import { getPermissionLevel, isBotManager } from '../../util';
+import { getMutualServers, getPermissionLevel } from '../../util';
 import { wsEvents, WSResponse } from '../api_communication';

 type ReqData = { user: string }
@ -15,20 +15,19 @@ wsEvents.on('req:getUserServers', async (data: ReqData, cb: (data: WSResponse) =
            return;
        }

-        const mutuals = await user.fetchMutual();
+        const mutuals = getMutualServers(user);

        type ServerResponse = { id: string, perms: 0|1|2|3, name: string, iconURL?: string, bannerURL?: string }

        const promises: Promise<ServerResponse>[] = [];

-        for (const sid of mutuals.servers) {
+        for (const server of mutuals) {
            promises.push(new Promise(async (resolve, reject) => {
                try {
-                    const server = client.servers.get(sid);
                    if (!server) return reject('Server not found');
                    const perms = await getPermissionLevel(user, server);
                    resolve({
-                        id: sid,
+                        id: server._id,
                        perms,
                        name: server.name,
                        bannerURL: server.generateBannerURL(),
--- a/bot/src/bot/modules/command_handler.ts
+++ b/bot/src/bot/modules/command_handler.ts
@ -138,6 +138,7 @@ let commands: SimpleCommand[];
        try {
            await cmd.run(message, args);
        } catch(e) {
+            console.error(e);
            message.reply(`### An error has occurred:\n\`\`\`js\n${e}\n\`\`\``);
        }
    });
--- a/bot/src/bot/modules/mod_logs.ts
+++ b/bot/src/bot/modules/mod_logs.ts
@ -149,11 +149,11 @@ async function logModAction(type: 'warn'|'kick'|'ban'|'votekick', server: Server
 }


-let fetchUsername = async (id: string) => {
+let fetchUsername = async (id: string, fallbackText?: string) => {
    try {
        let u = client.users.get(id) || await client.users.fetch(id);
        return `@${u.username}`;
-    } catch(e) { return 'Unknown user' }
+    } catch(e) { return fallbackText || 'Unknown user' }
 }

 export { fetchUsername, logModAction }
--- a/bot/src/bot/modules/user_scan.ts
+++ b/bot/src/bot/modules/user_scan.ts
@ -1,193 +0,0 @@
-import { client, dbs } from "../..";
-import fs from 'fs';
-import { FindOneResult } from "monk";
-import ScannedUser from "../../struct/ScannedUser";
-import { Member } from "@janderedev/revolt.js/dist/maps/Members";
-import ServerConfig from "../../struct/ServerConfig";
-import logger from "../logger";
-import { sendLogMessage } from "../util";
-
-let { USERSCAN_WORDLIST_PATH } = process.env;
-
-let wordlist = USERSCAN_WORDLIST_PATH
-    ? fs.readFileSync(USERSCAN_WORDLIST_PATH, 'utf8')
-        .split('\n')
-        .map(word => minifyText(word))
-        .filter(word => word.length > 0)
-    : null;
-
-if (wordlist) logger.info("Found word list; user scanning enabled");
-
-let serverConfig: Map<string, ServerConfig> = new Map();
-let userScanTimeout: Map<string, number> = new Map();
-
-async function scanServer(id: string, userScanned: () => void, done: () => void) {
-    if (!wordlist) return;
-    let conf = await dbs.SERVERS.findOne({ id: id });
-    serverConfig.set(id, conf as ServerConfig);
-    if (!conf?.enableUserScan) return;
-
-    try {
-        logger.debug(`Scanning user list for ${id}`);
-
-        let server = client.servers.get(id) || await client.servers.fetch(id);
-        let members = await server.fetchMembers(); // This can take multiple seconds, depending on the size of the server
-
-        for (const member of members.members) {
-            if (!member.user?.bot && member._id.user != client.user?._id) {
-                userScanned();
-                await scanUser(member);
-            }
-        }
-
-        done();
-    } catch(e) { console.error(e) }
-}
-
-async function scanUser(member: Member) {
-    if (!wordlist) return;
-
-    try {
-        let dbEntry: FindOneResult<ScannedUser|undefined>
-            = await dbs.SCANNED_USERS.findOne({ id: member._id.user, server: member.server?._id });
-        let user = member.user || await client.users.fetch(member._id.user);
-        let profile = await user.fetchProfile();
-        let report = false;
-
-        if (dbEntry) {
-            if (dbEntry.approved) return;
-            if (dbEntry.lastLog > Date.now() - (1000 * 60 * 60 * 48)) return;
-        }
-
-        for (const word of wordlist) {
-            for (const text of [ user?.username, member.nickname, profile.content, user.status?.text ]) {
-                if (text && minifyText(text).includes(word)) report = true;
-            }
-        }
-
-        if (report) {
-            if (dbEntry) {
-                await dbs.SCANNED_USERS.update({ _id: dbEntry._id }, {
-                    $set: {
-                        lastLog: Date.now(),
-                        lastLoggedProfile: {
-                            username: user.username,
-                            nickname: member.nickname || undefined,
-                            profile: profile.content || undefined,
-                            status: user.status?.text || undefined,
-                        }
-                    }
-                });
-            } else {
-                await dbs.SCANNED_USERS.insert({
-                    approved: false,
-                    id: user._id,
-                    lastLog: Date.now(),
-                    server: member.server!._id,
-                    lastLoggedProfile: {
-                        username: user.username,
-                        nickname: member.nickname,
-                        profile: profile.content,
-                        status: user.status?.text,
-                    }
-                } as ScannedUser);
-            }
-
-            await logUser(member, profile);
-        }
-    } catch(e) { console.error(e) }
-}
-
-
-async function logUser(member: Member, profile: any) { // `Profile` type doesn't seem to be exported by revolt.js
-    try {
-        let conf = serverConfig.get(member.server!._id);
-        if (!conf || !conf.enableUserScan) return;
-
-        logger.debug(`User ${member._id} matched word list; reporting`);
-
-        if (conf.enableUserScan && conf.logs?.userScan) {
-            let bannerUrl = client.generateFileURL({
-                _id: profile.background._id,
-                tag: profile.background.tag,
-                content_type: profile.background.content_type,
-            }, undefined, true);
-            let embedFields: { title: string, content: string, inline?: boolean }[] = [];
-            if (member.nickname) embedFields.push({ title: 'Nickname', content: member.nickname || 'None', inline: true });
-            if (member.user?.status?.text) embedFields.push({ title: 'Status', content: member.user.status.text || 'None', inline: true });
-            embedFields.push({ title: 'Profile', content: ((profile?.content || 'No about me text') as string).substring(0, 1000), inline: true });
-
-            sendLogMessage(conf.logs.userScan, {
-                title: 'Potentially suspicious user found',
-                description: `${member.user?.username ?? 'Unknown user'} | [${member._id.user}](/@${member._id.user}) | [Avatar](<${member.generateAvatarURL()}>)`,
-                color: '#ff9c11',
-                fields: embedFields,
-                image: bannerUrl ? {
-                    type: 'BIG',
-                    url: bannerUrl
-                } : undefined,
-            });
-
-        }
-    } catch(e) { console.error(e) }
-}
-
-// Removes symbols from a text to make it easier to match against the wordlist
-function minifyText(text: string) {
-    return text
-        .toLowerCase()
-        .replace(/\s_./g, '');
-}
-
-new Promise((res: (value: void) => void) => client.user ? res() : client.once('ready', res)).then(() => {
-    client.on('packet', async packet => {
-        if (!wordlist) return;
-        if (packet.type == 'UserUpdate') {
-            try {
-                let user = client.users.get(packet.id);
-                if (!user || user.bot || user._id == client.user?._id) return;
-                let mutual = await user.fetchMutual();
-
-                mutual.servers.forEach(async sid => {
-                    let server = client.servers.get(sid);
-                    if (!server) return;
-
-                    let conf = await dbs.SERVERS.findOne({ id: server._id });
-                    serverConfig.set(server._id, conf as ServerConfig);
-
-                    if (conf?.enableUserScan) {
-                        let member = await server.fetchMember(packet.id);
-                        let t = userScanTimeout.get(member._id.user);
-                        if (t && t > (Date.now() - 10000)) return;
-                        userScanTimeout.set(member._id.user, Date.now());
-                        scanUser(member);
-                    }
-                });
-            } catch(e) { console.error(e) }
-        }
-    });
-
-    client.on('member/join', async (member) => {
-        if (!wordlist) return;
-
-        try {
-            let user = member.user || await client.users.fetch(member._id.user);
-            if (!user || user.bot || user._id == client.user?._id) return;
-
-            let server = member.server || await client.servers.fetch(member._id.server);
-            if (!server) return;
-
-            let conf: FindOneResult<ServerConfig> = await dbs.SERVERS.findOne({ id: server._id });
-            serverConfig.set(server._id, conf as ServerConfig);
-
-            if (conf?.enableUserScan) {
-                let t = userScanTimeout.get(member._id.user);
-                if (t && t > (Date.now() - 10000)) return;
-                userScanTimeout.set(member._id.user, Date.now());
-                scanUser(member);
-            }
-        } catch(e) { console.error(e) }
-    });
-});
-
-export { scanServer, USERSCAN_WORDLIST_PATH, wordlist };
--- a/bot/src/bot/util.ts
+++ b/bot/src/bot/util.ts
@ -346,6 +346,14 @@ function dedupeArray<T>(...arrays: T[][]): T[] {
    return found;
 }

+function getMutualServers(user: User) {
+    const servers: Server[] = [];
+    for (const member of client.members) {
+        if (member[1]._id.user == user._id && member[1].server) servers.push(member[1].server);
+    }
+    return servers;
+}
+
 const awaitClient = () => new Promise<void>(async resolve => {
    if (!client.user) client.once('ready', () => resolve());
    else resolve();
@ -369,6 +377,7 @@ export {
    embed,
    dedupeArray,
    awaitClient,
+    getMutualServers,
    EmbedColor,
    NO_MANAGER_MSG,
    ULID_REGEX,
--- a/bot/src/index.ts
+++ b/bot/src/index.ts
@ -10,7 +10,6 @@ import Infraction from './struct/antispam/Infraction';
 import PendingLogin from './struct/PendingLogin';
 import TempBan from './struct/TempBan';
 import { VoteEntry } from './bot/commands/moderation/votekick';
-import ScannedUser from './struct/ScannedUser';
 import BridgeRequest from './struct/BridgeRequest';
 import BridgeConfig from './struct/BridgeConfig';
 import BridgedMessage from './struct/BridgedMessage';
@ -36,7 +35,6 @@ const dbs = {
    SESSIONS: db.get('sessions'),
    TEMPBANS: db.get<TempBan>('tempbans'),
    VOTEKICKS: db.get<VoteEntry>('votekicks'),
-    SCANNED_USERS: db.get<ScannedUser>('scanned_users'),
    BRIDGE_CONFIG: db.get<BridgeConfig>('bridge_config'),
    BRIDGED_MESSAGES: db.get<BridgedMessage>('bridged_messages'),
    BRIDGE_REQUESTS: db.get<BridgeRequest>('bridge_requests'),
@ -63,7 +61,6 @@ logger.info(`\
    import('./bot/modules/mod_logs');
    import('./bot/modules/event_handler');
    import('./bot/modules/tempbans');
-    import('./bot/modules/user_scan');
    import('./bot/modules/api_communication');
    import('./bot/modules/metrics');
    import('./bot/modules/bot_status');
--- a/bot/src/struct/ScannedUser.ts
+++ b/bot/src/struct/ScannedUser.ts
@ -1,14 +0,0 @@
-class ScannedUser {
-    id: string;
-    server: string;
-    lastLog: number;
-    approved: boolean = false;
-    lastLoggedProfile?: {
-        username: string;
-        nickname?: string;
-        status?: string;
-        profile?: string;
-    }
-}
-
-export default ScannedUser;
--- a/bot/src/struct/ServerConfig.ts
+++ b/bot/src/struct/ServerConfig.ts
@ -23,9 +23,7 @@ class ServerConfig {
    logs?: {
        messageUpdate?: LogConfig,  // Message edited or deleted
        modAction?: LogConfig,      // User warned, kicked or banned
-        userScan?: LogConfig        // User profile matched word list
    };
-    enableUserScan?: boolean;
    allowBlacklistedUsers?: boolean; // Whether the server explicitly allows users that are globally blacklisted
 }

--- a/bot/yarn.lock
+++ b/bot/yarn.lock
@ -47,10 +47,10 @@
    axios "^0.26.1"
    openapi-typescript "^5.2.0"

-"@janderedev/revolt.js@^6.0.0-rc.24-patch.1":
-  version "6.0.0-rc.24-patch.1"
-  resolved "https://registry.yarnpkg.com/@janderedev/revolt.js/-/revolt.js-6.0.0-rc.24-patch.1.tgz#58a9762fc887db16d34874d1e8a9c032112f6c83"
-  integrity sha512-+1Q94zNWj+OhRrBa4kVyFaG1U7OC2XwOrXqIk6Jiult8krH2lqKOjSqsxaKKITQLJ+8xIpM2TmMWVoXY52OdWQ==
+"@janderedev/revolt.js@^6.0.0-patch.3":
+  version "6.0.0-patch.3"
+  resolved "https://registry.yarnpkg.com/@janderedev/revolt.js/-/revolt.js-6.0.0-patch.3.tgz#7d9ad66e5a0d54fb2f5f0f8887cbd1382c69d301"
+  integrity sha512-aZ1vubm8+l10lTy5HwO3vAc7E2/bm3+hfFhNqU7l+9QKecIAm6f45p7RNC19afSYChIZiyhtGPtWTuVJ9pa0RA==
  dependencies:
    "@insertish/exponential-backoff" "3.1.0-patch.2"
    "@insertish/isomorphic-ws" "^4.0.1"
@ -61,7 +61,7 @@
    lodash.isequal "^4.5.0"
    long "^5.2.0"
    mobx "^6.3.2"
-    revolt-api "0.5.3-rc.15"
+    revolt-api "0.5.3"
    ulid "^2.3.0"
    ws "^8.2.2"

@ -600,7 +600,16 @@ require-at@^1.0.6:
  resolved "https://registry.yarnpkg.com/require-at/-/require-at-1.0.6.tgz#9eb7e3c5e00727f5a4744070a7f560d4de4f6e6a"
  integrity sha512-7i1auJbMUrXEAZCOQ0VNJgmcT2VOKPRl2YGJwgpHpC9CE91Mv4/4UYIUm4chGJaI381ZDq1JUicFii64Hapd8g==

-revolt-api@0.5.3-rc.15, revolt-api@^0.5.3-rc.15:
+revolt-api@0.5.3:
+  version "0.5.3"
+  resolved "https://registry.yarnpkg.com/revolt-api/-/revolt-api-0.5.3.tgz#e0ec2dcf812ea4338247b2eb77d67fc731d71b8a"
+  integrity sha512-hYdyStQiDZFvD+0dlf6SgQSiOk+JiEmQo0qIQHaqYRtrFN6FQBbGVNaiv7b5LzHHMPq7vks6ZVVA7hSNpcwlkA==
+  dependencies:
+    "@insertish/oapi" "0.1.15"
+    axios "^0.26.1"
+    lodash.defaultsdeep "^4.6.1"
+
+revolt-api@^0.5.3-rc.15:
  version "0.5.3-rc.15"
  resolved "https://registry.yarnpkg.com/revolt-api/-/revolt-api-0.5.3-rc.15.tgz#abd08dd8109d0ca31be118461eabbeb6c3b7792e"
  integrity sha512-MYin3U+KoObNkILPf2cz+FPperynExkUu7CjzurMJCRvBncpnhb2czvWDvnhLDKBHlpo8W597xNqzQnaklV4ug==