From 61d082d5092e3fc65a5c49aea4ff40e56dc68675 Mon Sep 17 00:00:00 2001 From: abirc8010 Date: Mon, 4 Aug 2025 02:06:03 +0530 Subject: [PATCH] feat:add auto-moderation --- AiServerGuideAgentApp.ts | 40 +++++++++- app.json | 1 + enums/promptEnum.ts | 1 + src/constants/PromptProvider.ts | 6 +- src/constants/UserPrompt.ts | 77 +++++++++++++++++++ .../message-handler/users/moderationHanler.ts | 58 ++++++++++++++ utils/message.ts | 5 +- 7 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 src/handlers/message-handler/users/moderationHanler.ts diff --git a/AiServerGuideAgentApp.ts b/AiServerGuideAgentApp.ts index bafd4f0..ba1bcbb 100644 --- a/AiServerGuideAgentApp.ts +++ b/AiServerGuideAgentApp.ts @@ -22,8 +22,9 @@ import { getDirectRoom, sendMessage } from './utils/message'; import { processAdminMessage, processUserMessage } from './utils/processMessage'; import { IApiConfig } from './definitions/IApiConfig'; import { ApiConfigPersistence } from './src/persistence/ApiConfigPersistence'; - -export class AiServerGuideAgentApp extends App implements IPostMessageSentToBot, IPostUserCreated { +import { IPostMessageSent } from '@rocket.chat/apps-engine/definition/messages'; +import { handleModeration } from './src/handlers/message-handler/users/moderationHanler'; +export class AiServerGuideAgentApp extends App implements IPostMessageSent, IPostMessageSentToBot, IPostUserCreated { constructor(info: IAppInfo, logger: ILogger, accessors: IAppAccessors) { super(info, logger, accessors); @@ -94,6 +95,41 @@ export class AiServerGuideAgentApp extends App implements IPostMessageSentToBot, } } + public async executePostMessageSent( + message: IMessage, + read: IRead, + http: IHttp, + persistence: IPersistence, + modify: IModify, + ): Promise { + if (message.room.type == 'd') { + return; + } + const botUser = await read.getUserReader().getAppUser(); + + if (!botUser) { + return; + } + if (botUser.id == message.sender.id) { + return; + } + const dmRoom = await getDirectRoom( + read, + modify, + botUser, + message.sender.username, + ); + if (dmRoom?.id === message.room.id) { + return; + } + const messageToSend = await handleModeration(botUser, modify, message, read, persistence, http); + if (!messageToSend) { + return; + } + sendMessage(modify, message.room, botUser, messageToSend, message.threadId); + } + + public async executePostUserCreated( context: IUserContext, read: IRead, diff --git a/app.json b/app.json index bc5683f..8a868d1 100644 --- a/app.json +++ b/app.json @@ -13,6 +13,7 @@ "classFile": "AiServerGuideAgentApp.ts", "description": "an intelligent assistant that helps onboard users and allows admins configure and manage server settings through a conversational interface.", "implements": [ + "IPostMessageSent", "IPostMessageSentToBot", "IPostUserCreated" ], diff --git a/enums/promptEnum.ts b/enums/promptEnum.ts index c4a8f33..31248bb 100644 --- a/enums/promptEnum.ts +++ b/enums/promptEnum.ts @@ -7,4 +7,5 @@ export enum PromptEnum { USER_WORKFLOW_DETECTION_PROMPT = 'USER_WORKFLOW_DETECTION_PROMPT', USER_COMMAND_EXECUTE_PROMPT = 'USER_COMMAND_EXECUTE_PROMPT', USER_COMMAND_SYSTEM_PROMPT = 'USER_COMMAND_SYSTEM_PROMPT', + USER_MODERATION_PROMPT = 'USER_MODERATION_PROMPT', } diff --git a/src/constants/PromptProvider.ts b/src/constants/PromptProvider.ts index 6707d1f..5266ecb 100644 --- a/src/constants/PromptProvider.ts +++ b/src/constants/PromptProvider.ts @@ -29,9 +29,9 @@ export class PromptProvider { } public static getUserPrompt( type: PromptEnum, - details: { userMessage: string, history?: string }, + details: { userMessage: string, history?: string, userConfig?: IAdminConfig, customInfo?: any }, ): string { - const { userMessage, history } = details; + const { userMessage, history, customInfo } = details; switch (type) { case PromptEnum.USER_WORKFLOW_DETECTION_PROMPT: return UserPrompt.getWorkflowDetectionPrompt(userMessage, history); @@ -39,6 +39,8 @@ export class PromptProvider { return UserPrompt.getToolExecuteUserPrompt(userMessage, history); case PromptEnum.USER_COMMAND_SYSTEM_PROMPT: return UserPrompt.getToolExecuteSystemPrompt(); + case PromptEnum.USER_MODERATION_PROMPT: + return UserPrompt.getModerationPrompt(userMessage, customInfo); default: throw new Error('Invalid prompt type'); } diff --git a/src/constants/UserPrompt.ts b/src/constants/UserPrompt.ts index 68db5a3..3ce8754 100644 --- a/src/constants/UserPrompt.ts +++ b/src/constants/UserPrompt.ts @@ -31,6 +31,83 @@ export class UserPrompt { } `; } + public static getModerationPrompt(userMessage: string, details: any): string { + return ` + You are aiserverguideagent.bot, a moderation assistant. You are tasked with evaluating messages to ensure they comply fully with server rules. + + Given: + + - Current Room Name: ${details.roomName} + + - Message Sender Username: ${details.userName} + + - Message: "${userMessage}" + + - Server Rules: "${details.serverRules}" + + - Message History: ### + ${details.messageHistory} + ### + Instructions: + + - Evaluate whether the message violates any specific rule from provided ServerRules. + + - Check the last 4 entries in messageHistory to determine if the same rule has already been violated. + + - Analyze the server rules and determine if there is an exception for any username and the current sender or for the current room + + - Respond according to the following logic: + + Case 1: First-time violation of a rule + + - "violates" = true + + - "deleteMessage" = true (if applicable) + + - "replyNeeded" = true + + - "reply" = A message starting with "@${details.userName}", clearly stating the rule broken. + + Case 2: Repeated violation of the same rule (within last 4 messages) + + - "violates" = true + + - "deleteMessage" = true + + - "replyNeeded" = false + + - "reply" = "" (leave blank) + + Case 3: Violation of new rule (within last 4 messages) and the serverRules specifies not to delete the message if this rule is violated + + - "violates" = true + + - "deleteMessage" = false + + - "replyNeeded" = true + + - "reply" = A message starting with "@${details.userName}", clearly stating the rule broken. + + Case 4: No rule violation + + - "violates" = false + + - "deleteMessage" = false + + - "replyNeeded" = false + + - "reply" = "" (leave blank) + + Use the following strict JSON structure in your response: + { + "violates": true | false, + "deleteMessage": true | false, + "replyNeeded": true | false, + "reply": "relevant reply if applicable." + } + + ` + } public static getToolExecuteSystemPrompt(): string { return ` You are a helpful assistant capable of executing internal commands (functions) to complete user requests. diff --git a/src/handlers/message-handler/users/moderationHanler.ts b/src/handlers/message-handler/users/moderationHanler.ts new file mode 100644 index 0000000..aab46f6 --- /dev/null +++ b/src/handlers/message-handler/users/moderationHanler.ts @@ -0,0 +1,58 @@ +import { IMessage } from "@rocket.chat/apps-engine/definition/messages"; +import { IModify, IRead } from "@rocket.chat/apps-engine/definition/accessors"; +import { IHttp } from "@rocket.chat/apps-engine/definition/accessors"; +import { IUser } from "@rocket.chat/apps-engine/definition/users"; +import { IPersistence } from "@rocket.chat/apps-engine/definition/accessors"; +import { AdminPersistence } from "../../../persistence/AdminPersistence"; +import { getModel } from "../../ai-handler/AIModelHandler"; +import { PromptProvider } from "../../../constants/PromptProvider"; +import { PromptEnum } from "../../../../enums/promptEnum"; +export async function handleModeration( + botUser: IUser, + modify: IModify, + message: IMessage, + read: IRead, + persistence: IPersistence, + http: IHttp, +): Promise { + const adminStore = new AdminPersistence(persistence, read.getPersistenceReader()); + const adminConfig = await adminStore.getAdminConfig(); + + if (!adminConfig?.serverRules) { + return null; + } + + const messages = await read.getRoomReader().getMessages(message.room.id, { + limit: 10, + }); + + const messageHistory = messages + .map((msg) => `sender : "${msg.sender.username}" , message: "${msg.text}"`) + .join('\n'); + + const details = { + roomName: message.room.displayName, + userName: message.sender.username, + serverRules: adminConfig?.serverRules ?? '', + messageHistory: messageHistory || 'No previous messages available', + } + const model = await getModel(read); + const moderationPrompt = PromptProvider.getUserPrompt( + PromptEnum.USER_MODERATION_PROMPT, + { userMessage: message.text ?? '', customInfo: details }); + try { + + const response = await model.generateResponse(moderationPrompt, http, read); + const { violates, reply, deleteMessage, replyNeeded } = JSON.parse(response); + if (violates) { + if (deleteMessage) { + await modify.getDeleter().deleteMessage(message, botUser); + } + if (replyNeeded) + return reply; + } + } catch (error) { + console.error('Error generating moderation response:', error); + } + return null; +} \ No newline at end of file diff --git a/utils/message.ts b/utils/message.ts index 1e47880..44af82a 100644 --- a/utils/message.ts +++ b/utils/message.ts @@ -44,6 +44,7 @@ export async function sendMessage( room: IRoom, sender: IUser, message: string, + ThreadId?: string, blocks?: Array, ): Promise { const msg = modify @@ -53,7 +54,9 @@ export async function sendMessage( .setRoom(room) .setParseUrls(true) .setText(message); - + if (ThreadId) { + msg.setThreadId(ThreadId); + } if (blocks !== undefined) { msg.setBlocks(blocks); }