From 59f064583a2e6234971aa74a097951e5577fe909 Mon Sep 17 00:00:00 2001 From: Gitead Date: Sat, 25 Jul 2026 17:24:27 -0400 Subject: [PATCH] feat: AI Type Agent - vision-based Open Dental automation via Windows agent Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend services to locate UI elements via vision and drive an existing-patient appointment flow in Open Dental, wired into the Copy/Type Agent page and socket progress updates. Co-Authored-By: Claude Sonnet 5 --- apps/Backend/.env | 1 + apps/Backend/.env.example | 1 + apps/Backend/package.json | 1 + apps/Backend/scripts/fake-windows-agent.js | 68 ++ apps/Backend/src/routes/index.ts | 2 + apps/Backend/src/routes/type-agent.ts | 109 ++++ apps/Backend/src/services/typeAgentRunner.ts | 138 +++++ apps/Backend/src/services/visionLocate.ts | 96 +++ .../src/services/windowsAgentBridge.ts | 58 ++ apps/Backend/src/socket.ts | 58 ++ .../Frontend/src/pages/ai-type-agent-page.tsx | 471 +++++++++++--- apps/WindowsAgent/README.md | 59 ++ apps/WindowsAgent/agent.py | 218 +++++++ apps/WindowsAgent/package.json | 4 + apps/WindowsAgent/requirements.txt | 7 + package-lock.json | 579 ++++++++++++++++++ 16 files changed, 1769 insertions(+), 101 deletions(-) create mode 100644 apps/Backend/scripts/fake-windows-agent.js create mode 100644 apps/Backend/src/routes/type-agent.ts create mode 100644 apps/Backend/src/services/typeAgentRunner.ts create mode 100644 apps/Backend/src/services/visionLocate.ts create mode 100644 apps/Backend/src/services/windowsAgentBridge.ts create mode 100644 apps/WindowsAgent/README.md create mode 100644 apps/WindowsAgent/agent.py create mode 100644 apps/WindowsAgent/package.json create mode 100644 apps/WindowsAgent/requirements.txt diff --git a/apps/Backend/.env b/apps/Backend/.env index b26a93d4..b0c7f8f1 100755 --- a/apps/Backend/.env +++ b/apps/Backend/.env @@ -4,6 +4,7 @@ PORT=5000 CLOUDFLARE_HOST= FRONTEND_URLS=http://localhost:3000,https://local-summit.mydentalofficemanagement.com SELENIUM_AGENT_BASE_URL=http://localhost:5002 +WINDOWS_AGENT_TOKEN=dev-windows-agent-token JWT_SECRET = 'dentalsecret' LICENSE_SECRET=3aa4ab937e46c6863b9e3c2b591a595b31ea3af1060bf5e7961ad722a8b54f92 DB_HOST=localhost diff --git a/apps/Backend/.env.example b/apps/Backend/.env.example index 0a4e46ed..291729a2 100755 --- a/apps/Backend/.env.example +++ b/apps/Backend/.env.example @@ -4,6 +4,7 @@ PORT=5000 CLOUDFLARE_HOST= FRONTEND_URLS=http://localhost:3000 SELENIUM_AGENT_BASE_URL=http://localhost:5002 +WINDOWS_AGENT_TOKEN= JWT_SECRET = 'dentalsecret' DB_HOST=localhost DB_USER=postgres diff --git a/apps/Backend/package.json b/apps/Backend/package.json index 8fb3d090..e30ac5be 100755 --- a/apps/Backend/package.json +++ b/apps/Backend/package.json @@ -33,6 +33,7 @@ "passport": "^0.7.0", "passport-local": "^1.0.0", "pdfkit": "^0.17.2", + "sharp": "^0.35.3", "socket.io": "^4.8.1", "twilio": "^6.0.0", "ws": "^8.18.0", diff --git a/apps/Backend/scripts/fake-windows-agent.js b/apps/Backend/scripts/fake-windows-agent.js new file mode 100644 index 00000000..dcb853c8 --- /dev/null +++ b/apps/Backend/scripts/fake-windows-agent.js @@ -0,0 +1,68 @@ +// Simulates DentalAgent.exe connecting to the backend's /agent socket.io namespace, +// so the Type Agent page's "Windows Agent: Connected" status can be tested end-to-end +// before the real pyautogui-based Windows client exists. +// +// Usage: node scripts/fake-windows-agent.js [serverUrl] +// Requires WINDOWS_AGENT_TOKEN to match the backend's .env value. +// The server identifies agents by connection IP address, not anything sent here. + +const { io } = require("socket.io-client"); + +const serverUrl = process.argv[2] || "http://localhost:5000"; +const token = process.env.WINDOWS_AGENT_TOKEN || "dev-windows-agent-token"; + +const socket = io(`${serverUrl}/agent`, { + auth: { token }, + reconnectionAttempts: 5, +}); + +socket.on("connect", () => { + console.log(`✅ Connected as fake Windows Agent, socket id: ${socket.id}`); +}); + +// 1x1 transparent PNG, base64 — stands in for a real screen capture. +const FAKE_SCREENSHOT = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +socket.on("cmd:screenshot", (_payload, ack) => { + console.log("📸 screenshot requested"); + ack({ image: FAKE_SCREENSHOT }); +}); + +socket.on("cmd:click", ({ x, y }, ack) => { + console.log(`🖱️ click at (${x}, ${y})`); + ack({ ok: true }); +}); + +socket.on("cmd:double_click", ({ x, y }, ack) => { + console.log(`🖱️ double-click at (${x}, ${y})`); + ack({ ok: true }); +}); + +socket.on("cmd:double_click_current", (_payload, ack) => { + console.log("🖱️ double-click at current cursor position"); + ack({ ok: true }); +}); + +socket.on("cmd:type", ({ text }, ack) => { + console.log(`⌨️ type "${text}"`); + ack({ ok: true }); +}); + +socket.on("cmd:key", ({ key }, ack) => { + console.log(`⌨️ press key "${key}"`); + ack({ ok: true }); +}); + +socket.on("connect_error", (err) => { + console.error("❌ Connection failed:", err.message); +}); + +socket.on("disconnect", (reason) => { + console.log("🔌 Disconnected:", reason); +}); + +process.on("SIGINT", () => { + socket.close(); + process.exit(0); +}); diff --git a/apps/Backend/src/routes/index.ts b/apps/Backend/src/routes/index.ts index 36d62590..8b8914ef 100755 --- a/apps/Backend/src/routes/index.ts +++ b/apps/Backend/src/routes/index.ts @@ -45,6 +45,7 @@ import licenseRoutes from "./license"; import insuranceStatusBcbsMaRoutes from "./insuranceStatusBcbsMa"; import labRxRoutes from "./lab-rx"; import seleniumSettingsRoutes from "./selenium-settings"; +import typeAgentRoutes from "./type-agent"; const router = Router(); @@ -82,6 +83,7 @@ router.use("/cloud-storage", cloudStorageRoutes); router.use("/payments-reports", paymentsReportsRoutes); router.use("/export-payments-reports", exportPaymentsReportsRoutes); router.use("/job-monitor", jobMonitorRoutes); +router.use("/type-agent", typeAgentRoutes); router.use("/twilio", twilioRoutes); router.use("/ai", aiSettingsRoutes); router.use("/office-hours", officeHoursRoutes); diff --git a/apps/Backend/src/routes/type-agent.ts b/apps/Backend/src/routes/type-agent.ts new file mode 100644 index 00000000..456611ae --- /dev/null +++ b/apps/Backend/src/routes/type-agent.ts @@ -0,0 +1,109 @@ +import crypto from "crypto"; +import { Router, Request, Response } from "express"; +import { getAgentStatus, io } from "../socket"; +import { captureScreenshot, click, disconnectAgent, doubleClick, doubleClickCurrent, pressKey, typeText } from "../services/windowsAgentBridge"; +import { runExistingPatientOpenDental } from "../services/typeAgentRunner"; +import { storage } from "../storage"; + +const router = Router(); + +// GET /api/type-agent/status +// Current Windows Agent (DentalAgent.exe) connection status, for the initial page load — +// live updates after that come over the "type-agent:status" socket.io event. +router.get("/status", (_req: Request, res: Response) => { + res.json(getAgentStatus()); +}); + +// POST /api/type-agent/disconnect +// Forcibly disconnects a connected agent by IP — the "x" on the Type Agent page. +router.post("/disconnect", (req: Request, res: Response) => { + const { ip } = req.body ?? {}; + if (!ip) { + res.status(400).json({ error: "ip is required" }); + return; + } + try { + disconnectAgent(ip); + res.json({ success: true }); + } catch (err) { + res.status(404).json({ error: err instanceof Error ? err.message : "Agent not found" }); + } +}); + +// POST /api/type-agent/test-command +// Sends a single low-level command (screenshot/click/double_click/type/key) straight to a +// connected Windows Agent. Exists to verify the command bridge before the AI-driven step +// orchestration is wired in — not meant to be called from the Type Agent UI directly. +router.post("/test-command", async (req: Request, res: Response) => { + const { type, x, y, text, key, ip } = req.body ?? {}; + try { + let result; + switch (type) { + case "screenshot": + result = await captureScreenshot(ip); + break; + case "click": + result = await click(Number(x), Number(y), ip); + break; + case "double_click": + result = await doubleClick(Number(x), Number(y), ip); + break; + case "double_click_current": + result = await doubleClickCurrent(ip); + break; + case "type": + result = await typeText(String(text ?? ""), ip); + break; + case "key": + result = await pressKey(String(key ?? ""), ip); + break; + default: + res.status(400).json({ error: "Unknown command type" }); + return; + } + res.json({ success: true, result }); + } catch (err) { + res.status(500).json({ error: err instanceof Error ? err.message : "Command failed" }); + } +}); + +// POST /api/type-agent/run +// Kicks off a real step-by-step run against a connected Windows Agent. Only Open Dental / +// existing-patient is wired up so far — everything else the Type Agent page still shows as +// a preview only. Responds immediately with a runId; progress streams over the +// "type-agent:run:" socket.io event so the browser doesn't have to hold the request +// open for what can be a slow, multi-step, vision-driven sequence. +router.post("/run", async (req: Request, res: Response) => { + const userId = req.user?.id; + if (!userId) { + res.status(401).json({ error: "Unauthorized" }); + return; + } + + const { software, action, ip, patientId } = req.body ?? {}; + if (software !== "open-dental" || action !== "existing-patient") { + res.status(400).json({ error: "This software/action combination isn't automated yet." }); + return; + } + if (!ip) { + res.status(400).json({ error: "ip is required" }); + return; + } + + const patient = await storage.getPatient(Number(patientId)); + if (!patient?.lastName || !patient?.firstName) { + res.status(400).json({ error: "Patient not found or missing a first/last name" }); + return; + } + + const runId = crypto.randomUUID(); + res.json({ runId }); + + runExistingPatientOpenDental(userId, ip, patient.lastName, patient.firstName, (index, total, label, status, error) => { + io?.emit(`type-agent:run:${runId}`, { index, total, label, status, error }); + }).catch((err) => { + console.error("[type-agent/run] failed:", err); + }); +}); + +export default router; diff --git a/apps/Backend/src/services/typeAgentRunner.ts b/apps/Backend/src/services/typeAgentRunner.ts new file mode 100644 index 00000000..f598546c --- /dev/null +++ b/apps/Backend/src/services/typeAgentRunner.ts @@ -0,0 +1,138 @@ +import { captureScreenshot, click, doubleClick, doubleClickCurrent, typeText } from "./windowsAgentBridge"; +import { locateOnScreenshot } from "./visionLocate"; + +export type StepStatus = "running" | "done" | "error"; +export type ProgressCallback = ( + index: number, + total: number, + label: string, + status: StepStatus, + error?: string +) => void; + +interface RunContext { + ip: string; + userId: number; + patientLastName: string; + patientFirstName: string; +} + +interface RunStep { + label: string; + execute: (ctx: RunContext) => Promise; + // Time to let the target app's UI settle/render before the next step's screenshot. + delayAfterMs: number; +} + +async function visionClick(ctx: RunContext, goal: string, opts?: { doubleClick?: boolean }) { + const { image } = await captureScreenshot(ctx.ip); + const loc = await locateOnScreenshot(ctx.userId, image, goal); + if (!loc) throw new Error(`Could not find on screen: ${goal}`); + console.log(`[type-agent] ${opts?.doubleClick ? "double-click" : "click"} at (${loc.x}, ${loc.y}) — goal: ${goal.slice(0, 80)}...`); + if (opts?.doubleClick) { + await doubleClick(loc.x, loc.y, ctx.ip); + } else { + await click(loc.x, loc.y, ctx.ip); + } +} + +// Open Dental, "make an appointment for an existing patient" — the exact sequence confirmed +// against a real Open Dental instance: double-click the pre-positioned schedule slot (opens +// Select Patient) → type last name → double-click the matching patient row (opens Edit +// Appointment with the appointment already created) → click the Exam procedure → click Save. +const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ + { + label: "Double-click the schedule at the cursor position", + execute: async (ctx) => { + await doubleClickCurrent(ctx.ip); + }, + delayAfterMs: 1000, // Select Patient window opening + }, + { + label: "Find and click the Last Name field", + execute: async (ctx) => { + await visionClick( + ctx, + "the Last Name text input field in the Select Patient window — it's in the \"Search by:\" " + + "panel on the RIGHT half of the window (if you split the window into a left half and a " + + "right half), not in the results grid on the left" + ); + }, + delayAfterMs: 300, + }, + { + label: "Type the patient's last name", + execute: async (ctx) => { + await typeText(ctx.patientLastName, ctx.ip); + }, + delayAfterMs: 600, // patient list filtering + }, + { + label: "Find and double-click the matching patient row", + execute: async (ctx) => { + const lastPrefix = ctx.patientLastName.slice(0, 3); + const firstPrefix = ctx.patientFirstName.slice(0, 3); + await visionClick( + ctx, + `Search the LEFT half of the Select Patient window for the row of patient data whose ` + + `LastName column starts with "${lastPrefix}" AND whose First Name column starts with ` + + `"${firstPrefix}" — matching is case-insensitive and only needs to match the first 3 ` + + `letters of each name, not the full name exactly. Both the last name prefix and first name ` + + `prefix must match — there may be other rows with a matching last name (family members, ` + + `e.g. a sibling or spouse) but a different first name; do NOT click those, they are the ` + + `wrong patient. That row is likely shaded/highlighted in LIGHT BLUE as the currently-selected ` + + `row. Do NOT click the window's own title bar — a DARK navy-blue bar at the very top of the ` + + `window that just reads "Select Patient" in white text; that is window chrome, not a patient ` + + `row, and is well above the results grid. Also do not click the column header row (labels ` + + `like "PatNum", "LastName", "First Name"). Click on the actual patient text in the matching data row.`, + { doubleClick: true } + ); + }, + delayAfterMs: 1200, // Edit Appointment window opening + }, + { + label: "Find and click the Exam procedure", + execute: async (ctx) => { + await visionClick( + ctx, + 'the "Exam" item in the procedures list of the Edit Appointment window — it\'s in the ' + + "UPPER-MIDDLE area of the window (if you split the window into an upper half and a lower " + + "half, and again into left/middle/right thirds, it's in the upper half, middle third), " + + 'in a plain text list that also contains items like "Ex,Pro,Flo", "Prophy-Adult", "Pano"' + ); + }, + delayAfterMs: 500, + }, + { + label: "Find and click Save", + execute: async (ctx) => { + await visionClick(ctx, "the Save button in the lower right corner of the Edit Appointment window"); + }, + delayAfterMs: 300, + }, +]; + +export async function runExistingPatientOpenDental( + userId: number, + ip: string, + patientLastName: string, + patientFirstName: string, + onProgress: ProgressCallback +): Promise { + const ctx: RunContext = { ip, userId, patientLastName, patientFirstName }; + const steps = OPEN_DENTAL_EXISTING_PATIENT; + + for (let i = 0; i < steps.length; i++) { + const step = steps[i]!; + onProgress(i, steps.length, step.label, "running"); + try { + await step.execute(ctx); + onProgress(i, steps.length, step.label, "done"); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + onProgress(i, steps.length, step.label, "error", message); + throw err; + } + await new Promise((resolve) => setTimeout(resolve, step.delayAfterMs)); + } +} diff --git a/apps/Backend/src/services/visionLocate.ts b/apps/Backend/src/services/visionLocate.ts new file mode 100644 index 00000000..803076eb --- /dev/null +++ b/apps/Backend/src/services/visionLocate.ts @@ -0,0 +1,96 @@ +import { storage } from "../storage"; +import { resolveAiProvider, getLlm } from "../ai/llm-factory"; + +interface ParsedLocation { + found: boolean; + x?: number; + y?: number; +} + +// Parses the model's response into {found, x, y}, tolerating near-miss JSON rather than +// only accepting strictly valid JSON. Seen in practice: {"found": true, "x": 1023, 187} — +// a dropped "y" key with the value still present positionally. Since this response should +// only ever contain two numbers (the coordinates), falling back to "first two numbers after +// found" is safe and recovers without burning another AI call. +function parseLocationResponse(raw: string): ParsedLocation | null { + const match = raw.match(/\{[\s\S]*\}/); + if (!match) return null; + const block = match[0]; + + try { + return JSON.parse(block) as ParsedLocation; + } catch { + // fall through to lenient repair below + } + + const foundMatch = block.match(/"found"\s*:\s*(true|false)/i); + if (!foundMatch) return null; + const found = foundMatch[1]!.toLowerCase() === "true"; + if (!found) return { found: false }; + + const afterFound = block.slice(foundMatch.index! + foundMatch[0].length); + const numbers = afterFound.match(/-?\d+(\.\d+)?/g); + if (!numbers || numbers.length < 2) return null; + + return { found: true, x: Number(numbers[0]), y: Number(numbers[1]) }; +} + +// Asks the configured vision AI to find a described UI element in a screenshot and return +// its pixel coordinates — the core primitive the Type Agent step runner clicks/types through. +// Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info. +export async function locateOnScreenshot( + userId: number, + imageBase64: string, + goal: string +): Promise<{ x: number; y: number } | null> { + const aiSettings = await storage.getAiSettings(userId); + const activeAi = resolveAiProvider(aiSettings ?? {}); + if (!activeAi) { + throw new Error("AI is not configured. Add an API key in AI Settings."); + } + if (activeAi.provider !== "claude") { + throw new Error("Vision-guided steps require Claude to be the active AI provider."); + } + + const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + + const content = [ + { + type: "text", + text: + `This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` + + "Respond with strict JSON only, no prose, no markdown fences: " + + '{"found": true, "x": , "y": } if you can locate it (coordinates must be ' + + 'pixel positions within this exact image, at the center of the element), or {"found": false} ' + + "if it isn't visible in this screenshot.", + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${imageBase64}` }, + }, + ]; + + // A response that's fully unparseable even after the lenient repair above is worth one + // retry (rare model hiccup); anything the repair can salvage is used immediately, no + // retry needed. + const MAX_ATTEMPTS = 2; + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const response = await llm.invoke([{ role: "user", content }] as any); + const raw = String(response.content).trim(); + + const parsed = parseLocationResponse(raw); + if (!parsed) { + lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); + continue; + } + + if (!parsed.found || typeof parsed.x !== "number" || typeof parsed.y !== "number") { + return null; + } + return { x: Math.round(parsed.x), y: Math.round(parsed.y) }; + } + + throw lastError ?? new Error("AI returned an unparseable response"); +} diff --git a/apps/Backend/src/services/windowsAgentBridge.ts b/apps/Backend/src/services/windowsAgentBridge.ts new file mode 100644 index 00000000..f550179d --- /dev/null +++ b/apps/Backend/src/services/windowsAgentBridge.ts @@ -0,0 +1,58 @@ +import { io } from "../socket"; + +const COMMAND_TIMEOUT_MS = 10000; + +// Picks a connected Windows Agent socket by IP address — falls back to the first connected +// agent if none is given (fine while offices only ever have one PC connected). +function getAgentSocket(ip?: string) { + if (!io) throw new Error("Socket.io not initialized"); + const namespaceSockets = Array.from(io.of("/agent").sockets.values()); + const target = ip + ? namespaceSockets.find((s) => { + const socketIp = (s.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim() + || s.handshake.address; + return socketIp === ip; + }) + : namespaceSockets[0]; + if (!target) throw new Error("No Windows Agent connected"); + return target; +} + +async function sendCommand(event: string, payload: unknown, ip?: string): Promise { + const socket = getAgentSocket(ip); + return socket.timeout(COMMAND_TIMEOUT_MS).emitWithAck(event, payload) as Promise; +} + +// Captures the Windows PC's screen. Returns a base64-encoded PNG for the app's AI to read. +export function captureScreenshot(ip?: string) { + return sendCommand<{ image: string }>("cmd:screenshot", {}, ip); +} + +export function click(x: number, y: number, ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:click", { x, y }, ip); +} + +export function doubleClick(x: number, y: number, ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:double_click", { x, y }, ip); +} + +// Double-clicks wherever the mouse already is — no coordinates needed. Used for the first +// step of a run, where staff pre-position the cursor before triggering it from the browser. +export function doubleClickCurrent(ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:double_click_current", {}, ip); +} + +export function typeText(text: string, ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:type", { text }, ip); +} + +export function pressKey(key: string, ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:key", { key }, ip); +} + +// Forcibly closes a connected agent's socket — used by the "x" on the Type Agent page to +// let staff stop controlling a PC without touching the exe on that machine. +export function disconnectAgent(ip: string) { + const socket = getAgentSocket(ip); + socket.disconnect(true); +} diff --git a/apps/Backend/src/socket.ts b/apps/Backend/src/socket.ts index b5e764fe..a523b1e3 100755 --- a/apps/Backend/src/socket.ts +++ b/apps/Backend/src/socket.ts @@ -3,6 +3,22 @@ import { Server, Socket } from "socket.io"; let io: Server | null = null; +interface ConnectedAgent { + socketId: string; + ip: string; + connectedAt: string; +} + +// Windows Agents (DentalAgent.exe) connected via the /agent namespace, keyed by socket id. +const connectedAgents = new Map(); + +export function getAgentStatus() { + return { + connected: connectedAgents.size > 0, + agents: Array.from(connectedAgents.values()), + }; +} + export function initSocket(server: HttpServer) { const NODE_ENV = ( process.env.NODE_ENV || @@ -47,7 +63,49 @@ export function initSocket(server: HttpServer) { console.error("Socket engine connection_error:", err); }); + initAgentNamespace(io); + return io; } +// Windows Agent (DentalAgent.exe) bridge. Separate namespace so browser clients on the +// default namespace never see agent traffic, and so we can gate it with a shared secret — +// this connection will eventually be able to click/type on a real front-desk PC. +function initAgentNamespace(server: Server) { + const agentNamespace = server.of("/agent"); + const AGENT_TOKEN = process.env.WINDOWS_AGENT_TOKEN; + + agentNamespace.use((socket, next) => { + if (!AGENT_TOKEN) { + // No token configured — refuse rather than silently accepting any connection. + return next(new Error("Windows Agent bridge is not configured (WINDOWS_AGENT_TOKEN unset)")); + } + if (socket.handshake.auth?.token !== AGENT_TOKEN) { + return next(new Error("Invalid agent token")); + } + next(); + }); + + agentNamespace.on("connection", (socket: Socket) => { + // x-forwarded-for is only trustworthy behind a proxy we control (nginx in prod); + // falls back to the raw socket address for direct LAN connections in dev. + const ip = (socket.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim() + || socket.handshake.address; + + connectedAgents.set(socket.id, { + socketId: socket.id, + ip, + connectedAt: new Date().toISOString(), + }); + console.log("🖥️ Windows Agent connected:", socket.id, ip); + io?.emit("type-agent:status", getAgentStatus()); + + socket.on("disconnect", () => { + connectedAgents.delete(socket.id); + console.log("🖥️ Windows Agent disconnected:", socket.id); + io?.emit("type-agent:status", getAgentStatus()); + }); + }); +} + export { io }; diff --git a/apps/Frontend/src/pages/ai-type-agent-page.tsx b/apps/Frontend/src/pages/ai-type-agent-page.tsx index 30580fe3..81fab9fe 100644 --- a/apps/Frontend/src/pages/ai-type-agent-page.tsx +++ b/apps/Frontend/src/pages/ai-type-agent-page.tsx @@ -1,37 +1,208 @@ -import { Keyboard, FileCheck, CreditCard, Shield, Zap, ArrowRight } from "lucide-react"; -import { Card, CardContent } from "@/components/ui/card"; +import { ReactNode, useEffect, useState } from "react"; +import { Keyboard, CalendarPlus, UserPlus, CreditCard, Circle, CheckCircle2, Monitor, X, Loader2, XCircle, Maximize2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { PatientTable } from "@/components/patients/patient-table"; +import { Patient } from "@repo/db/types"; +import { apiRequest } from "@/lib/queryClient"; +import { socket } from "@/lib/socket"; -const SOFTWARE_TARGETS = [ - { name: "Open Dental", color: "bg-blue-100 text-blue-700 border-blue-200" }, - { name: "Eaglesoft", color: "bg-emerald-100 text-emerald-700 border-emerald-200" }, - { name: "Dentrix", color: "bg-violet-100 text-violet-700 border-violet-200" }, - { name: "Curve Dental", color: "bg-orange-100 text-orange-700 border-orange-200" }, +type SoftwareId = "open-dental" | "dentrix" | "eaglesoft"; +type ActionId = "existing-patient" | "new-patient" | "payment"; + +interface ConnectedAgent { + socketId: string; + ip: string; + connectedAt: string; +} + +interface LiveStep { + index: number; + total: number; + label: string; + status: "running" | "done" | "error"; + error?: string; +} + +const SOFTWARE_OPTIONS: { id: SoftwareId; name: string }[] = [ + { id: "open-dental", name: "Open Dental" }, + { id: "dentrix", name: "Dentrix" }, + { id: "eaglesoft", name: "Eaglesoft" }, ]; -const CAPABILITIES = [ - { - icon: , - title: "Eligibility Results", - description: - "Automatically type insurance eligibility information — coverage status, plan details, deductibles, and co-pays — directly into the patient chart in your dental software.", - }, - { - icon: , - title: "Claim Information", - description: - "Transfer claim numbers, claim status updates, and denial reasons from the insurance portal results into your claims module without manual re-entry.", - }, - { - icon: , - title: "Insurance Payments", - description: - "Post ERA / EOB payment details — amounts, adjustments, patient responsibility — directly into your payment ledger by typing them into the active software window.", - }, +const ACTIONS: { id: ActionId; icon: ReactNode; label: string }[] = [ + { id: "existing-patient", icon: , label: "Make an appointment for an existing patient" }, + { id: "new-patient", icon: , label: "Make an appointment for a new patient" }, + { id: "payment", icon: , label: "Type payments" }, ]; +// Preview-only step sequences. Same shape across the three software targets for now — +// each software's real step semantics (field positions, screen titles) will differ once +// the Windows Agent drives them via vision, but the flow order is the same. +const STEP_TEMPLATES: Record string[]> = { + "existing-patient": (sw, patient) => [ + `Double-click to open the patient selection window in ${sw}`, + `Screenshot the window and locate the Last Name field`, + `Type "${patient}"'s last name into the field`, + `Locate and click the Search button`, + `Locate and click the matching patient row`, + `Screenshot the Add Procedure window and locate the default procedure (Exam)`, + `Click the procedure, then click Create Appointment`, + ], + "new-patient": (sw, patient) => [ + `Open the New Patient entry screen in ${sw}`, + `Screenshot the form and locate the First/Last Name fields`, + `Type "${patient}"'s name, date of birth, and insurance into the form`, + `Locate and click Save / Create Patient`, + `Screenshot the Add Procedure window and locate the default procedure (Exam)`, + `Click the procedure, then click Create Appointment`, + ], + payment: (sw, patient) => [ + `Locate "${patient}" in ${sw} and open their ledger / payment screen`, + `Screenshot the screen and locate the payment amount field`, + `Type the payment amount, date, and type`, + `Locate and click Save / Post Payment`, + ], +}; + export default function AiTypeAgentPage() { + const [selectedPatient, setSelectedPatient] = useState(null); + const [software, setSoftware] = useState("open-dental"); + const [runAction, setRunAction] = useState(null); + const [agents, setAgents] = useState([]); + const [selectedAgentIp, setSelectedAgentIp] = useState(null); + + useEffect(() => { + apiRequest("GET", "/api/type-agent/status") + .then((res) => res.json()) + .then((data) => setAgents(Array.isArray(data?.agents) ? data.agents : [])) + .catch(() => {}); + + const handleStatus = (data: { agents: ConnectedAgent[] }) => + setAgents(Array.isArray(data?.agents) ? data.agents : []); + socket.on("type-agent:status", handleStatus); + return () => { + socket.off("type-agent:status", handleStatus); + }; + }, []); + + // Keep the selected PC valid as the connected list changes — auto-pick when there's + // exactly one, and drop the selection if its PC disconnects. + useEffect(() => { + if (selectedAgentIp && !agents.some((a) => a.ip === selectedAgentIp)) { + setSelectedAgentIp(null); + } else if (!selectedAgentIp && agents.length === 1) { + setSelectedAgentIp(agents[0]!.ip); + } + }, [agents, selectedAgentIp]); + + const [liveSteps, setLiveSteps] = useState([]); + const [runStartError, setRunStartError] = useState(null); + const [startCountdown, setStartCountdown] = useState(null); + + // Open Dental / existing-patient is the only combination actually wired to the real + // Windows Agent so far — everything else still shows the static preview below. + const isAutomated = software === "open-dental" && runAction === "existing-patient"; + const START_DELAY_SECONDS = 5; + + useEffect(() => { + if (!isAutomated || !runAction || !selectedAgentIp || !selectedPatient) return; + + let cancelled = false; + let liveRunId: string | null = null; + const handleProgress = (data: LiveStep) => { + setLiveSteps((prev) => { + const next = [...prev]; + next[data.index] = data; + return next; + }); + }; + + setLiveSteps([]); + setRunStartError(null); + setStartCountdown(START_DELAY_SECONDS); + + // Gives time to minimize/switch away from Chrome and let the target software have + // focus before the first (mouse-position-dependent) step fires. + const countdownInterval = setInterval(() => { + setStartCountdown((c) => (c && c > 1 ? c - 1 : 0)); + }, 1000); + + const startTimeout = setTimeout(() => { + clearInterval(countdownInterval); + setStartCountdown(null); + + apiRequest("POST", "/api/type-agent/run", { + software, + action: runAction, + ip: selectedAgentIp, + patientId: selectedPatient.id, + }) + .then((res) => res.json()) + .then((data) => { + if (cancelled) return; + if (data?.error) { + setRunStartError(data.error); + return; + } + liveRunId = data.runId; + socket.on(`type-agent:run:${data.runId}`, handleProgress); + }) + .catch((err) => { + if (!cancelled) setRunStartError(err instanceof Error ? err.message : "Failed to start run"); + }); + }, START_DELAY_SECONDS * 1000); + + return () => { + cancelled = true; + clearInterval(countdownInterval); + clearTimeout(startTimeout); + if (liveRunId) socket.off(`type-agent:run:${liveRunId}`, handleProgress); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id]); + + const getPatientName = (patient: Patient) => + patient.firstName && patient.lastName + ? `${patient.firstName} ${patient.lastName}` + : patient.firstName ?? `patient-${patient.id}`; + + const softwareName = SOFTWARE_OPTIONS.find((s) => s.id === software)?.name ?? ""; + + const handleDisconnect = async (ip: string) => { + try { + await apiRequest("POST", "/api/type-agent/disconnect", { ip }); + } catch (error) { + console.error("Error disconnecting agent:", error); + } + }; + + const agentConnected = agents.length > 0; + const canRun = Boolean(software && selectedPatient && selectedAgentIp); + + const activeAction = ACTIONS.find((a) => a.id === runAction); + const steps = + runAction && selectedPatient + ? STEP_TEMPLATES[runAction](softwareName, getPatientName(selectedPatient)) + : []; + return ( -
+
{/* Hero */}
@@ -48,89 +219,187 @@ export default function AiTypeAgentPage() {
- {/* How it works */} + {/* Run setup */} - -
- -

How it works

+ + Run Setup + + Choose your dental software, the PC to control, and a patient below, then run one + of the actions. + + + +
+
+ +
+ + {agentConnected ? ( + + + {agents.length} PC{agents.length > 1 ? "s" : ""} Connected + + ) : ( + + + No PC Connected + + )} + + {selectedPatient ? ( + Patient: {getPatientName(selectedPatient)} + ) : ( + Select a patient below + )}
-
- {[ - "Retrieve data in this app\n(eligibility, claim, payment)", - "Agent identifies the active\nfield in your dental software", - "Types the value directly\nvia keyboard automation", - ].map((step, i) => ( -
-
- {i + 1} -
-

{step}

- {i < 2 && } + + {/* Connected PCs */} +
+

Windows PC to control

+ {agents.length === 0 ? ( +

+ No DentalAgent.exe is connected yet. Run it on the front-desk PC you want to + control — it will show up here. +

+ ) : ( +
+ {agents.map((agent) => ( +
+ + +
+ ))}
+ )} +
+ +

+ + Maximize {softwareName || "your dental software"} to full screen on the target PC before running an action — the agent reads the whole screen, so a small or floating window makes it harder to find things reliably. +

+ +
+ {ACTIONS.map((action) => ( + ))}
-

- The agent runs as a lightweight local process on your workstation. It receives - instructions from this app and uses keyboard automation to type data into whatever - window is currently focused in your dental software — no clipboard involved. -

- {/* What it will type */} -
-

What it will type

-
- {CAPABILITIES.map((cap) => ( - - -
- {cap.icon} - {cap.title} -
-

- {cap.description} + {/* Patient search */} + + + Patient Records + + Select the patient to use for the action above. + + + + + + + + {/* Run panel */} +

!open && setRunAction(null)}> + + + + {activeAction?.label} — {softwareName} + + + {isAutomated + ? `Running live on the connected PC at ${selectedAgentIp}.` + : selectedAgentIp + ? `Running on the connected PC at ${selectedAgentIp}.` + : "Preview only — no PC selected, so these steps won't run automatically."} + + + + {isAutomated ? ( +
+ {runStartError && ( +

{runStartError}

+ )} + {startCountdown !== null && ( +

+ Starting in {startCountdown}s — minimize or switch away from this window now.

- - - ))} -
-
- - {/* Supported software */} -
-

Planned software support

-
- {SOFTWARE_TARGETS.map((s) => ( - - {s.name} - - ))} -
-

- Each software has its own field layout. The agent will include pre-built templates - for common workflows in each platform. -

-
- - {/* Why */} - - - -
-

Why this matters

-

- Staff currently check eligibility or look up a claim here, then manually retype - every value into Open Dental or Eaglesoft. This agent eliminates that step entirely — - one click sends the data straight into the right field. -

-
- - + )} + {!runStartError && startCountdown === null && liveSteps.length === 0 && ( +

Starting run...

+ )} + {liveSteps.map((step, i) => ( +
+
+ {step.status === "running" && } + {step.status === "done" && } + {step.status === "error" && } +
+
+

{step.label}

+ {step.error &&

{step.error}

} +
+
+ ))} +
+ ) : ( +
+ {steps.map((step, i) => ( +
+
+ + {i + 1} + +
+

{step}

+
+ ))} +
+ )} + +
); diff --git a/apps/WindowsAgent/README.md b/apps/WindowsAgent/README.md new file mode 100644 index 00000000..f6208d85 --- /dev/null +++ b/apps/WindowsAgent/README.md @@ -0,0 +1,59 @@ +# Dental Agent (Windows) + +`agent.py` runs on a staff front-desk Windows PC. It connects to the app's server over the +`/agent` socket.io namespace and executes low-level commands — screenshot, click, +double-click, type, key press — sent by the server. It has no knowledge of dental software +or workflows; the app's AI reads the screenshots this agent sends back and decides what to +click or type next. + +Not wired into the repo's root `npm install` on purpose — it targets Windows (`pyautogui`, +`tkinter`) and doesn't need to install on every contributor's machine. + +## Command protocol + +Sent by the server, over the `/agent` namespace, as socket.io events with an ack callback: + +| Event | Payload | Ack response | +|---------------------|-----------------------|--------------------| +| `cmd:screenshot` | `{}` | `{ image: }` | +| `cmd:click` | `{ x, y }` | `{ ok: true }` | +| `cmd:double_click` | `{ x, y }` | `{ ok: true }` | +| `cmd:double_click_current` | `{}` | `{ ok: true }` | +| `cmd:type` | `{ text }` | `{ ok: true }` | +| `cmd:key` | `{ key }` | `{ ok: true }` | + +Auth on connect: `{ auth: { token } }`, where `token` must match the server's +`WINDOWS_AGENT_TOKEN`. The token is a constant baked into `agent.py` (`AGENT_TOKEN` near the +top) — set it before building for a real office, staff never see or type it. The server tells +front-desk PCs apart by connection IP address (shown on the Type Agent page), not anything +the agent sends. + +## Local dev / testing + +```bash +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +.venv/bin/python agent.py +``` + +On first run it asks only for the Server URL, then saves it to `agent_config.json` next to +the script so it reconnects automatically after that. After that it has no window — just a +system tray icon (green = Connected, gray = Connecting, red = Disconnected). Click it (or +right-click for the same menu) for status, the server URL, "Settings..." (reopens the setup +window pre-filled with the current URL — close without submitting to leave it unchanged), and +Quit. + +`apps/Backend/scripts/fake-windows-agent.js` is a Node stand-in for this agent — useful for +testing the server-side bridge without a Windows PC or a real screen to click on. + +## Building DentalAgent.exe + +Must be run **on Windows** (PyInstaller doesn't cross-compile): + +```bat +py -m venv .venv +.venv\Scripts\pip install -r requirements.txt +.venv\Scripts\pyinstaller --onefile --windowed --name DentalAgent agent.py +``` + +Output: `dist\DentalAgent.exe`. Staff download and run it once — no other setup. diff --git a/apps/WindowsAgent/agent.py b/apps/WindowsAgent/agent.py new file mode 100644 index 00000000..c44316ba --- /dev/null +++ b/apps/WindowsAgent/agent.py @@ -0,0 +1,218 @@ +""" +DentalAgent — runs on a staff front-desk Windows PC. Connects to the app's server over +the /agent socket.io namespace and executes low-level mouse/keyboard/screenshot commands +sent by the server. It has no knowledge of dental software, patients, or workflows — the +app's AI decides what to click/type by reading the screenshots this agent sends back. + +Configuration (via a small popup on first run, saved to agent_config.json next to the exe): + - Server URL (e.g. http://192.168.0.240:5000) — the only thing staff need to enter. + +The server identifies which PC is which by connection IP address (shown on the Type Agent +page), so there's no Office ID to type. The AGENT_TOKEN below authenticates the exe itself +to the server — set it to match the server's WINDOWS_AGENT_TOKEN before distributing this +to real front-desk PCs; it's baked into the build, not something staff ever see or type. + +After connecting it has no visible window — just a system tray icon showing +Connected/Connecting/Disconnected. Clicking the icon reopens the same setup window +(pre-filled with the current URL) without restarting the process — a self-relaunch turned +out to be unreliable with PyInstaller --onefile builds on Windows (the freshly spawned copy +can crash trying to reuse the outgoing process's temp extraction folder), so instead a single +hidden Tk window is kept alive for the whole run and just shown/hidden as needed. +""" + +AGENT_TOKEN = "dev-windows-agent-token" + +import base64 +import io +import json +import os +import sys +import threading +import tkinter as tk +from tkinter import messagebox + +import pyautogui +import pystray +import socketio +from PIL import Image, ImageDraw, ImageGrab + +CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "agent_config.json") + +pyautogui.FAILSAFE = True # moving mouse to a screen corner aborts an in-progress action + + +def load_config(): + if os.path.exists(CONFIG_PATH): + with open(CONFIG_PATH, "r") as f: + return json.load(f) + return {} + + +def save_config(config): + with open(CONFIG_PATH, "w") as f: + json.dump(config, f) + + +def make_status_icon(color): + image = Image.new("RGBA", (64, 64), (0, 0, 0, 0)) + draw = ImageDraw.Draw(image) + draw.ellipse((8, 8, 56, 56), fill=color) + return image + + +ICON_CONNECTED = make_status_icon("#22c55e") +ICON_DISCONNECTED = make_status_icon("#ef4444") +ICON_CONNECTING = make_status_icon("#9ca3af") +ICON_NOT_CONFIGURED = make_status_icon("#9ca3af") + + +class DentalAgent: + def __init__(self): + self.server_url = None + self.status = "Not configured" + self.tray_icon = None + self.sio = socketio.Client(reconnection=True, reconnection_attempts=0) + self._register_handlers() + + def _set_status(self, status, icon_image): + self.status = status + if self.tray_icon: + self.tray_icon.icon = icon_image + self.tray_icon.title = f"Dental Agent - {status}" + self.tray_icon.update_menu() + + def _register_handlers(self): + sio = self.sio + + @sio.event(namespace="/agent") + def connect(): + print(f"Connected to {self.server_url}") + self._set_status("Connected", ICON_CONNECTED) + + @sio.event(namespace="/agent") + def connect_error(data): + print(f"Connection failed: {data}") + self._set_status("Connection failed", ICON_DISCONNECTED) + + @sio.event(namespace="/agent") + def disconnect(): + print("Disconnected") + self._set_status("Disconnected", ICON_DISCONNECTED) + + @sio.on("cmd:screenshot", namespace="/agent") + def on_screenshot(_data=None): + image = ImageGrab.grab() + buf = io.BytesIO() + image.save(buf, format="PNG") + encoded = base64.b64encode(buf.getvalue()).decode("ascii") + return {"image": encoded} + + @sio.on("cmd:click", namespace="/agent") + def on_click(data): + pyautogui.click(data["x"], data["y"]) + return {"ok": True} + + @sio.on("cmd:double_click", namespace="/agent") + def on_double_click(data): + pyautogui.doubleClick(data["x"], data["y"]) + return {"ok": True} + + @sio.on("cmd:double_click_current", namespace="/agent") + def on_double_click_current(_data=None): + # No x/y — clicks wherever the mouse already is. Lets staff pre-position the + # cursor over the target (e.g. an open time slot) before minimizing the browser, + # so this step doesn't need a screenshot/vision lookup at all. + pyautogui.doubleClick() + return {"ok": True} + + @sio.on("cmd:type", namespace="/agent") + def on_type(data): + pyautogui.write(data["text"], interval=0.02) + return {"ok": True} + + @sio.on("cmd:key", namespace="/agent") + def on_key(data): + pyautogui.press(data["key"]) + return {"ok": True} + + def connect_to(self, server_url): + """(Re)connects to server_url, reusing the same socketio.Client for the process's + whole lifetime — safe to call again later with a new URL to switch servers.""" + self.server_url = server_url + self._set_status("Connecting...", ICON_CONNECTING) + + def _run(): + try: + if self.sio.connected: + self.sio.disconnect() + self.sio.connect( + server_url, + namespaces=["/agent"], + auth={"token": AGENT_TOKEN}, + wait_timeout=10, + ) + self.sio.wait() + except Exception as exc: + print(f"Agent stopped: {exc}") + self._set_status("Connection failed", ICON_DISCONNECTED) + + threading.Thread(target=_run, daemon=True).start() + + +def main(): + config = load_config() + agent = DentalAgent() + + # --- Settings window: built once, shown/hidden for the process's whole lifetime --- + root = tk.Tk() + root.title("Dental Agent Setup") + root.resizable(False, False) + root.withdraw() + root.protocol("WM_DELETE_WINDOW", root.withdraw) # closing the window just hides it + + tk.Label(root, text="Server URL").pack(pady=(15, 0)) + server_entry = tk.Entry(root, width=40) + server_entry.pack(padx=15) + + def on_connect(): + server_url = server_entry.get().strip() + if not server_url: + messagebox.showerror("Dental Agent", "Server URL is required.") + return + save_config({"server_url": server_url}) + root.withdraw() + agent.connect_to(server_url) + + tk.Button(root, text="Connect", command=on_connect).pack(pady=15) + root.bind("", lambda _event: on_connect()) + + def show_settings(): + server_entry.delete(0, tk.END) + server_entry.insert(0, agent.server_url or config.get("server_url", "http://")) + root.deiconify() + root.eval("tk::PlaceWindow . center") + root.lift() + root.focus_force() + + # --- Tray icon: runs on its own thread so Tk can own the main thread/event loop --- + menu = pystray.Menu( + pystray.MenuItem(lambda _item: f"Status: {agent.status}", None, enabled=False), + pystray.MenuItem(lambda _item: f"Server: {agent.server_url or '(not set)'}", None, enabled=False), + pystray.Menu.SEPARATOR, + pystray.MenuItem("Settings...", lambda: root.after(0, show_settings), default=True), + pystray.MenuItem("Quit", lambda icon: (icon.stop(), root.after(0, root.quit))), + ) + icon = pystray.Icon("DentalAgent", ICON_NOT_CONFIGURED, "Dental Agent - Not configured", menu) + agent.tray_icon = icon + threading.Thread(target=icon.run, daemon=True).start() + + if "server_url" in config: + agent.connect_to(config["server_url"]) + else: + show_settings() # first run — nothing saved yet, ask right away + + root.mainloop() + + +if __name__ == "__main__": + main() diff --git a/apps/WindowsAgent/package.json b/apps/WindowsAgent/package.json new file mode 100644 index 00000000..bef4ed8e --- /dev/null +++ b/apps/WindowsAgent/package.json @@ -0,0 +1,4 @@ +{ + "name": "windowsagent", + "private": true +} diff --git a/apps/WindowsAgent/requirements.txt b/apps/WindowsAgent/requirements.txt new file mode 100644 index 00000000..7f47cf90 --- /dev/null +++ b/apps/WindowsAgent/requirements.txt @@ -0,0 +1,7 @@ +python-socketio[client]==5.13.0 +python-engineio==4.11.2 +websocket-client==1.8.0 +pyautogui==0.9.54 +Pillow==11.1.0 +pystray==0.19.5 +pyinstaller==6.11.1 diff --git a/package-lock.json b/package-lock.json index ffe4beb3..d7d9a0e6 100755 --- a/package-lock.json +++ b/package-lock.json @@ -57,6 +57,7 @@ "passport": "^0.7.0", "passport-local": "^1.0.0", "pdfkit": "^0.17.2", + "sharp": "^0.35.3", "socket.io": "^4.8.1", "twilio": "^6.0.0", "ws": "^8.18.0", @@ -212,6 +213,9 @@ "extraneous": true, "hasInstallScript": true }, + "apps/WindowsAgent": { + "name": "windowsagent" + }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -653,6 +657,16 @@ "@electric-sql/pglite": "0.3.15" } }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", @@ -1346,6 +1360,506 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.3.tgz", + "integrity": "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.3.tgz", + "integrity": "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.3.tgz", + "integrity": "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.2.tgz", + "integrity": "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.2.tgz", + "integrity": "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.2.tgz", + "integrity": "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.2.tgz", + "integrity": "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.2.tgz", + "integrity": "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.2.tgz", + "integrity": "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.2.tgz", + "integrity": "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.2.tgz", + "integrity": "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.2.tgz", + "integrity": "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.2.tgz", + "integrity": "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.3.tgz", + "integrity": "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.3.tgz", + "integrity": "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.3.tgz", + "integrity": "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.3.tgz", + "integrity": "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.3.tgz", + "integrity": "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.3.tgz", + "integrity": "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.3.tgz", + "integrity": "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.3.tgz", + "integrity": "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", + "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.3.tgz", + "integrity": "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.3.tgz", + "integrity": "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.3.tgz", + "integrity": "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.3.tgz", + "integrity": "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@ioredis/commands": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz", @@ -12663,6 +13177,67 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.3.tgz", + "integrity": "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==", + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.3", + "@img/sharp-darwin-x64": "0.35.3", + "@img/sharp-freebsd-wasm32": "0.35.3", + "@img/sharp-libvips-darwin-arm64": "1.3.2", + "@img/sharp-libvips-darwin-x64": "1.3.2", + "@img/sharp-libvips-linux-arm": "1.3.2", + "@img/sharp-libvips-linux-arm64": "1.3.2", + "@img/sharp-libvips-linux-ppc64": "1.3.2", + "@img/sharp-libvips-linux-riscv64": "1.3.2", + "@img/sharp-libvips-linux-s390x": "1.3.2", + "@img/sharp-libvips-linux-x64": "1.3.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", + "@img/sharp-libvips-linuxmusl-x64": "1.3.2", + "@img/sharp-linux-arm": "0.35.3", + "@img/sharp-linux-arm64": "0.35.3", + "@img/sharp-linux-ppc64": "0.35.3", + "@img/sharp-linux-riscv64": "0.35.3", + "@img/sharp-linux-s390x": "0.35.3", + "@img/sharp-linux-x64": "0.35.3", + "@img/sharp-linuxmusl-arm64": "0.35.3", + "@img/sharp-linuxmusl-x64": "0.35.3", + "@img/sharp-webcontainers-wasm32": "0.35.3", + "@img/sharp-win32-arm64": "0.35.3", + "@img/sharp-win32-ia32": "0.35.3", + "@img/sharp-win32-x64": "0.35.3" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -14913,6 +15488,10 @@ "node": ">=8" } }, + "node_modules/windowsagent": { + "resolved": "apps/WindowsAgent", + "link": true + }, "node_modules/wmf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",