feat: AI Type Agent - vision-based Open Dental automation via Windows agent

Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend
services to locate UI elements via vision and drive an existing-patient
appointment flow in Open Dental, wired into the Copy/Type Agent page and
socket progress updates.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

apps/Backend/src/routes/index.ts

@@ -45,6 +45,7 @@ import licenseRoutes from "./license";
 import insuranceStatusBcbsMaRoutes from "./insuranceStatusBcbsMa";
 import labRxRoutes from "./lab-rx";
 import seleniumSettingsRoutes from "./selenium-settings";
+import typeAgentRoutes from "./type-agent";
 
 const router = Router();
 
@@ -82,6 +83,7 @@ router.use("/cloud-storage", cloudStorageRoutes);
 router.use("/payments-reports", paymentsReportsRoutes);
 router.use("/export-payments-reports", exportPaymentsReportsRoutes);
 router.use("/job-monitor", jobMonitorRoutes);
+router.use("/type-agent", typeAgentRoutes);
 router.use("/twilio", twilioRoutes);
 router.use("/ai", aiSettingsRoutes);
 router.use("/office-hours", officeHoursRoutes);

apps/Backend/src/routes/type-agent.ts

@@ -0,0 +1,109 @@
+import crypto from "crypto";
+import { Router, Request, Response } from "express";
+import { getAgentStatus, io } from "../socket";
+import { captureScreenshot, click, disconnectAgent, doubleClick, doubleClickCurrent, pressKey, typeText } from "../services/windowsAgentBridge";
+import { runExistingPatientOpenDental } from "../services/typeAgentRunner";
+import { storage } from "../storage";
+
+const router = Router();
+
+// GET /api/type-agent/status
+// Current Windows Agent (DentalAgent.exe) connection status, for the initial page load —
+// live updates after that come over the "type-agent:status" socket.io event.
+router.get("/status", (_req: Request, res: Response) => {
+  res.json(getAgentStatus());
+});
+
+// POST /api/type-agent/disconnect
+// Forcibly disconnects a connected agent by IP — the "x" on the Type Agent page.
+router.post("/disconnect", (req: Request, res: Response) => {
+  const { ip } = req.body ?? {};
+  if (!ip) {
+    res.status(400).json({ error: "ip is required" });
+    return;
+  }
+  try {
+    disconnectAgent(ip);
+    res.json({ success: true });
+  } catch (err) {
+    res.status(404).json({ error: err instanceof Error ? err.message : "Agent not found" });
+  }
+});
+
+// POST /api/type-agent/test-command
+// Sends a single low-level command (screenshot/click/double_click/type/key) straight to a
+// connected Windows Agent. Exists to verify the command bridge before the AI-driven step
+// orchestration is wired in — not meant to be called from the Type Agent UI directly.
+router.post("/test-command", async (req: Request, res: Response) => {
+  const { type, x, y, text, key, ip } = req.body ?? {};
+  try {
+    let result;
+    switch (type) {
+      case "screenshot":
+        result = await captureScreenshot(ip);
+        break;
+      case "click":
+        result = await click(Number(x), Number(y), ip);
+        break;
+      case "double_click":
+        result = await doubleClick(Number(x), Number(y), ip);
+        break;
+      case "double_click_current":
+        result = await doubleClickCurrent(ip);
+        break;
+      case "type":
+        result = await typeText(String(text ?? ""), ip);
+        break;
+      case "key":
+        result = await pressKey(String(key ?? ""), ip);
+        break;
+      default:
+        res.status(400).json({ error: "Unknown command type" });
+        return;
+    }
+    res.json({ success: true, result });
+  } catch (err) {
+    res.status(500).json({ error: err instanceof Error ? err.message : "Command failed" });
+  }
+});
+
+// POST /api/type-agent/run
+// Kicks off a real step-by-step run against a connected Windows Agent. Only Open Dental /
+// existing-patient is wired up so far — everything else the Type Agent page still shows as
+// a preview only. Responds immediately with a runId; progress streams over the
+// "type-agent:run:<runId>" socket.io event so the browser doesn't have to hold the request
+// open for what can be a slow, multi-step, vision-driven sequence.
+router.post("/run", async (req: Request, res: Response) => {
+  const userId = req.user?.id;
+  if (!userId) {
+    res.status(401).json({ error: "Unauthorized" });
+    return;
+  }
+
+  const { software, action, ip, patientId } = req.body ?? {};
+  if (software !== "open-dental" || action !== "existing-patient") {
+    res.status(400).json({ error: "This software/action combination isn't automated yet." });
+    return;
+  }
+  if (!ip) {
+    res.status(400).json({ error: "ip is required" });
+    return;
+  }
+
+  const patient = await storage.getPatient(Number(patientId));
+  if (!patient?.lastName || !patient?.firstName) {
+    res.status(400).json({ error: "Patient not found or missing a first/last name" });
+    return;
+  }
+
+  const runId = crypto.randomUUID();
+  res.json({ runId });
+
+  runExistingPatientOpenDental(userId, ip, patient.lastName, patient.firstName, (index, total, label, status, error) => {
+    io?.emit(`type-agent:run:${runId}`, { index, total, label, status, error });
+  }).catch((err) => {
+    console.error("[type-agent/run] failed:", err);
+  });
+});
+
+export default router;

apps/Backend/src/services/typeAgentRunner.ts

@@ -0,0 +1,138 @@
+import { captureScreenshot, click, doubleClick, doubleClickCurrent, typeText } from "./windowsAgentBridge";
+import { locateOnScreenshot } from "./visionLocate";
+
+export type StepStatus = "running" | "done" | "error";
+export type ProgressCallback = (
+  index: number,
+  total: number,
+  label: string,
+  status: StepStatus,
+  error?: string
+) => void;
+
+interface RunContext {
+  ip: string;
+  userId: number;
+  patientLastName: string;
+  patientFirstName: string;
+}
+
+interface RunStep {
+  label: string;
+  execute: (ctx: RunContext) => Promise<void>;
+  // Time to let the target app's UI settle/render before the next step's screenshot.
+  delayAfterMs: number;
+}
+
+async function visionClick(ctx: RunContext, goal: string, opts?: { doubleClick?: boolean }) {
+  const { image } = await captureScreenshot(ctx.ip);
+  const loc = await locateOnScreenshot(ctx.userId, image, goal);
+  if (!loc) throw new Error(`Could not find on screen: ${goal}`);
+  console.log(`[type-agent] ${opts?.doubleClick ? "double-click" : "click"} at (${loc.x}, ${loc.y}) — goal: ${goal.slice(0, 80)}...`);
+  if (opts?.doubleClick) {
+    await doubleClick(loc.x, loc.y, ctx.ip);
+  } else {
+    await click(loc.x, loc.y, ctx.ip);
+  }
+}
+
+// Open Dental, "make an appointment for an existing patient" — the exact sequence confirmed
+// against a real Open Dental instance: double-click the pre-positioned schedule slot (opens
+// Select Patient) → type last name → double-click the matching patient row (opens Edit
+// Appointment with the appointment already created) → click the Exam procedure → click Save.
+const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
+  {
+    label: "Double-click the schedule at the cursor position",
+    execute: async (ctx) => {
+      await doubleClickCurrent(ctx.ip);
+    },
+    delayAfterMs: 1000, // Select Patient window opening
+  },
+  {
+    label: "Find and click the Last Name field",
+    execute: async (ctx) => {
+      await visionClick(
+        ctx,
+        "the Last Name text input field in the Select Patient window — it's in the \"Search by:\" " +
+          "panel on the RIGHT half of the window (if you split the window into a left half and a " +
+          "right half), not in the results grid on the left"
+      );
+    },
+    delayAfterMs: 300,
+  },
+  {
+    label: "Type the patient's last name",
+    execute: async (ctx) => {
+      await typeText(ctx.patientLastName, ctx.ip);
+    },
+    delayAfterMs: 600, // patient list filtering
+  },
+  {
+    label: "Find and double-click the matching patient row",
+    execute: async (ctx) => {
+      const lastPrefix = ctx.patientLastName.slice(0, 3);
+      const firstPrefix = ctx.patientFirstName.slice(0, 3);
+      await visionClick(
+        ctx,
+        `Search the LEFT half of the Select Patient window for the row of patient data whose ` +
+          `LastName column starts with "${lastPrefix}" AND whose First Name column starts with ` +
+          `"${firstPrefix}" — matching is case-insensitive and only needs to match the first 3 ` +
+          `letters of each name, not the full name exactly. Both the last name prefix and first name ` +
+          `prefix must match — there may be other rows with a matching last name (family members, ` +
+          `e.g. a sibling or spouse) but a different first name; do NOT click those, they are the ` +
+          `wrong patient. That row is likely shaded/highlighted in LIGHT BLUE as the currently-selected ` +
+          `row. Do NOT click the window's own title bar — a DARK navy-blue bar at the very top of the ` +
+          `window that just reads "Select Patient" in white text; that is window chrome, not a patient ` +
+          `row, and is well above the results grid. Also do not click the column header row (labels ` +
+          `like "PatNum", "LastName", "First Name"). Click on the actual patient text in the matching data row.`,
+        { doubleClick: true }
+      );
+    },
+    delayAfterMs: 1200, // Edit Appointment window opening
+  },
+  {
+    label: "Find and click the Exam procedure",
+    execute: async (ctx) => {
+      await visionClick(
+        ctx,
+        'the "Exam" item in the procedures list of the Edit Appointment window — it\'s in the ' +
+          "UPPER-MIDDLE area of the window (if you split the window into an upper half and a lower " +
+          "half, and again into left/middle/right thirds, it's in the upper half, middle third), " +
+          'in a plain text list that also contains items like "Ex,Pro,Flo", "Prophy-Adult", "Pano"'
+      );
+    },
+    delayAfterMs: 500,
+  },
+  {
+    label: "Find and click Save",
+    execute: async (ctx) => {
+      await visionClick(ctx, "the Save button in the lower right corner of the Edit Appointment window");
+    },
+    delayAfterMs: 300,
+  },
+];
+
+export async function runExistingPatientOpenDental(
+  userId: number,
+  ip: string,
+  patientLastName: string,
+  patientFirstName: string,
+  onProgress: ProgressCallback
+): Promise<void> {
+  const ctx: RunContext = { ip, userId, patientLastName, patientFirstName };
+  const steps = OPEN_DENTAL_EXISTING_PATIENT;
+
+  for (let i = 0; i < steps.length; i++) {
+    const step = steps[i]!;
+    onProgress(i, steps.length, step.label, "running");
+    try {
+      await step.execute(ctx);
+      onProgress(i, steps.length, step.label, "done");
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      onProgress(i, steps.length, step.label, "error", message);
+      throw err;
+    }
+    await new Promise((resolve) => setTimeout(resolve, step.delayAfterMs));
+  }
+}

apps/Backend/src/services/visionLocate.ts

@@ -0,0 +1,96 @@
+import { storage } from "../storage";
+import { resolveAiProvider, getLlm } from "../ai/llm-factory";
+
+interface ParsedLocation {
+  found: boolean;
+  x?: number;
+  y?: number;
+}
+
+// Parses the model's response into {found, x, y}, tolerating near-miss JSON rather than
+// only accepting strictly valid JSON. Seen in practice: {"found": true, "x": 1023, 187} —
+// a dropped "y" key with the value still present positionally. Since this response should
+// only ever contain two numbers (the coordinates), falling back to "first two numbers after
+// found" is safe and recovers without burning another AI call.
+function parseLocationResponse(raw: string): ParsedLocation | null {
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) return null;
+  const block = match[0];
+
+  try {
+    return JSON.parse(block) as ParsedLocation;
+  } catch {
+    // fall through to lenient repair below
+  }
+
+  const foundMatch = block.match(/"found"\s*:\s*(true|false)/i);
+  if (!foundMatch) return null;
+  const found = foundMatch[1]!.toLowerCase() === "true";
+  if (!found) return { found: false };
+
+  const afterFound = block.slice(foundMatch.index! + foundMatch[0].length);
+  const numbers = afterFound.match(/-?\d+(\.\d+)?/g);
+  if (!numbers || numbers.length < 2) return null;
+
+  return { found: true, x: Number(numbers[0]), y: Number(numbers[1]) };
+}
+
+// Asks the configured vision AI to find a described UI element in a screenshot and return
+// its pixel coordinates — the core primitive the Type Agent step runner clicks/types through.
+// Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info.
+export async function locateOnScreenshot(
+  userId: number,
+  imageBase64: string,
+  goal: string
+): Promise<{ x: number; y: number } | null> {
+  const aiSettings = await storage.getAiSettings(userId);
+  const activeAi = resolveAiProvider(aiSettings ?? {});
+  if (!activeAi) {
+    throw new Error("AI is not configured. Add an API key in AI Settings.");
+  }
+  if (activeAi.provider !== "claude") {
+    throw new Error("Vision-guided steps require Claude to be the active AI provider.");
+  }
+
+  const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
+
+  const content = [
+    {
+      type: "text",
+      text:
+        `This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` +
+        "Respond with strict JSON only, no prose, no markdown fences: " +
+        '{"found": true, "x": <pixel x>, "y": <pixel y>} if you can locate it (coordinates must be ' +
+        'pixel positions within this exact image, at the center of the element), or {"found": false} ' +
+        "if it isn't visible in this screenshot.",
+    },
+    {
+      type: "image_url",
+      image_url: { url: `data:image/png;base64,${imageBase64}` },
+    },
+  ];
+
+  // A response that's fully unparseable even after the lenient repair above is worth one
+  // retry (rare model hiccup); anything the repair can salvage is used immediately, no
+  // retry needed.
+  const MAX_ATTEMPTS = 2;
+  let lastError: Error | null = null;
+
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    const response = await llm.invoke([{ role: "user", content }] as any);
+    const raw = String(response.content).trim();
+
+    const parsed = parseLocationResponse(raw);
+    if (!parsed) {
+      lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
+      continue;
+    }
+
+    if (!parsed.found || typeof parsed.x !== "number" || typeof parsed.y !== "number") {
+      return null;
+    }
+    return { x: Math.round(parsed.x), y: Math.round(parsed.y) };
+  }
+
+  throw lastError ?? new Error("AI returned an unparseable response");
+}

apps/Backend/src/services/windowsAgentBridge.ts

@@ -0,0 +1,58 @@
+import { io } from "../socket";
+
+const COMMAND_TIMEOUT_MS = 10000;
+
+// Picks a connected Windows Agent socket by IP address — falls back to the first connected
+// agent if none is given (fine while offices only ever have one PC connected).
+function getAgentSocket(ip?: string) {
+  if (!io) throw new Error("Socket.io not initialized");
+  const namespaceSockets = Array.from(io.of("/agent").sockets.values());
+  const target = ip
+    ? namespaceSockets.find((s) => {
+        const socketIp = (s.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim()
+          || s.handshake.address;
+        return socketIp === ip;
+      })
+    : namespaceSockets[0];
+  if (!target) throw new Error("No Windows Agent connected");
+  return target;
+}
+
+async function sendCommand<T>(event: string, payload: unknown, ip?: string): Promise<T> {
+  const socket = getAgentSocket(ip);
+  return socket.timeout(COMMAND_TIMEOUT_MS).emitWithAck(event, payload) as Promise<T>;
+}
+
+// Captures the Windows PC's screen. Returns a base64-encoded PNG for the app's AI to read.
+export function captureScreenshot(ip?: string) {
+  return sendCommand<{ image: string }>("cmd:screenshot", {}, ip);
+}
+
+export function click(x: number, y: number, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:click", { x, y }, ip);
+}
+
+export function doubleClick(x: number, y: number, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:double_click", { x, y }, ip);
+}
+
+// Double-clicks wherever the mouse already is — no coordinates needed. Used for the first
+// step of a run, where staff pre-position the cursor before triggering it from the browser.
+export function doubleClickCurrent(ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:double_click_current", {}, ip);
+}
+
+export function typeText(text: string, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:type", { text }, ip);
+}
+
+export function pressKey(key: string, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:key", { key }, ip);
+}
+
+// Forcibly closes a connected agent's socket — used by the "x" on the Type Agent page to
+// let staff stop controlling a PC without touching the exe on that machine.
+export function disconnectAgent(ip: string) {
+  const socket = getAgentSocket(ip);
+  socket.disconnect(true);
+}

apps/Backend/src/socket.ts

@@ -3,6 +3,22 @@ import { Server, Socket } from "socket.io";
 
 let io: Server | null = null;
 
+interface ConnectedAgent {
+  socketId: string;
+  ip: string;
+  connectedAt: string;
+}
+
+// Windows Agents (DentalAgent.exe) connected via the /agent namespace, keyed by socket id.
+const connectedAgents = new Map<string, ConnectedAgent>();
+
+export function getAgentStatus() {
+  return {
+    connected: connectedAgents.size > 0,
+    agents: Array.from(connectedAgents.values()),
+  };
+}
+
 export function initSocket(server: HttpServer) {
   const NODE_ENV = (
     process.env.NODE_ENV ||
@@ -47,7 +63,49 @@ export function initSocket(server: HttpServer) {
     console.error("Socket engine connection_error:", err);
   });
 
+  initAgentNamespace(io);
+
   return io;
 }
 
+// Windows Agent (DentalAgent.exe) bridge. Separate namespace so browser clients on the
+// default namespace never see agent traffic, and so we can gate it with a shared secret —
+// this connection will eventually be able to click/type on a real front-desk PC.
+function initAgentNamespace(server: Server) {
+  const agentNamespace = server.of("/agent");
+  const AGENT_TOKEN = process.env.WINDOWS_AGENT_TOKEN;
+
+  agentNamespace.use((socket, next) => {
+    if (!AGENT_TOKEN) {
+      // No token configured — refuse rather than silently accepting any connection.
+      return next(new Error("Windows Agent bridge is not configured (WINDOWS_AGENT_TOKEN unset)"));
+    }
+    if (socket.handshake.auth?.token !== AGENT_TOKEN) {
+      return next(new Error("Invalid agent token"));
+    }
+    next();
+  });
+
+  agentNamespace.on("connection", (socket: Socket) => {
+    // x-forwarded-for is only trustworthy behind a proxy we control (nginx in prod);
+    // falls back to the raw socket address for direct LAN connections in dev.
+    const ip = (socket.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim()
+      || socket.handshake.address;
+
+    connectedAgents.set(socket.id, {
+      socketId: socket.id,
+      ip,
+      connectedAt: new Date().toISOString(),
+    });
+    console.log("🖥️  Windows Agent connected:", socket.id, ip);
+    io?.emit("type-agent:status", getAgentStatus());
+
+    socket.on("disconnect", () => {
+      connectedAgents.delete(socket.id);
+      console.log("🖥️  Windows Agent disconnected:", socket.id);
+      io?.emit("type-agent:status", getAgentStatus());
+    });
+  });
+}
+
 export { io };