feat: AI Type Agent - vision-based Open Dental automation via Windows agent

Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend
services to locate UI elements via vision and drive an existing-patient
appointment flow in Open Dental, wired into the Copy/Type Agent page and
socket progress updates.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

apps/Backend/src/routes/index.ts

@@ -45,6 +45,7 @@ import licenseRoutes from "./license";
 import insuranceStatusBcbsMaRoutes from "./insuranceStatusBcbsMa";
 import labRxRoutes from "./lab-rx";
 import seleniumSettingsRoutes from "./selenium-settings";
+import typeAgentRoutes from "./type-agent";
 
 const router = Router();
 
@@ -82,6 +83,7 @@ router.use("/cloud-storage", cloudStorageRoutes);
 router.use("/payments-reports", paymentsReportsRoutes);
 router.use("/export-payments-reports", exportPaymentsReportsRoutes);
 router.use("/job-monitor", jobMonitorRoutes);
+router.use("/type-agent", typeAgentRoutes);
 router.use("/twilio", twilioRoutes);
 router.use("/ai", aiSettingsRoutes);
 router.use("/office-hours", officeHoursRoutes);

apps/Backend/src/routes/type-agent.ts

@@ -0,0 +1,109 @@
+import crypto from "crypto";
+import { Router, Request, Response } from "express";
+import { getAgentStatus, io } from "../socket";
+import { captureScreenshot, click, disconnectAgent, doubleClick, doubleClickCurrent, pressKey, typeText } from "../services/windowsAgentBridge";
+import { runExistingPatientOpenDental } from "../services/typeAgentRunner";
+import { storage } from "../storage";
+
+const router = Router();
+
+// GET /api/type-agent/status
+// Current Windows Agent (DentalAgent.exe) connection status, for the initial page load —
+// live updates after that come over the "type-agent:status" socket.io event.
+router.get("/status", (_req: Request, res: Response) => {
+  res.json(getAgentStatus());
+});
+
+// POST /api/type-agent/disconnect
+// Forcibly disconnects a connected agent by IP — the "x" on the Type Agent page.
+router.post("/disconnect", (req: Request, res: Response) => {
+  const { ip } = req.body ?? {};
+  if (!ip) {
+    res.status(400).json({ error: "ip is required" });
+    return;
+  }
+  try {
+    disconnectAgent(ip);
+    res.json({ success: true });
+  } catch (err) {
+    res.status(404).json({ error: err instanceof Error ? err.message : "Agent not found" });
+  }
+});
+
+// POST /api/type-agent/test-command
+// Sends a single low-level command (screenshot/click/double_click/type/key) straight to a
+// connected Windows Agent. Exists to verify the command bridge before the AI-driven step
+// orchestration is wired in — not meant to be called from the Type Agent UI directly.
+router.post("/test-command", async (req: Request, res: Response) => {
+  const { type, x, y, text, key, ip } = req.body ?? {};
+  try {
+    let result;
+    switch (type) {
+      case "screenshot":
+        result = await captureScreenshot(ip);
+        break;
+      case "click":
+        result = await click(Number(x), Number(y), ip);
+        break;
+      case "double_click":
+        result = await doubleClick(Number(x), Number(y), ip);
+        break;
+      case "double_click_current":
+        result = await doubleClickCurrent(ip);
+        break;
+      case "type":
+        result = await typeText(String(text ?? ""), ip);
+        break;
+      case "key":
+        result = await pressKey(String(key ?? ""), ip);
+        break;
+      default:
+        res.status(400).json({ error: "Unknown command type" });
+        return;
+    }
+    res.json({ success: true, result });
+  } catch (err) {
+    res.status(500).json({ error: err instanceof Error ? err.message : "Command failed" });
+  }
+});
+
+// POST /api/type-agent/run
+// Kicks off a real step-by-step run against a connected Windows Agent. Only Open Dental /
+// existing-patient is wired up so far — everything else the Type Agent page still shows as
+// a preview only. Responds immediately with a runId; progress streams over the
+// "type-agent:run:<runId>" socket.io event so the browser doesn't have to hold the request
+// open for what can be a slow, multi-step, vision-driven sequence.
+router.post("/run", async (req: Request, res: Response) => {
+  const userId = req.user?.id;
+  if (!userId) {
+    res.status(401).json({ error: "Unauthorized" });
+    return;
+  }
+
+  const { software, action, ip, patientId } = req.body ?? {};
+  if (software !== "open-dental" || action !== "existing-patient") {
+    res.status(400).json({ error: "This software/action combination isn't automated yet." });
+    return;
+  }
+  if (!ip) {
+    res.status(400).json({ error: "ip is required" });
+    return;
+  }
+
+  const patient = await storage.getPatient(Number(patientId));
+  if (!patient?.lastName || !patient?.firstName) {
+    res.status(400).json({ error: "Patient not found or missing a first/last name" });
+    return;
+  }
+
+  const runId = crypto.randomUUID();
+  res.json({ runId });
+
+  runExistingPatientOpenDental(userId, ip, patient.lastName, patient.firstName, (index, total, label, status, error) => {
+    io?.emit(`type-agent:run:${runId}`, { index, total, label, status, error });
+  }).catch((err) => {
+    console.error("[type-agent/run] failed:", err);
+  });
+});
+
+export default router;