feat: AI Type Agent - vision-based Open Dental automation via Windows agent

Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend
services to locate UI elements via vision and drive an existing-patient
appointment flow in Open Dental, wired into the Copy/Type Agent page and
socket progress updates.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

apps/Backend/src/services/typeAgentRunner.ts

@@ -0,0 +1,138 @@
+import { captureScreenshot, click, doubleClick, doubleClickCurrent, typeText } from "./windowsAgentBridge";
+import { locateOnScreenshot } from "./visionLocate";
+
+export type StepStatus = "running" | "done" | "error";
+export type ProgressCallback = (
+  index: number,
+  total: number,
+  label: string,
+  status: StepStatus,
+  error?: string
+) => void;
+
+interface RunContext {
+  ip: string;
+  userId: number;
+  patientLastName: string;
+  patientFirstName: string;
+}
+
+interface RunStep {
+  label: string;
+  execute: (ctx: RunContext) => Promise<void>;
+  // Time to let the target app's UI settle/render before the next step's screenshot.
+  delayAfterMs: number;
+}
+
+async function visionClick(ctx: RunContext, goal: string, opts?: { doubleClick?: boolean }) {
+  const { image } = await captureScreenshot(ctx.ip);
+  const loc = await locateOnScreenshot(ctx.userId, image, goal);
+  if (!loc) throw new Error(`Could not find on screen: ${goal}`);
+  console.log(`[type-agent] ${opts?.doubleClick ? "double-click" : "click"} at (${loc.x}, ${loc.y}) — goal: ${goal.slice(0, 80)}...`);
+  if (opts?.doubleClick) {
+    await doubleClick(loc.x, loc.y, ctx.ip);
+  } else {
+    await click(loc.x, loc.y, ctx.ip);
+  }
+}
+
+// Open Dental, "make an appointment for an existing patient" — the exact sequence confirmed
+// against a real Open Dental instance: double-click the pre-positioned schedule slot (opens
+// Select Patient) → type last name → double-click the matching patient row (opens Edit
+// Appointment with the appointment already created) → click the Exam procedure → click Save.
+const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
+  {
+    label: "Double-click the schedule at the cursor position",
+    execute: async (ctx) => {
+      await doubleClickCurrent(ctx.ip);
+    },
+    delayAfterMs: 1000, // Select Patient window opening
+  },
+  {
+    label: "Find and click the Last Name field",
+    execute: async (ctx) => {
+      await visionClick(
+        ctx,
+        "the Last Name text input field in the Select Patient window — it's in the \"Search by:\" " +
+          "panel on the RIGHT half of the window (if you split the window into a left half and a " +
+          "right half), not in the results grid on the left"
+      );
+    },
+    delayAfterMs: 300,
+  },
+  {
+    label: "Type the patient's last name",
+    execute: async (ctx) => {
+      await typeText(ctx.patientLastName, ctx.ip);
+    },
+    delayAfterMs: 600, // patient list filtering
+  },
+  {
+    label: "Find and double-click the matching patient row",
+    execute: async (ctx) => {
+      const lastPrefix = ctx.patientLastName.slice(0, 3);
+      const firstPrefix = ctx.patientFirstName.slice(0, 3);
+      await visionClick(
+        ctx,
+        `Search the LEFT half of the Select Patient window for the row of patient data whose ` +
+          `LastName column starts with "${lastPrefix}" AND whose First Name column starts with ` +
+          `"${firstPrefix}" — matching is case-insensitive and only needs to match the first 3 ` +
+          `letters of each name, not the full name exactly. Both the last name prefix and first name ` +
+          `prefix must match — there may be other rows with a matching last name (family members, ` +
+          `e.g. a sibling or spouse) but a different first name; do NOT click those, they are the ` +
+          `wrong patient. That row is likely shaded/highlighted in LIGHT BLUE as the currently-selected ` +
+          `row. Do NOT click the window's own title bar — a DARK navy-blue bar at the very top of the ` +
+          `window that just reads "Select Patient" in white text; that is window chrome, not a patient ` +
+          `row, and is well above the results grid. Also do not click the column header row (labels ` +
+          `like "PatNum", "LastName", "First Name"). Click on the actual patient text in the matching data row.`,
+        { doubleClick: true }
+      );
+    },
+    delayAfterMs: 1200, // Edit Appointment window opening
+  },
+  {
+    label: "Find and click the Exam procedure",
+    execute: async (ctx) => {
+      await visionClick(
+        ctx,
+        'the "Exam" item in the procedures list of the Edit Appointment window — it\'s in the ' +
+          "UPPER-MIDDLE area of the window (if you split the window into an upper half and a lower " +
+          "half, and again into left/middle/right thirds, it's in the upper half, middle third), " +
+          'in a plain text list that also contains items like "Ex,Pro,Flo", "Prophy-Adult", "Pano"'
+      );
+    },
+    delayAfterMs: 500,
+  },
+  {
+    label: "Find and click Save",
+    execute: async (ctx) => {
+      await visionClick(ctx, "the Save button in the lower right corner of the Edit Appointment window");
+    },
+    delayAfterMs: 300,
+  },
+];
+
+export async function runExistingPatientOpenDental(
+  userId: number,
+  ip: string,
+  patientLastName: string,
+  patientFirstName: string,
+  onProgress: ProgressCallback
+): Promise<void> {
+  const ctx: RunContext = { ip, userId, patientLastName, patientFirstName };
+  const steps = OPEN_DENTAL_EXISTING_PATIENT;
+
+  for (let i = 0; i < steps.length; i++) {
+    const step = steps[i]!;
+    onProgress(i, steps.length, step.label, "running");
+    try {
+      await step.execute(ctx);
+      onProgress(i, steps.length, step.label, "done");
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      onProgress(i, steps.length, step.label, "error", message);
+      throw err;
+    }
+    await new Promise((resolve) => setTimeout(resolve, step.delayAfterMs));
+  }
+}

apps/Backend/src/services/visionLocate.ts

@@ -0,0 +1,96 @@
+import { storage } from "../storage";
+import { resolveAiProvider, getLlm } from "../ai/llm-factory";
+
+interface ParsedLocation {
+  found: boolean;
+  x?: number;
+  y?: number;
+}
+
+// Parses the model's response into {found, x, y}, tolerating near-miss JSON rather than
+// only accepting strictly valid JSON. Seen in practice: {"found": true, "x": 1023, 187} —
+// a dropped "y" key with the value still present positionally. Since this response should
+// only ever contain two numbers (the coordinates), falling back to "first two numbers after
+// found" is safe and recovers without burning another AI call.
+function parseLocationResponse(raw: string): ParsedLocation | null {
+  const match = raw.match(/\{[\s\S]*\}/);
+  if (!match) return null;
+  const block = match[0];
+
+  try {
+    return JSON.parse(block) as ParsedLocation;
+  } catch {
+    // fall through to lenient repair below
+  }
+
+  const foundMatch = block.match(/"found"\s*:\s*(true|false)/i);
+  if (!foundMatch) return null;
+  const found = foundMatch[1]!.toLowerCase() === "true";
+  if (!found) return { found: false };
+
+  const afterFound = block.slice(foundMatch.index! + foundMatch[0].length);
+  const numbers = afterFound.match(/-?\d+(\.\d+)?/g);
+  if (!numbers || numbers.length < 2) return null;
+
+  return { found: true, x: Number(numbers[0]), y: Number(numbers[1]) };
+}
+
+// Asks the configured vision AI to find a described UI element in a screenshot and return
+// its pixel coordinates — the core primitive the Type Agent step runner clicks/types through.
+// Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info.
+export async function locateOnScreenshot(
+  userId: number,
+  imageBase64: string,
+  goal: string
+): Promise<{ x: number; y: number } | null> {
+  const aiSettings = await storage.getAiSettings(userId);
+  const activeAi = resolveAiProvider(aiSettings ?? {});
+  if (!activeAi) {
+    throw new Error("AI is not configured. Add an API key in AI Settings.");
+  }
+  if (activeAi.provider !== "claude") {
+    throw new Error("Vision-guided steps require Claude to be the active AI provider.");
+  }
+
+  const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
+
+  const content = [
+    {
+      type: "text",
+      text:
+        `This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` +
+        "Respond with strict JSON only, no prose, no markdown fences: " +
+        '{"found": true, "x": <pixel x>, "y": <pixel y>} if you can locate it (coordinates must be ' +
+        'pixel positions within this exact image, at the center of the element), or {"found": false} ' +
+        "if it isn't visible in this screenshot.",
+    },
+    {
+      type: "image_url",
+      image_url: { url: `data:image/png;base64,${imageBase64}` },
+    },
+  ];
+
+  // A response that's fully unparseable even after the lenient repair above is worth one
+  // retry (rare model hiccup); anything the repair can salvage is used immediately, no
+  // retry needed.
+  const MAX_ATTEMPTS = 2;
+  let lastError: Error | null = null;
+
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    const response = await llm.invoke([{ role: "user", content }] as any);
+    const raw = String(response.content).trim();
+
+    const parsed = parseLocationResponse(raw);
+    if (!parsed) {
+      lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
+      continue;
+    }
+
+    if (!parsed.found || typeof parsed.x !== "number" || typeof parsed.y !== "number") {
+      return null;
+    }
+    return { x: Math.round(parsed.x), y: Math.round(parsed.y) };
+  }
+
+  throw lastError ?? new Error("AI returned an unparseable response");
+}

apps/Backend/src/services/windowsAgentBridge.ts

@@ -0,0 +1,58 @@
+import { io } from "../socket";
+
+const COMMAND_TIMEOUT_MS = 10000;
+
+// Picks a connected Windows Agent socket by IP address — falls back to the first connected
+// agent if none is given (fine while offices only ever have one PC connected).
+function getAgentSocket(ip?: string) {
+  if (!io) throw new Error("Socket.io not initialized");
+  const namespaceSockets = Array.from(io.of("/agent").sockets.values());
+  const target = ip
+    ? namespaceSockets.find((s) => {
+        const socketIp = (s.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim()
+          || s.handshake.address;
+        return socketIp === ip;
+      })
+    : namespaceSockets[0];
+  if (!target) throw new Error("No Windows Agent connected");
+  return target;
+}
+
+async function sendCommand<T>(event: string, payload: unknown, ip?: string): Promise<T> {
+  const socket = getAgentSocket(ip);
+  return socket.timeout(COMMAND_TIMEOUT_MS).emitWithAck(event, payload) as Promise<T>;
+}
+
+// Captures the Windows PC's screen. Returns a base64-encoded PNG for the app's AI to read.
+export function captureScreenshot(ip?: string) {
+  return sendCommand<{ image: string }>("cmd:screenshot", {}, ip);
+}
+
+export function click(x: number, y: number, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:click", { x, y }, ip);
+}
+
+export function doubleClick(x: number, y: number, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:double_click", { x, y }, ip);
+}
+
+// Double-clicks wherever the mouse already is — no coordinates needed. Used for the first
+// step of a run, where staff pre-position the cursor before triggering it from the browser.
+export function doubleClickCurrent(ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:double_click_current", {}, ip);
+}
+
+export function typeText(text: string, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:type", { text }, ip);
+}
+
+export function pressKey(key: string, ip?: string) {
+  return sendCommand<{ ok: boolean }>("cmd:key", { key }, ip);
+}
+
+// Forcibly closes a connected agent's socket — used by the "x" on the Type Agent page to
+// let staff stop controlling a PC without touching the exe on that machine.
+export function disconnectAgent(ip: string) {
+  const socket = getAgentSocket(ip);
+  socket.disconnect(true);
+}