From ad2e15ec62643ea216d3fc119529e2718cb601fa Mon Sep 17 00:00:00 2001 From: Gitead Date: Mon, 27 Jul 2026 08:50:47 -0400 Subject: [PATCH] feat: window-diff-based Type Agent locating for Open Dental appointment flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces fragile whole-screen text matching with a screenshot-diff step that finds each dialog's actual pixel bounds, then restricts every subsequent AI locate/click to that cropped region β€” eliminating false matches from text elsewhere on screen (title bars, side panels). Adds column-boundary and row-height detection so the patient row and Exam procedure click positions are computed geometrically instead of relying on repeated fuzzy AI guesses for visually similar neighbors. Also adds per-run screenshot/debug-crop backups and a structured run.log for diagnosing failed runs, plus a cmd:move primitive on the Windows agent for pre-click confirmation crops. Co-Authored-By: Claude Sonnet 5 --- apps/Backend/scripts/fake-windows-agent.js | 5 + apps/Backend/src/routes/type-agent.ts | 7 +- apps/Backend/src/services/typeAgentRunner.ts | 450 +++++++++++++++--- apps/Backend/src/services/visionLocate.ts | 388 ++++++++++++++- .../src/services/windowsAgentBridge.ts | 6 + apps/Backend/src/utils/screenshotBackup.ts | 35 ++ .../Frontend/src/pages/ai-type-agent-page.tsx | 15 +- apps/WindowsAgent/agent.py | 7 + 8 files changed, 817 insertions(+), 96 deletions(-) diff --git a/apps/Backend/scripts/fake-windows-agent.js b/apps/Backend/scripts/fake-windows-agent.js index dcb853c8..b2b3415a 100644 --- a/apps/Backend/scripts/fake-windows-agent.js +++ b/apps/Backend/scripts/fake-windows-agent.js @@ -29,6 +29,11 @@ socket.on("cmd:screenshot", (_payload, ack) => { ack({ image: FAKE_SCREENSHOT }); }); +socket.on("cmd:move", ({ x, y }, ack) => { + console.log(`πŸ–±οΈ move to (${x}, ${y}) (no click)`); + ack({ ok: true }); +}); + socket.on("cmd:click", ({ x, y }, ack) => { console.log(`πŸ–±οΈ click at (${x}, ${y})`); ack({ ok: true }); diff --git a/apps/Backend/src/routes/type-agent.ts b/apps/Backend/src/routes/type-agent.ts index 456611ae..a0dfc109 100644 --- a/apps/Backend/src/routes/type-agent.ts +++ b/apps/Backend/src/routes/type-agent.ts @@ -2,7 +2,7 @@ import crypto from "crypto"; import { Router, Request, Response } from "express"; import { getAgentStatus, io } from "../socket"; import { captureScreenshot, click, disconnectAgent, doubleClick, doubleClickCurrent, pressKey, typeText } from "../services/windowsAgentBridge"; -import { runExistingPatientOpenDental } from "../services/typeAgentRunner"; +import { runExistingPatientOpenDental, runExistingPatientOpenDentalRatioTest } from "../services/typeAgentRunner"; import { storage } from "../storage"; const router = Router(); @@ -80,7 +80,7 @@ router.post("/run", async (req: Request, res: Response) => { return; } - const { software, action, ip, patientId } = req.body ?? {}; + const { software, action, ip, patientId, ratioTest } = req.body ?? {}; if (software !== "open-dental" || action !== "existing-patient") { res.status(400).json({ error: "This software/action combination isn't automated yet." }); return; @@ -99,7 +99,8 @@ router.post("/run", async (req: Request, res: Response) => { const runId = crypto.randomUUID(); res.json({ runId }); - runExistingPatientOpenDental(userId, ip, patient.lastName, patient.firstName, (index, total, label, status, error) => { + const runFn = ratioTest ? runExistingPatientOpenDentalRatioTest : runExistingPatientOpenDental; + runFn(userId, runId, ip, patient.lastName, patient.firstName, (index, total, label, status, error) => { io?.emit(`type-agent:run:${runId}`, { index, total, label, status, error }); }).catch((err) => { console.error("[type-agent/run] failed:", err); diff --git a/apps/Backend/src/services/typeAgentRunner.ts b/apps/Backend/src/services/typeAgentRunner.ts index f598546c..3c2e1a1a 100644 --- a/apps/Backend/src/services/typeAgentRunner.ts +++ b/apps/Backend/src/services/typeAgentRunner.ts @@ -1,5 +1,17 @@ -import { captureScreenshot, click, doubleClick, doubleClickCurrent, typeText } from "./windowsAgentBridge"; -import { locateOnScreenshot } from "./visionLocate"; +import sharp from "sharp"; +import { captureScreenshot, click, doubleClick, doubleClickCurrent, moveMouse, typeText } from "./windowsAgentBridge"; +import { + confirmTextTyped, + cropAroundPoint, + cropToRegion, + detectColumnBoundaries, + detectRowHeightRatio, + diffBoundingBox, + locateAllOnScreenshot, + locateOnScreenshot, + WindowBounds, +} from "./visionLocate"; +import { backupTypeAgentScreenshot, logTypeAgentStep } from "../utils/screenshotBackup"; export type StepStatus = "running" | "done" | "error"; export type ProgressCallback = ( @@ -13,8 +25,15 @@ export type ProgressCallback = ( interface RunContext { ip: string; userId: number; + runId: string; patientLastName: string; patientFirstName: string; + // Set once a dialog's bounds are detected (via diffBoundingBox) β€” every locate/click after + // that point is restricted to this region until it's replaced by the next dialog's bounds. + windowBounds: WindowBounds | null; + // The screenshot taken right before the action that's expected to open the *next* dialog β€” + // stashed here so the following step can diff against it without re-capturing. + beforeNextWindow: string | null; } interface RunStep { @@ -24,23 +43,334 @@ interface RunStep { delayAfterMs: number; } -async function visionClick(ctx: RunContext, goal: string, opts?: { doubleClick?: boolean }) { +// Captures the current screen and, if a window has been detected, crops it down to just that +// window β€” so every locate call after that point only ever sees the relevant dialog, never the +// rest of the screen (the main window's title bar, side panels, etc.). +async function captureWindowScreenshot(ctx: RunContext): Promise<{ full: string; region: string }> { const { image } = await captureScreenshot(ctx.ip); - const loc = await locateOnScreenshot(ctx.userId, image, goal); - if (!loc) throw new Error(`Could not find on screen: ${goal}`); - console.log(`[type-agent] ${opts?.doubleClick ? "double-click" : "click"} at (${loc.x}, ${loc.y}) β€” goal: ${goal.slice(0, 80)}...`); + const region = ctx.windowBounds ? await cropToRegion(image, ctx.windowBounds) : image; + return { full: image, region }; +} + +// Translates a point from window-cropped-image coordinates back to real screen coordinates. +function toScreenPoint(ctx: RunContext, point: { x: number; y: number }): { x: number; y: number } { + if (!ctx.windowBounds) return point; + return { x: point.x + ctx.windowBounds.left, y: point.y + ctx.windowBounds.top }; +} + +// Same, for a whole list β€” used only when logging matches, so run.log always reports real +// screen coordinates (matching the full-screen debug screenshots) even though the underlying +// comparisons that pick a target run in cropped-image space. +function toScreenPoints(ctx: RunContext, points: { x: number; y: number }[]): { x: number; y: number }[] { + return points.map((p) => toScreenPoint(ctx, p)); +} + +// Keeps a point inside the currently-tracked window β€” every movement/click is restricted to +// this region once one is set, so a bad locate can't send the cursor wandering off into the +// rest of the app. +function clampToWindow(ctx: RunContext, x: number, y: number): { x: number; y: number } { + if (!ctx.windowBounds) return { x, y }; + const { left, top, width, height } = ctx.windowBounds; + return { + x: Math.min(Math.max(x, left), left + width - 1), + y: Math.min(Math.max(y, top), top + height - 1), + }; +} + +// Moves to (x, y) β€” clamped to the tracked window β€” saves a small marked debug crop of what's +// there (for after-the-fact review, not fed back to any AI call), then clicks. +async function debugClickAt( + ctx: RunContext, + rawX: number, + rawY: number, + label: string, + opts?: { doubleClick?: boolean } +) { + const { x, y } = clampToWindow(ctx, rawX, rawY); + await moveMouse(x, y, ctx.ip); + const { image } = await captureScreenshot(ctx.ip); + const crop = await cropAroundPoint(image, x, y); + backupTypeAgentScreenshot(ctx.runId, `${label}_debug`, crop); + + const action = opts?.doubleClick ? "double_click" : "click"; + console.log(`[type-agent] ${opts?.doubleClick ? "double-click" : "click"} at (${x}, ${y}) β€” ${label}`); + logTypeAgentStep(ctx.runId, { event: action, label, pixelX: x, pixelY: y }); if (opts?.doubleClick) { - await doubleClick(loc.x, loc.y, ctx.ip); + await doubleClick(x, y, ctx.ip); } else { - await click(loc.x, loc.y, ctx.ip); + await click(x, y, ctx.ip); } } -// Open Dental, "make an appointment for an existing patient" β€” the exact sequence confirmed -// against a real Open Dental instance: double-click the pre-positioned schedule slot (opens -// Select Patient) β†’ type last name β†’ double-click the matching patient row (opens Edit -// Appointment with the appointment already created) β†’ click the Exam procedure β†’ click Save. +// Detects a newly-opened dialog by diffing the screenshot stashed in ctx.beforeNextWindow +// against the current screen, and adopts its bounds as the window every subsequent locate/click +// is restricted to. The four corners (A/B/C/D) are just this rectangle's corners, logged for +// visibility. +async function detectAndTrackNewWindow(ctx: RunContext, label: string): Promise { + if (!ctx.beforeNextWindow) { + throw new Error(`No "before" screenshot captured for detecting the ${label} window`); + } + const { image: after } = await captureScreenshot(ctx.ip); + const bounds = await diffBoundingBox(ctx.beforeNextWindow, after); + if (!bounds) { + throw new Error(`Could not detect the ${label} window opening (no significant screen change)`); + } + ctx.windowBounds = bounds; + ctx.beforeNextWindow = null; + logTypeAgentStep(ctx.runId, { + event: "window_detected", + label, + A: { x: bounds.left, y: bounds.top }, + B: { x: bounds.left + bounds.width, y: bounds.top }, + C: { x: bounds.left + bounds.width, y: bounds.top + bounds.height }, + D: { x: bounds.left, y: bounds.top + bounds.height }, + }); +} + +// Open Dental, "make an appointment for an existing patient": +// 1. Screenshot, then double-click the pre-positioned schedule slot. +// 2. Diff against that screenshot to find the newly-opened Select Patient dialog's bounds β€” +// every following locate/click in this dialog is restricted to that region, so nothing +// outside it (the main window's title bar, side panels) can ever be matched by mistake. +// 3. Click the Last Name field (found by vision, now unambiguous since the window is small +// and cropped) and type the first three letters β€” no first name is typed. +// 4. Find the topmost actual patient row in the now-filtered results grid and double-click +// it (opens Edit Appointment). +// 5. Diff again to find the Edit Appointment window's bounds, then click Exam and Save +// within it. const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ + { + label: "Screenshot, then double-click the schedule at the cursor position", + execute: async (ctx) => { + const { image } = await captureScreenshot(ctx.ip); + ctx.beforeNextWindow = image; + await doubleClickCurrent(ctx.ip); + }, + delayAfterMs: 1000, // Select Patient window opening + }, + { + label: "Detect the Select Patient window", + execute: async (ctx) => { + await detectAndTrackNewWindow(ctx, "Select Patient"); + }, + delayAfterMs: 100, + }, + { + label: "Find and click the Last Name field, type the first three letters", + execute: async (ctx) => { + const lastThree = ctx.patientLastName.slice(0, 3); + const { region } = await captureWindowScreenshot(ctx); + const loc = await locateOnScreenshot( + ctx.userId, + region, + 'the text input field immediately to the right of the "Last Name" label' + ); + if (!loc) throw new Error('Could not find the "Last Name" field in the Select Patient window'); + const screen = toScreenPoint(ctx, loc); + logTypeAgentStep(ctx.runId, { event: "locate", label: "Last Name field", pixelX: screen.x, pixelY: screen.y }); + + await debugClickAt(ctx, screen.x, screen.y, "last_name_field"); + await typeText(lastThree, ctx.ip); + }, + delayAfterMs: 600, // patient list filtering + }, + { + label: "Verify the last name fragment was typed", + execute: async (ctx) => { + const { full } = await captureWindowScreenshot(ctx); + backupTypeAgentScreenshot(ctx.runId, "typed_check", full); + const lastThree = ctx.patientLastName.slice(0, 3); + const typed = await confirmTextTyped(ctx.userId, full, lastThree); + logTypeAgentStep(ctx.runId, { event: "typed_check", typed }); + if (!typed) { + throw new Error(`"${lastThree}" doesn't appear to have been typed anywhere on screen`); + } + }, + delayAfterMs: 200, + }, + { + label: "Find and double-click the first patient row", + execute: async (ctx) => { + const { region } = await captureWindowScreenshot(ctx); + backupTypeAgentScreenshot(ctx.runId, "patient_row_locate", region); + + const headerMatches = await locateAllOnScreenshot(ctx.userId, region, "PatNum"); + logTypeAgentStep(ctx.runId, { event: "locate_all", label: "PatNum header", matches: toScreenPoints(ctx, headerMatches) }); + if (headerMatches.length === 0) throw new Error('Could not find the "PatNum" column header'); + const header = headerMatches.reduce((a, b) => (a.y < b.y ? a : b)); + + // A separate "find the first data row, not the header" locate call kept landing too close + // to (or on) the header β€” the two rows are visually similar and only ~15px apart. Since + // "PatNum" is an unambiguous anchor and the row height is measurable, the first row's + // position is computed directly (headerY + one row height) instead of asking the AI to + // visually tell two adjacent, similarly-styled rows apart a second time β€” nothing left to + // confuse once it's arithmetic on two already-known values. + const rowHeightRatio = await detectRowHeightRatio(ctx.userId, region); + const rowHeightPx = Math.round(rowHeightRatio * (ctx.windowBounds?.height ?? 0)); + if (rowHeightPx <= 0) throw new Error(`AI reported an invalid row height ratio: ${rowHeightRatio}`); + const finalY = header.y + rowHeightPx; + + const screen = toScreenPoint(ctx, { x: header.x, y: finalY }); + logTypeAgentStep(ctx.runId, { + event: "locate", + label: "first patient row", + pixelX: screen.x, + finalY: screen.y, + rowHeightPx, + headerY: toScreenPoint(ctx, { x: 0, y: header.y }).y, + }); + + ctx.beforeNextWindow = (await captureScreenshot(ctx.ip)).image; + await debugClickAt(ctx, screen.x, screen.y, "patient_row", { doubleClick: true }); + }, + delayAfterMs: 1200, // Edit Appointment window opening + }, + { + label: "Detect the Edit Appointment window", + execute: async (ctx) => { + await detectAndTrackNewWindow(ctx, "Edit Appointment"); + }, + delayAfterMs: 100, + }, + { + label: "Find and click the Exam procedure in the upper middle column", + execute: async (ctx) => { + const { region } = await captureWindowScreenshot(ctx); + backupTypeAgentScreenshot(ctx.runId, "exam_locate", region); + + // The Edit Appointment window is laid out in three big panels: Patient Info (left), the + // clickable single-click procedures list (middle), and the already-added procedures grid + // plus comm log (right). "Exam" appears both as a clickable item in the middle list AND + // as an already-added row in the right grid β€” telling them apart by y-position alone isn't + // reliable since both can land at a similar height. Finding the actual column dividers and + // keeping only the "Exam" match inside the middle column resolves this structurally. + const dividers = await detectColumnBoundaries(ctx.userId, region); + logTypeAgentStep(ctx.runId, { event: "column_boundaries", dividers: dividers.map((x) => toScreenPoint(ctx, { x, y: 0 }).x) }); + if (dividers.length !== 2) { + throw new Error(`Expected 2 column dividers (3 columns), found ${dividers.length}`); + } + const [middleLeft, middleRight] = dividers as [number, number]; + + const examMatches = await locateAllOnScreenshot(ctx.userId, region, "Exam"); + const singleClickMatches = await locateAllOnScreenshot(ctx.userId, region, "Single click"); + logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Exam", matches: toScreenPoints(ctx, examMatches) }); + logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Single click", matches: toScreenPoints(ctx, singleClickMatches) }); + + if (examMatches.length === 0) throw new Error('Could not find "Exam" in the Edit Appointment window'); + if (singleClickMatches.length === 0) throw new Error('Could not find "Single click" in the Edit Appointment window'); + + const inMiddleColumn = examMatches.filter((m) => m.x > middleLeft && m.x < middleRight); + if (inMiddleColumn.length === 0) { + throw new Error('Found "Exam", but none were in the middle column of the window'); + } + // "Upper" β€” the topmost match in that column, in case more than one qualifies. + const examLoc = inMiddleColumn.reduce((a, b) => (a.y < b.y ? a : b)); + + // "Single click" is the hint text right above this list β€” same row-height/grid-average + // technique used for the patient list, anchored on that text instead of a column header, + // so the click lands toward the middle of the Exam row rather than its top edge. + const singleClickY = Math.min(...singleClickMatches.map((m) => m.y)); + const rowHeightRatio = await detectRowHeightRatio(ctx.userId, region); + const rowHeightPx = Math.round(rowHeightRatio * (ctx.windowBounds?.height ?? 0)); + if (rowHeightPx <= 0) throw new Error(`AI reported an invalid row height ratio: ${rowHeightRatio}`); + const rowIndex = Math.max(1, Math.round((examLoc.y - singleClickY) / rowHeightPx)); + const gridY = singleClickY + rowHeightPx * rowIndex; + const finalY = Math.round((examLoc.y + gridY) / 2); + + const screen = toScreenPoint(ctx, { x: examLoc.x, y: finalY }); + logTypeAgentStep(ctx.runId, { + event: "exam_match", + pixelX: screen.x, + directY: toScreenPoint(ctx, { x: 0, y: examLoc.y }).y, + gridY: toScreenPoint(ctx, { x: 0, y: gridY }).y, + finalY: screen.y, + rowHeightPx, + }); + + await debugClickAt(ctx, screen.x, screen.y, "exam"); + }, + delayAfterMs: 500, + }, + { + label: "Find and click Save", + execute: async (ctx) => { + const { region } = await captureWindowScreenshot(ctx); + backupTypeAgentScreenshot(ctx.runId, "save_locate", region); + const loc = await locateOnScreenshot( + ctx.userId, + region, + "the Save button in the Edit Appointment window β€” it's in the LOWER part of the " + + "RIGHTMOST of the window's three big columns (the one with the already-added " + + "procedures grid and comm log above it), below the other action buttons in that column " + + '(e.g. "Delete", "To Task List", "Audit Trail")' + ); + if (!loc) throw new Error("Could not find the Save button in the Edit Appointment window"); + const screen = toScreenPoint(ctx, loc); + logTypeAgentStep(ctx.runId, { event: "locate", label: "Save", pixelX: screen.x, pixelY: screen.y }); + + await debugClickAt(ctx, screen.x, screen.y, "save"); + }, + delayAfterMs: 300, + }, +]; + +async function runSteps(ctx: RunContext, steps: RunStep[], onProgress: ProgressCallback): Promise { + logTypeAgentStep(ctx.runId, { + event: "run_start", + patientLastName: ctx.patientLastName, + patientFirstName: ctx.patientFirstName, + ip: ctx.ip, + totalSteps: steps.length, + }); + for (let i = 0; i < steps.length; i++) { + const step = steps[i]!; + onProgress(i, steps.length, step.label, "running"); + logTypeAgentStep(ctx.runId, { event: "step_start", index: i, label: step.label }); + try { + await step.execute(ctx); + onProgress(i, steps.length, step.label, "done"); + logTypeAgentStep(ctx.runId, { event: "step_done", index: i, label: step.label }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + onProgress(i, steps.length, step.label, "error", message); + logTypeAgentStep(ctx.runId, { event: "step_error", index: i, label: step.label, error: message }); + throw err; + } + await new Promise((resolve) => setTimeout(resolve, step.delayAfterMs)); + } + logTypeAgentStep(ctx.runId, { event: "run_done" }); +} + +export async function runExistingPatientOpenDental( + userId: number, + runId: string, + ip: string, + patientLastName: string, + patientFirstName: string, + onProgress: ProgressCallback +): Promise { + const ctx: RunContext = { + ip, + userId, + runId, + patientLastName, + patientFirstName, + windowBounds: null, + beforeNextWindow: null, + }; + await runSteps(ctx, OPEN_DENTAL_EXISTING_PATIENT, onProgress); +} + +// TEMPORARY TEST PATH β€” types the full last name directly (relying on the Select Patient +// window's default focus, no field click) and then moves to a hardcoded ratio (0.31, 0.21) β€” +// the position manually read off a real "Allowed, Allen" row screenshot β€” instead of any +// vision lookup. Exists purely to test the ratioβ†’pixel conversion and move/double-click +// mechanics in isolation from AI locate mistakes. Kept separate from the real flow above, +// which now types into both name fields individually and uses window-relative locates instead. +const TEST_ROW_X_RATIO = 0.31; +const TEST_ROW_Y_RATIO = 0.21; + +const OPEN_DENTAL_EXISTING_PATIENT_RATIO_TEST: RunStep[] = [ { label: "Double-click the schedule at the cursor position", execute: async (ctx) => { @@ -49,90 +379,56 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ delayAfterMs: 1000, // Select Patient window opening }, { - label: "Find and click the Last Name field", - execute: async (ctx) => { - await visionClick( - ctx, - "the Last Name text input field in the Select Patient window β€” it's in the \"Search by:\" " + - "panel on the RIGHT half of the window (if you split the window into a left half and a " + - "right half), not in the results grid on the left" - ); - }, - delayAfterMs: 300, - }, - { - label: "Type the patient's last name", + label: "Type the patient's last name (no field click first β€” relies on default focus)", execute: async (ctx) => { await typeText(ctx.patientLastName, ctx.ip); }, delayAfterMs: 600, // patient list filtering }, { - label: "Find and double-click the matching patient row", + label: "TEST: verify the last name was typed", execute: async (ctx) => { - const lastPrefix = ctx.patientLastName.slice(0, 3); - const firstPrefix = ctx.patientFirstName.slice(0, 3); - await visionClick( - ctx, - `Search the LEFT half of the Select Patient window for the row of patient data whose ` + - `LastName column starts with "${lastPrefix}" AND whose First Name column starts with ` + - `"${firstPrefix}" β€” matching is case-insensitive and only needs to match the first 3 ` + - `letters of each name, not the full name exactly. Both the last name prefix and first name ` + - `prefix must match β€” there may be other rows with a matching last name (family members, ` + - `e.g. a sibling or spouse) but a different first name; do NOT click those, they are the ` + - `wrong patient. That row is likely shaded/highlighted in LIGHT BLUE as the currently-selected ` + - `row. Do NOT click the window's own title bar β€” a DARK navy-blue bar at the very top of the ` + - `window that just reads "Select Patient" in white text; that is window chrome, not a patient ` + - `row, and is well above the results grid. Also do not click the column header row (labels ` + - `like "PatNum", "LastName", "First Name"). Click on the actual patient text in the matching data row.`, - { doubleClick: true } - ); + const { image } = await captureScreenshot(ctx.ip); + backupTypeAgentScreenshot(ctx.runId, `ratio_test_typed_check_${ctx.patientLastName}`, image); + const typed = await confirmTextTyped(ctx.userId, image, ctx.patientLastName); + if (!typed) { + throw new Error(`"${ctx.patientLastName}" doesn't appear to have been typed anywhere on screen`); + } }, - delayAfterMs: 1200, // Edit Appointment window opening + delayAfterMs: 200, }, { - label: "Find and click the Exam procedure", + label: `TEST: move to fixed ratio (${TEST_ROW_X_RATIO}, ${TEST_ROW_Y_RATIO}) and double-click`, execute: async (ctx) => { - await visionClick( - ctx, - 'the "Exam" item in the procedures list of the Edit Appointment window β€” it\'s in the ' + - "UPPER-MIDDLE area of the window (if you split the window into an upper half and a lower " + - "half, and again into left/middle/right thirds, it's in the upper half, middle third), " + - 'in a plain text list that also contains items like "Ex,Pro,Flo", "Prophy-Adult", "Pano"' - ); + const { image } = await captureScreenshot(ctx.ip); + const { width = 0, height = 0 } = await sharp(Buffer.from(image, "base64")).metadata(); + const x = Math.round(TEST_ROW_X_RATIO * width); + const y = Math.round(TEST_ROW_Y_RATIO * height); + + console.log(`[type-agent] TEST double-click at ratio (${TEST_ROW_X_RATIO}, ${TEST_ROW_Y_RATIO}) -> pixel (${x}, ${y})`); + await moveMouse(x, y, ctx.ip); + await doubleClick(x, y, ctx.ip); }, - delayAfterMs: 500, - }, - { - label: "Find and click Save", - execute: async (ctx) => { - await visionClick(ctx, "the Save button in the lower right corner of the Edit Appointment window"); - }, - delayAfterMs: 300, + delayAfterMs: 1200, }, ]; -export async function runExistingPatientOpenDental( +export async function runExistingPatientOpenDentalRatioTest( userId: number, + runId: string, ip: string, patientLastName: string, patientFirstName: string, onProgress: ProgressCallback ): Promise { - const ctx: RunContext = { ip, userId, patientLastName, patientFirstName }; - const steps = OPEN_DENTAL_EXISTING_PATIENT; - - for (let i = 0; i < steps.length; i++) { - const step = steps[i]!; - onProgress(i, steps.length, step.label, "running"); - try { - await step.execute(ctx); - onProgress(i, steps.length, step.label, "done"); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - onProgress(i, steps.length, step.label, "error", message); - throw err; - } - await new Promise((resolve) => setTimeout(resolve, step.delayAfterMs)); - } + const ctx: RunContext = { + ip, + userId, + runId, + patientLastName, + patientFirstName, + windowBounds: null, + beforeNextWindow: null, + }; + await runSteps(ctx, OPEN_DENTAL_EXISTING_PATIENT_RATIO_TEST, onProgress); } diff --git a/apps/Backend/src/services/visionLocate.ts b/apps/Backend/src/services/visionLocate.ts index 803076eb..7cd2154c 100644 --- a/apps/Backend/src/services/visionLocate.ts +++ b/apps/Backend/src/services/visionLocate.ts @@ -1,17 +1,22 @@ +import sharp from "sharp"; import { storage } from "../storage"; import { resolveAiProvider, getLlm } from "../ai/llm-factory"; +// Side length (px) of the debug crop saved around a click point β€” big enough to show +// surrounding context (e.g. neighboring grid rows) when reviewing what a run actually clicked. +const DEBUG_CROP_SIZE = 220; + interface ParsedLocation { found: boolean; - x?: number; - y?: number; + xRatio?: number; + yRatio?: number; } -// Parses the model's response into {found, x, y}, tolerating near-miss JSON rather than -// only accepting strictly valid JSON. Seen in practice: {"found": true, "x": 1023, 187} β€” -// a dropped "y" key with the value still present positionally. Since this response should -// only ever contain two numbers (the coordinates), falling back to "first two numbers after -// found" is safe and recovers without burning another AI call. +// Parses the model's response into {found, xRatio, yRatio}, tolerating near-miss JSON rather +// than only accepting strictly valid JSON. Seen in practice with the old pixel-based format: +// {"found": true, "x": 1023, 187} β€” a dropped key with the value still present positionally. +// Since this response should only ever contain two numbers (the coordinates), falling back to +// "first two numbers after found" is safe and recovers without burning another AI call. function parseLocationResponse(raw: string): ParsedLocation | null { const match = raw.match(/\{[\s\S]*\}/); if (!match) return null; @@ -32,17 +37,24 @@ function parseLocationResponse(raw: string): ParsedLocation | null { const numbers = afterFound.match(/-?\d+(\.\d+)?/g); if (!numbers || numbers.length < 2) return null; - return { found: true, x: Number(numbers[0]), y: Number(numbers[1]) }; + return { found: true, xRatio: Number(numbers[0]), yRatio: Number(numbers[1]) }; } // Asks the configured vision AI to find a described UI element in a screenshot and return -// its pixel coordinates β€” the core primitive the Type Agent step runner clicks/types through. +// its position β€” the core primitive the Type Agent step runner clicks/types through. // Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info. +// +// The AI reports position as a 0-1 ratio of image width/height rather than absolute pixels. +// Vision models are more reliable at "this is 4/10 of the way across, 3/10 down" than at +// naming an exact pixel β€” asking for pixels directly produced coordinates that were close but +// occasionally off by enough to land on a neighboring row or the title bar above it. A ratio +// is resolution-independent by construction, so it's converted to real screen pixels here +// using the actual screenshot dimensions rather than trusting the model to know the image size. export async function locateOnScreenshot( userId: number, imageBase64: string, goal: string -): Promise<{ x: number; y: number } | null> { +): Promise<{ x: number; y: number; xRatio: number; yRatio: number } | null> { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { @@ -53,6 +65,7 @@ export async function locateOnScreenshot( } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); const content = [ { @@ -60,9 +73,12 @@ export async function locateOnScreenshot( text: `This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` + "Respond with strict JSON only, no prose, no markdown fences: " + - '{"found": true, "x": , "y": } if you can locate it (coordinates must be ' + - 'pixel positions within this exact image, at the center of the element), or {"found": false} ' + - "if it isn't visible in this screenshot.", + '{"found": true, "xRatio": <0 to 1>, "yRatio": <0 to 1>} if you can locate it, or ' + + '{"found": false} if it isn\'t visible in this screenshot. xRatio is how far across the ' + + "image the element's center is, as a fraction of the TOTAL image width (0 = left edge, " + + "0.5 = horizontal center, 1 = right edge). yRatio is how far down the element's center is, " + + "as a fraction of the TOTAL image height (0 = top edge, 1 = bottom edge). Use two decimal " + + "places of precision β€” do not report pixel coordinates.", }, { type: "image_url", @@ -86,11 +102,353 @@ export async function locateOnScreenshot( continue; } - if (!parsed.found || typeof parsed.x !== "number" || typeof parsed.y !== "number") { + if (!parsed.found || typeof parsed.xRatio !== "number" || typeof parsed.yRatio !== "number") { return null; } - return { x: Math.round(parsed.x), y: Math.round(parsed.y) }; + return { + x: Math.round(parsed.xRatio * width), + y: Math.round(parsed.yRatio * height), + xRatio: parsed.xRatio, + yRatio: parsed.yRatio, + }; } throw lastError ?? new Error("AI returned an unparseable response"); } + +// Crops a small square around (x, y) and draws a red crosshair marker at that exact point β€” +// a debug artifact saved before each click so a person can see afterward exactly where the +// run was about to click. A real screen capture won't reliably include the OS mouse cursor +// (BitBlt-based grabs typically omit it), so this synthetic marker stands in for it. +export async function cropAroundPoint(imageBase64: string, x: number, y: number): Promise { + const source = sharp(Buffer.from(imageBase64, "base64")); + const { width = 0, height = 0 } = await source.metadata(); + + const cropWidth = Math.min(DEBUG_CROP_SIZE, width); + const cropHeight = Math.min(DEBUG_CROP_SIZE, height); + const left = Math.round(Math.max(0, Math.min(x - cropWidth / 2, width - cropWidth))); + const top = Math.round(Math.max(0, Math.min(y - cropHeight / 2, height - cropHeight))); + const markerX = x - left; + const markerY = y - top; + + const crosshair = Buffer.from( + `` + + `` + + `` + + `` + + `` + ); + + const cropped = await source + .extract({ left, top, width: cropWidth, height: cropHeight }) + .composite([{ input: crosshair }]) + .png() + .toBuffer(); + + return cropped.toString("base64"); +} + +// Parses {"matches": [{"xRatio":.., "yRatio":..}, ...]}, tolerating near-miss JSON the same +// way parseLocationResponse does β€” falls back to regex-extracting every xRatio/yRatio pair in +// the response if strict parsing fails, rather than treating a malformed-but-salvageable +// response as zero matches. +function parseMatchesResponse(raw: string): { xRatio: number; yRatio: number }[] | null { + const block = raw.match(/\{[\s\S]*\}/)?.[0]; + if (!block) return null; + + try { + const parsed = JSON.parse(block); + if (Array.isArray(parsed.matches)) return parsed.matches; + } catch { + // fall through to lenient repair below + } + + const pairs: { xRatio: number; yRatio: number }[] = []; + const pairRegex = /"xRatio"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"yRatio"\s*:\s*(-?\d+(?:\.\d+)?)/g; + let match: RegExpExecArray | null; + while ((match = pairRegex.exec(block))) { + pairs.push({ xRatio: Number(match[1]), yRatio: Number(match[2]) }); + } + return pairs.length > 0 ? pairs : null; +} + +// Like locateOnScreenshot, but finds every occurrence of an exact piece of text rather than +// one best guess for a described element. Used to disambiguate by position instead of by +// asking the AI to judge which occurrence is "the right one" β€” e.g. a last name shows up once +// in the search box and once per matching patient row, so the caller pairs it against another +// text's occurrences (a first name) that share the same row instead of relying on a single +// fuzzy vision judgment call. +export async function locateAllOnScreenshot( + userId: number, + imageBase64: string, + text: string +): Promise<{ x: number; y: number }[]> { + const aiSettings = await storage.getAiSettings(userId); + const activeAi = resolveAiProvider(aiSettings ?? {}); + if (!activeAi) { + throw new Error("AI is not configured. Add an API key in AI Settings."); + } + if (activeAi.provider !== "claude") { + throw new Error("Vision-guided steps require Claude to be the active AI provider."); + } + + const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); + + const content = [ + { + type: "text", + text: + `This is a screenshot of a Windows desktop application. Find EVERY occurrence of the text ` + + `"${text}" visible anywhere in this screenshot (case-insensitive) β€” there may be zero, one, ` + + "or several. Respond with strict JSON only, no prose, no markdown fences: " + + '{"matches": [{"xRatio": <0 to 1>, "yRatio": <0 to 1>}, ...]} β€” one entry per occurrence, ' + + "at the center of that occurrence's text, using the same fraction-of-image-width/height " + + "convention as before. Use an empty array if there are no occurrences.", + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${imageBase64}` }, + }, + ]; + + const MAX_ATTEMPTS = 2; + let lastError: Error | null = null; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const response = await llm.invoke([{ role: "user", content }] as any); + const raw = String(response.content).trim(); + + const matches = parseMatchesResponse(raw); + if (!matches) { + lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); + continue; + } + return matches.map((m) => ({ + x: Math.round(m.xRatio * width), + y: Math.round(m.yRatio * height), + })); + } + + throw lastError ?? new Error("AI returned an unparseable response"); +} + +// Asks the AI to measure the height of one row in a results-grid list, as a ratio of the +// image's total height β€” used to build a grid of valid row-center y-positions so a click can be +// snapped toward the middle of whichever row it's closest to, rather than trusting a single +// direct locate call's y-coordinate on its own (which tends to land near a row's top edge). +export async function detectRowHeightRatio(userId: number, imageBase64: string): Promise { + const aiSettings = await storage.getAiSettings(userId); + const activeAi = resolveAiProvider(aiSettings ?? {}); + if (!activeAi) { + throw new Error("AI is not configured. Add an API key in AI Settings."); + } + if (activeAi.provider !== "claude") { + throw new Error("Vision-guided steps require Claude to be the active AI provider."); + } + + const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + + const content = [ + { + type: "text", + text: + "This is a screenshot containing a results-grid list of rows (e.g. a patient list), each " + + "row the same height. Measure the height of a single row. Respond with strict JSON only, " + + 'no prose, no markdown fences: {"rowHeightRatio": <0 to 1>} β€” the height of one row as a ' + + "fraction of the TOTAL image height. Use two decimal places of precision.", + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${imageBase64}` }, + }, + ]; + + const response = await llm.invoke([{ role: "user", content }] as any); + const raw = String(response.content).trim(); + const match = raw.match(/"rowHeightRatio"\s*:\s*(-?\d+(?:\.\d+)?)/); + if (!match) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); + return Number(match[1]); +} + +// Asks the AI to find the x-positions of the MAJOR vertical dividers that split a window into +// its big panels/columns (e.g. a "Patient Info" panel, a middle fields+list panel, a right-side +// grid panel) β€” as opposed to minor lines inside a single table's own columns. Used to tell +// apart same-text matches that live in different panels (e.g. "Exam" appearing both in a +// clickable procedures list and, separately, in an already-added-procedures grid) by which +// column they actually fall in, rather than guessing from y-position alone. +export async function detectColumnBoundaries(userId: number, imageBase64: string): Promise { + const aiSettings = await storage.getAiSettings(userId); + const activeAi = resolveAiProvider(aiSettings ?? {}); + if (!activeAi) { + throw new Error("AI is not configured. Add an API key in AI Settings."); + } + if (activeAi.provider !== "claude") { + throw new Error("Vision-guided steps require Claude to be the active AI provider."); + } + + const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + const { width = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); + + const content = [ + { + type: "text", + text: + "This screenshot shows a Windows dialog laid out in a small number of large vertical " + + "sections (columns) β€” e.g. a distinct panel on the left, a middle section with fields " + + 'and/or a list, and another panel or grid on the right. Find the x-position of each MAJOR ' + + "vertical divider between these big sections (a visible line, border, or clear gap β€” NOT " + + "the minor column lines inside a single table). Respond with strict JSON only, no prose, " + + 'no markdown fences: {"dividerXRatios": [<0 to 1>, ...]} β€” one entry per major divider, ' + + "sorted left to right, as a fraction of the TOTAL image width. Use two decimal places of " + + "precision.", + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${imageBase64}` }, + }, + ]; + + const response = await llm.invoke([{ role: "user", content }] as any); + const raw = String(response.content).trim(); + const block = raw.match(/\{[\s\S]*\}/)?.[0]; + if (!block) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); + + let ratios: number[]; + try { + const parsed = JSON.parse(block); + if (!Array.isArray(parsed.dividerXRatios)) throw new Error("not an array"); + ratios = parsed.dividerXRatios; + } catch { + const arrayMatch = block.match(/"dividerXRatios"\s*:\s*\[([^\]]*)\]/); + if (!arrayMatch) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); + ratios = arrayMatch[1]! + .split(",") + .map((s) => Number(s.trim())) + .filter((n) => !Number.isNaN(n)); + } + + return ratios.map((r) => Math.round(r * width)).sort((a, b) => a - b); +} + +export interface WindowBounds { + left: number; + top: number; + width: number; + height: number; +} + +// Compares two full screenshots and returns the bounding box of whatever changed +// significantly between them β€” used to find a newly-opened dialog's exact position/size with +// pixel math instead of an AI call, so later steps can crop every screenshot down to just that +// window and never even show the AI anything outside it (the earlier bug where a name prefix +// happened to also match unrelated text elsewhere on screen, e.g. "All Received" in a side +// panel, is structurally impossible once matching only ever sees the cropped window). +// +// Rather than a raw min/max over any differing pixel, this counts changed pixels per row and +// per column and only counts a row/column as part of the new window once a meaningful fraction +// of it changed β€” a real window opening changes a large contiguous block, whereas incidental +// noise elsewhere (a blinking clock, a flickering tray icon) only touches a handful of pixels +// and would otherwise blow the bounding box out to include it. +export async function diffBoundingBox( + beforeBase64: string, + afterBase64: string +): Promise { + const [before, after] = await Promise.all([ + sharp(Buffer.from(beforeBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }), + sharp(Buffer.from(afterBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }), + ]); + const { data: b, info } = before; + const { data: a, info: infoAfter } = after; + if (info.width !== infoAfter.width || info.height !== infoAfter.height) return null; + + const { width, height, channels } = info; + const PIXEL_DIFF_THRESHOLD = 40; // sum of |dR| + |dG| + |dB| to count a pixel as "changed" + const rowCounts = new Uint32Array(height); + const colCounts = new Uint32Array(width); + + for (let y = 0; y < height; y++) { + const rowBase = y * width * channels; + for (let x = 0; x < width; x++) { + const idx = rowBase + x * channels; + const diff = Math.abs(a[idx]! - b[idx]!) + Math.abs(a[idx + 1]! - b[idx + 1]!) + Math.abs(a[idx + 2]! - b[idx + 2]!); + if (diff > PIXEL_DIFF_THRESHOLD) { + rowCounts[y] = (rowCounts[y] ?? 0) + 1; + colCounts[x] = (colCounts[x] ?? 0) + 1; + } + } + } + + const rowThreshold = width * 0.05; + const colThreshold = height * 0.05; + + let top = -1; + let bottom = -1; + for (let y = 0; y < height; y++) { + if (rowCounts[y]! > rowThreshold) { + if (top === -1) top = y; + bottom = y; + } + } + let left = -1; + let right = -1; + for (let x = 0; x < width; x++) { + if (colCounts[x]! > colThreshold) { + if (left === -1) left = x; + right = x; + } + } + + if (top === -1 || left === -1) return null; + return { left, top, width: right - left + 1, height: bottom - top + 1 }; +} + +// Plain crop, no marker β€” used to restrict a screenshot to a previously-detected window's +// bounds before handing it to any locate call. +export async function cropToRegion(imageBase64: string, region: WindowBounds): Promise { + const cropped = await sharp(Buffer.from(imageBase64, "base64")) + .extract({ left: region.left, top: region.top, width: region.width, height: region.height }) + .png() + .toBuffer(); + return cropped.toString("base64"); +} + +// Sanity check for steps that type without first vision-clicking into a field (relying on +// whatever already has focus) β€” confirms the text actually landed somewhere visible on +// screen before moving on, catching the case where focus wasn't where it was assumed to be. +export async function confirmTextTyped( + userId: number, + fullScreenshotBase64: string, + expectedText: string +): Promise { + const aiSettings = await storage.getAiSettings(userId); + const activeAi = resolveAiProvider(aiSettings ?? {}); + if (!activeAi) { + throw new Error("AI is not configured. Add an API key in AI Settings."); + } + if (activeAi.provider !== "claude") { + throw new Error("Vision-guided steps require Claude to be the active AI provider."); + } + + const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); + + const content = [ + { + type: "text", + text: + `This is a screenshot of a Windows desktop application. Is the text "${expectedText}" ` + + "visible typed into some text input field on screen? Respond with strict JSON only, no " + + 'prose, no markdown fences: {"confirmed": true} if that text is visible in a field, or ' + + '{"confirmed": false} if it is not visible anywhere.', + }, + { + type: "image_url", + image_url: { url: `data:image/png;base64,${fullScreenshotBase64}` }, + }, + ]; + + const response = await llm.invoke([{ role: "user", content }] as any); + const raw = String(response.content).trim(); + const match = raw.match(/"confirmed"\s*:\s*(true|false)/i); + return match?.[1]?.toLowerCase() === "true"; +} diff --git a/apps/Backend/src/services/windowsAgentBridge.ts b/apps/Backend/src/services/windowsAgentBridge.ts index f550179d..72a7b5d4 100644 --- a/apps/Backend/src/services/windowsAgentBridge.ts +++ b/apps/Backend/src/services/windowsAgentBridge.ts @@ -28,6 +28,12 @@ export function captureScreenshot(ip?: string) { return sendCommand<{ image: string }>("cmd:screenshot", {}, ip); } +// Moves the cursor without clicking β€” lets a step take a confirmation screenshot before +// committing to a click. +export function moveMouse(x: number, y: number, ip?: string) { + return sendCommand<{ ok: boolean }>("cmd:move", { x, y }, ip); +} + export function click(x: number, y: number, ip?: string) { return sendCommand<{ ok: boolean }>("cmd:click", { x, y }, ip); } diff --git a/apps/Backend/src/utils/screenshotBackup.ts b/apps/Backend/src/utils/screenshotBackup.ts index a62213cf..c0e02da2 100644 --- a/apps/Backend/src/utils/screenshotBackup.ts +++ b/apps/Backend/src/utils/screenshotBackup.ts @@ -19,3 +19,38 @@ export function backupScreenshots(files: { originalname: string; buffer: Buffer console.error("[screenshotBackup] failed to write backup:", err); } } + +function getTypeAgentRunDir(runId: string): string { + const dateDir = new Date().toISOString().slice(0, 10); // YYYY-MM-DD + return path.join(BACKUP_ROOT, "type-agent", dateDir, runId); +} + +// Same convention as backupScreenshots, but grouped under a "type-agent//" +// subfolder so every screenshot from one Type Agent run stays together for debugging +// (e.g. "locate" and "confirm" images per step, in click order via the timestamp prefix). +export function backupTypeAgentScreenshot(runId: string, label: string, imageBase64: string): void { + try { + const dir = getTypeAgentRunDir(runId); + fs.mkdirSync(dir, { recursive: true }); + const safeLabel = label.replace(/[/\\?%*:|"<>]/g, "-").slice(0, 80); + const fileName = `${Date.now()}_${safeLabel}.png`; + fs.writeFileSync(path.join(dir, fileName), Buffer.from(imageBase64, "base64")); + } catch (err) { + console.error("[screenshotBackup] failed to write type-agent backup:", err); + } +} + +// One human-readable line per event (step start/attempt/locate/confirm/click/error), appended +// to run.log in the same per-run folder as that run's screenshots β€” so opening one folder +// shows both what was clicked and what it looked like at the time, including the exact +// AI-reported ratio for every locate attempt. +export function logTypeAgentStep(runId: string, entry: Record): void { + try { + const dir = getTypeAgentRunDir(runId); + fs.mkdirSync(dir, { recursive: true }); + const line = `[${new Date().toISOString()}] ${JSON.stringify(entry)}\n`; + fs.appendFileSync(path.join(dir, "run.log"), line); + } catch (err) { + console.error("[screenshotBackup] failed to write type-agent log:", err); + } +} diff --git a/apps/Frontend/src/pages/ai-type-agent-page.tsx b/apps/Frontend/src/pages/ai-type-agent-page.tsx index 81fab9fe..ff533c3b 100644 --- a/apps/Frontend/src/pages/ai-type-agent-page.tsx +++ b/apps/Frontend/src/pages/ai-type-agent-page.tsx @@ -3,6 +3,7 @@ import { Keyboard, CalendarPlus, UserPlus, CreditCard, Circle, CheckCircle2, Mon import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; import { Select, SelectContent, @@ -86,6 +87,10 @@ export default function AiTypeAgentPage() { const [runAction, setRunAction] = useState(null); const [agents, setAgents] = useState([]); const [selectedAgentIp, setSelectedAgentIp] = useState(null); + // Temporary debug toggle: skips vision lookup for the patient-row click and moves to a + // hardcoded ratio instead, to test move/double-click mechanics in isolation. Remove once + // the vision-based row click is confirmed reliable. + const [ratioTest, setRatioTest] = useState(false); useEffect(() => { apiRequest("GET", "/api/type-agent/status") @@ -152,6 +157,7 @@ export default function AiTypeAgentPage() { action: runAction, ip: selectedAgentIp, patientId: selectedPatient.id, + ratioTest, }) .then((res) => res.json()) .then((data) => { @@ -175,7 +181,7 @@ export default function AiTypeAgentPage() { if (liveRunId) socket.off(`type-agent:run:${liveRunId}`, handleProgress); }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id]); + }, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id, ratioTest]); const getPatientName = (patient: Patient) => patient.firstName && patient.lastName @@ -264,6 +270,13 @@ export default function AiTypeAgentPage() { )} + {/* Temporary debug toggle β€” remove once the vision-based patient-row click is + confirmed reliable */} + + {/* Connected PCs */}

Windows PC to control

diff --git a/apps/WindowsAgent/agent.py b/apps/WindowsAgent/agent.py index c44316ba..0dd28960 100644 --- a/apps/WindowsAgent/agent.py +++ b/apps/WindowsAgent/agent.py @@ -107,6 +107,13 @@ class DentalAgent: encoded = base64.b64encode(buf.getvalue()).decode("ascii") return {"image": encoded} + @sio.on("cmd:move", namespace="/agent") + def on_move(data): + # Moves the cursor without clicking β€” used to double-check a location (via a + # follow-up cropped screenshot) before committing to a click. + pyautogui.moveTo(data["x"], data["y"]) + return {"ok": True} + @sio.on("cmd:click", namespace="/agent") def on_click(data): pyautogui.click(data["x"], data["y"])