feat: window-diff-based Type Agent locating for Open Dental appointment flow

Replaces fragile whole-screen text matching with a screenshot-diff step that
finds each dialog's actual pixel bounds, then restricts every subsequent AI
locate/click to that cropped region — eliminating false matches from text
elsewhere on screen (title bars, side panels). Adds column-boundary and
row-height detection so the patient row and Exam procedure click positions
are computed geometrically instead of relying on repeated fuzzy AI guesses
for visually similar neighbors. Also adds per-run screenshot/debug-crop
backups and a structured run.log for diagnosing failed runs, plus a
cmd:move primitive on the Windows agent for pre-click confirmation crops.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 08:50:47 -04:00
parent 59f064583a
commit ad2e15ec62
8 changed files with 817 additions and 96 deletions

View File

@@ -3,6 +3,7 @@ import { Keyboard, CalendarPlus, UserPlus, CreditCard, Circle, CheckCircle2, Mon
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card";
import { Checkbox } from "@/components/ui/checkbox";
import {
Select,
SelectContent,
@@ -86,6 +87,10 @@ export default function AiTypeAgentPage() {
const [runAction, setRunAction] = useState<ActionId | null>(null);
const [agents, setAgents] = useState<ConnectedAgent[]>([]);
const [selectedAgentIp, setSelectedAgentIp] = useState<string | null>(null);
// Temporary debug toggle: skips vision lookup for the patient-row click and moves to a
// hardcoded ratio instead, to test move/double-click mechanics in isolation. Remove once
// the vision-based row click is confirmed reliable.
const [ratioTest, setRatioTest] = useState(false);
useEffect(() => {
apiRequest("GET", "/api/type-agent/status")
@@ -152,6 +157,7 @@ export default function AiTypeAgentPage() {
action: runAction,
ip: selectedAgentIp,
patientId: selectedPatient.id,
ratioTest,
})
.then((res) => res.json())
.then((data) => {
@@ -175,7 +181,7 @@ export default function AiTypeAgentPage() {
if (liveRunId) socket.off(`type-agent:run:${liveRunId}`, handleProgress);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id]);
}, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id, ratioTest]);
const getPatientName = (patient: Patient) =>
patient.firstName && patient.lastName
@@ -264,6 +270,13 @@ export default function AiTypeAgentPage() {
)}
</div>
{/* Temporary debug toggle — remove once the vision-based patient-row click is
confirmed reliable */}
<label className="flex items-center gap-2 text-xs text-muted-foreground">
<Checkbox checked={ratioTest} onCheckedChange={(v) => setRatioTest(v === true)} />
Ratio test: skip AI lookup for the patient row, move to fixed ratio (0.31, 0.21) instead
</label>
{/* Connected PCs */}
<div className="space-y-1.5">
<p className="text-xs font-medium text-muted-foreground">Windows PC to control</p>