feat: window-diff-based Type Agent locating for Open Dental appointment flow
Replaces fragile whole-screen text matching with a screenshot-diff step that finds each dialog's actual pixel bounds, then restricts every subsequent AI locate/click to that cropped region — eliminating false matches from text elsewhere on screen (title bars, side panels). Adds column-boundary and row-height detection so the patient row and Exam procedure click positions are computed geometrically instead of relying on repeated fuzzy AI guesses for visually similar neighbors. Also adds per-run screenshot/debug-crop backups and a structured run.log for diagnosing failed runs, plus a cmd:move primitive on the Windows agent for pre-click confirmation crops. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import { Keyboard, CalendarPlus, UserPlus, CreditCard, Circle, CheckCircle2, Mon
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
@@ -86,6 +87,10 @@ export default function AiTypeAgentPage() {
|
||||
const [runAction, setRunAction] = useState<ActionId | null>(null);
|
||||
const [agents, setAgents] = useState<ConnectedAgent[]>([]);
|
||||
const [selectedAgentIp, setSelectedAgentIp] = useState<string | null>(null);
|
||||
// Temporary debug toggle: skips vision lookup for the patient-row click and moves to a
|
||||
// hardcoded ratio instead, to test move/double-click mechanics in isolation. Remove once
|
||||
// the vision-based row click is confirmed reliable.
|
||||
const [ratioTest, setRatioTest] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
apiRequest("GET", "/api/type-agent/status")
|
||||
@@ -152,6 +157,7 @@ export default function AiTypeAgentPage() {
|
||||
action: runAction,
|
||||
ip: selectedAgentIp,
|
||||
patientId: selectedPatient.id,
|
||||
ratioTest,
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
@@ -175,7 +181,7 @@ export default function AiTypeAgentPage() {
|
||||
if (liveRunId) socket.off(`type-agent:run:${liveRunId}`, handleProgress);
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id]);
|
||||
}, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id, ratioTest]);
|
||||
|
||||
const getPatientName = (patient: Patient) =>
|
||||
patient.firstName && patient.lastName
|
||||
@@ -264,6 +270,13 @@ export default function AiTypeAgentPage() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Temporary debug toggle — remove once the vision-based patient-row click is
|
||||
confirmed reliable */}
|
||||
<label className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Checkbox checked={ratioTest} onCheckedChange={(v) => setRatioTest(v === true)} />
|
||||
Ratio test: skip AI lookup for the patient row, move to fixed ratio (0.31, 0.21) instead
|
||||
</label>
|
||||
|
||||
{/* Connected PCs */}
|
||||
<div className="space-y-1.5">
|
||||
<p className="text-xs font-medium text-muted-foreground">Windows PC to control</p>
|
||||
|
||||
Reference in New Issue
Block a user