feat: AI Type Agent - vision-based Open Dental automation via Windows agent
Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend services to locate UI elements via vision and drive an existing-patient appointment flow in Open Dental, wired into the Copy/Type Agent page and socket progress updates. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -1,37 +1,208 @@
|
||||
import { Keyboard, FileCheck, CreditCard, Shield, Zap, ArrowRight } from "lucide-react";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { ReactNode, useEffect, useState } from "react";
|
||||
import { Keyboard, CalendarPlus, UserPlus, CreditCard, Circle, CheckCircle2, Monitor, X, Loader2, XCircle, Maximize2 } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogDescription,
|
||||
} from "@/components/ui/dialog";
|
||||
import { PatientTable } from "@/components/patients/patient-table";
|
||||
import { Patient } from "@repo/db/types";
|
||||
import { apiRequest } from "@/lib/queryClient";
|
||||
import { socket } from "@/lib/socket";
|
||||
|
||||
const SOFTWARE_TARGETS = [
|
||||
{ name: "Open Dental", color: "bg-blue-100 text-blue-700 border-blue-200" },
|
||||
{ name: "Eaglesoft", color: "bg-emerald-100 text-emerald-700 border-emerald-200" },
|
||||
{ name: "Dentrix", color: "bg-violet-100 text-violet-700 border-violet-200" },
|
||||
{ name: "Curve Dental", color: "bg-orange-100 text-orange-700 border-orange-200" },
|
||||
type SoftwareId = "open-dental" | "dentrix" | "eaglesoft";
|
||||
type ActionId = "existing-patient" | "new-patient" | "payment";
|
||||
|
||||
interface ConnectedAgent {
|
||||
socketId: string;
|
||||
ip: string;
|
||||
connectedAt: string;
|
||||
}
|
||||
|
||||
interface LiveStep {
|
||||
index: number;
|
||||
total: number;
|
||||
label: string;
|
||||
status: "running" | "done" | "error";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const SOFTWARE_OPTIONS: { id: SoftwareId; name: string }[] = [
|
||||
{ id: "open-dental", name: "Open Dental" },
|
||||
{ id: "dentrix", name: "Dentrix" },
|
||||
{ id: "eaglesoft", name: "Eaglesoft" },
|
||||
];
|
||||
|
||||
const CAPABILITIES = [
|
||||
{
|
||||
icon: <Shield className="h-5 w-5 text-teal-600" />,
|
||||
title: "Eligibility Results",
|
||||
description:
|
||||
"Automatically type insurance eligibility information — coverage status, plan details, deductibles, and co-pays — directly into the patient chart in your dental software.",
|
||||
},
|
||||
{
|
||||
icon: <FileCheck className="h-5 w-5 text-blue-600" />,
|
||||
title: "Claim Information",
|
||||
description:
|
||||
"Transfer claim numbers, claim status updates, and denial reasons from the insurance portal results into your claims module without manual re-entry.",
|
||||
},
|
||||
{
|
||||
icon: <CreditCard className="h-5 w-5 text-indigo-600" />,
|
||||
title: "Insurance Payments",
|
||||
description:
|
||||
"Post ERA / EOB payment details — amounts, adjustments, patient responsibility — directly into your payment ledger by typing them into the active software window.",
|
||||
},
|
||||
const ACTIONS: { id: ActionId; icon: ReactNode; label: string }[] = [
|
||||
{ id: "existing-patient", icon: <CalendarPlus className="h-4 w-4" />, label: "Make an appointment for an existing patient" },
|
||||
{ id: "new-patient", icon: <UserPlus className="h-4 w-4" />, label: "Make an appointment for a new patient" },
|
||||
{ id: "payment", icon: <CreditCard className="h-4 w-4" />, label: "Type payments" },
|
||||
];
|
||||
|
||||
// Preview-only step sequences. Same shape across the three software targets for now —
|
||||
// each software's real step semantics (field positions, screen titles) will differ once
|
||||
// the Windows Agent drives them via vision, but the flow order is the same.
|
||||
const STEP_TEMPLATES: Record<ActionId, (softwareName: string, patientName: string) => string[]> = {
|
||||
"existing-patient": (sw, patient) => [
|
||||
`Double-click to open the patient selection window in ${sw}`,
|
||||
`Screenshot the window and locate the Last Name field`,
|
||||
`Type "${patient}"'s last name into the field`,
|
||||
`Locate and click the Search button`,
|
||||
`Locate and click the matching patient row`,
|
||||
`Screenshot the Add Procedure window and locate the default procedure (Exam)`,
|
||||
`Click the procedure, then click Create Appointment`,
|
||||
],
|
||||
"new-patient": (sw, patient) => [
|
||||
`Open the New Patient entry screen in ${sw}`,
|
||||
`Screenshot the form and locate the First/Last Name fields`,
|
||||
`Type "${patient}"'s name, date of birth, and insurance into the form`,
|
||||
`Locate and click Save / Create Patient`,
|
||||
`Screenshot the Add Procedure window and locate the default procedure (Exam)`,
|
||||
`Click the procedure, then click Create Appointment`,
|
||||
],
|
||||
payment: (sw, patient) => [
|
||||
`Locate "${patient}" in ${sw} and open their ledger / payment screen`,
|
||||
`Screenshot the screen and locate the payment amount field`,
|
||||
`Type the payment amount, date, and type`,
|
||||
`Locate and click Save / Post Payment`,
|
||||
],
|
||||
};
|
||||
|
||||
export default function AiTypeAgentPage() {
|
||||
const [selectedPatient, setSelectedPatient] = useState<Patient | null>(null);
|
||||
const [software, setSoftware] = useState<SoftwareId | "">("open-dental");
|
||||
const [runAction, setRunAction] = useState<ActionId | null>(null);
|
||||
const [agents, setAgents] = useState<ConnectedAgent[]>([]);
|
||||
const [selectedAgentIp, setSelectedAgentIp] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
apiRequest("GET", "/api/type-agent/status")
|
||||
.then((res) => res.json())
|
||||
.then((data) => setAgents(Array.isArray(data?.agents) ? data.agents : []))
|
||||
.catch(() => {});
|
||||
|
||||
const handleStatus = (data: { agents: ConnectedAgent[] }) =>
|
||||
setAgents(Array.isArray(data?.agents) ? data.agents : []);
|
||||
socket.on("type-agent:status", handleStatus);
|
||||
return () => {
|
||||
socket.off("type-agent:status", handleStatus);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Keep the selected PC valid as the connected list changes — auto-pick when there's
|
||||
// exactly one, and drop the selection if its PC disconnects.
|
||||
useEffect(() => {
|
||||
if (selectedAgentIp && !agents.some((a) => a.ip === selectedAgentIp)) {
|
||||
setSelectedAgentIp(null);
|
||||
} else if (!selectedAgentIp && agents.length === 1) {
|
||||
setSelectedAgentIp(agents[0]!.ip);
|
||||
}
|
||||
}, [agents, selectedAgentIp]);
|
||||
|
||||
const [liveSteps, setLiveSteps] = useState<LiveStep[]>([]);
|
||||
const [runStartError, setRunStartError] = useState<string | null>(null);
|
||||
const [startCountdown, setStartCountdown] = useState<number | null>(null);
|
||||
|
||||
// Open Dental / existing-patient is the only combination actually wired to the real
|
||||
// Windows Agent so far — everything else still shows the static preview below.
|
||||
const isAutomated = software === "open-dental" && runAction === "existing-patient";
|
||||
const START_DELAY_SECONDS = 5;
|
||||
|
||||
useEffect(() => {
|
||||
if (!isAutomated || !runAction || !selectedAgentIp || !selectedPatient) return;
|
||||
|
||||
let cancelled = false;
|
||||
let liveRunId: string | null = null;
|
||||
const handleProgress = (data: LiveStep) => {
|
||||
setLiveSteps((prev) => {
|
||||
const next = [...prev];
|
||||
next[data.index] = data;
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
setLiveSteps([]);
|
||||
setRunStartError(null);
|
||||
setStartCountdown(START_DELAY_SECONDS);
|
||||
|
||||
// Gives time to minimize/switch away from Chrome and let the target software have
|
||||
// focus before the first (mouse-position-dependent) step fires.
|
||||
const countdownInterval = setInterval(() => {
|
||||
setStartCountdown((c) => (c && c > 1 ? c - 1 : 0));
|
||||
}, 1000);
|
||||
|
||||
const startTimeout = setTimeout(() => {
|
||||
clearInterval(countdownInterval);
|
||||
setStartCountdown(null);
|
||||
|
||||
apiRequest("POST", "/api/type-agent/run", {
|
||||
software,
|
||||
action: runAction,
|
||||
ip: selectedAgentIp,
|
||||
patientId: selectedPatient.id,
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
if (cancelled) return;
|
||||
if (data?.error) {
|
||||
setRunStartError(data.error);
|
||||
return;
|
||||
}
|
||||
liveRunId = data.runId;
|
||||
socket.on(`type-agent:run:${data.runId}`, handleProgress);
|
||||
})
|
||||
.catch((err) => {
|
||||
if (!cancelled) setRunStartError(err instanceof Error ? err.message : "Failed to start run");
|
||||
});
|
||||
}, START_DELAY_SECONDS * 1000);
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(countdownInterval);
|
||||
clearTimeout(startTimeout);
|
||||
if (liveRunId) socket.off(`type-agent:run:${liveRunId}`, handleProgress);
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isAutomated, runAction, selectedAgentIp, selectedPatient?.id]);
|
||||
|
||||
const getPatientName = (patient: Patient) =>
|
||||
patient.firstName && patient.lastName
|
||||
? `${patient.firstName} ${patient.lastName}`
|
||||
: patient.firstName ?? `patient-${patient.id}`;
|
||||
|
||||
const softwareName = SOFTWARE_OPTIONS.find((s) => s.id === software)?.name ?? "";
|
||||
|
||||
const handleDisconnect = async (ip: string) => {
|
||||
try {
|
||||
await apiRequest("POST", "/api/type-agent/disconnect", { ip });
|
||||
} catch (error) {
|
||||
console.error("Error disconnecting agent:", error);
|
||||
}
|
||||
};
|
||||
|
||||
const agentConnected = agents.length > 0;
|
||||
const canRun = Boolean(software && selectedPatient && selectedAgentIp);
|
||||
|
||||
const activeAction = ACTIONS.find((a) => a.id === runAction);
|
||||
const steps =
|
||||
runAction && selectedPatient
|
||||
? STEP_TEMPLATES[runAction](softwareName, getPatientName(selectedPatient))
|
||||
: [];
|
||||
|
||||
return (
|
||||
<div className="max-w-3xl mx-auto px-4 py-10 space-y-10">
|
||||
<div className="container mx-auto space-y-6">
|
||||
|
||||
{/* Hero */}
|
||||
<div className="text-center space-y-3">
|
||||
@@ -48,89 +219,187 @@ export default function AiTypeAgentPage() {
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* How it works */}
|
||||
{/* Run setup */}
|
||||
<Card>
|
||||
<CardContent className="py-6 space-y-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<Keyboard className="h-5 w-5 text-violet-500" />
|
||||
<h2 className="text-base font-semibold">How it works</h2>
|
||||
<CardHeader>
|
||||
<CardTitle>Run Setup</CardTitle>
|
||||
<CardDescription>
|
||||
Choose your dental software, the PC to control, and a patient below, then run one
|
||||
of the actions.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<div className="w-56">
|
||||
<Select value={software} onValueChange={(v) => setSoftware(v as SoftwareId)}>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Choose software..." />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{SOFTWARE_OPTIONS.map((s) => (
|
||||
<SelectItem key={s.id} value={s.id}>
|
||||
{s.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
{agentConnected ? (
|
||||
<Badge variant="outline" className="gap-1.5 text-muted-foreground border-emerald-200 bg-emerald-50">
|
||||
<Circle className="h-2 w-2 fill-emerald-500 text-emerald-500" />
|
||||
{agents.length} PC{agents.length > 1 ? "s" : ""} Connected
|
||||
</Badge>
|
||||
) : (
|
||||
<Badge variant="outline" className="gap-1.5 text-muted-foreground border-red-200 bg-red-50">
|
||||
<Circle className="h-2 w-2 fill-red-500 text-red-500" />
|
||||
No PC Connected
|
||||
</Badge>
|
||||
)}
|
||||
|
||||
{selectedPatient ? (
|
||||
<Badge variant="secondary">Patient: {getPatientName(selectedPatient)}</Badge>
|
||||
) : (
|
||||
<span className="text-xs text-muted-foreground">Select a patient below</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-col sm:flex-row items-start sm:items-center gap-3 text-sm">
|
||||
{[
|
||||
"Retrieve data in this app\n(eligibility, claim, payment)",
|
||||
"Agent identifies the active\nfield in your dental software",
|
||||
"Types the value directly\nvia keyboard automation",
|
||||
].map((step, i) => (
|
||||
<div key={i} className="flex items-center gap-3 flex-1">
|
||||
<div className="flex-shrink-0 w-7 h-7 rounded-full bg-violet-100 text-violet-700 text-xs font-bold flex items-center justify-center">
|
||||
{i + 1}
|
||||
</div>
|
||||
<p className="whitespace-pre-line text-muted-foreground leading-snug">{step}</p>
|
||||
{i < 2 && <ArrowRight className="h-4 w-4 text-muted-foreground/40 hidden sm:block flex-shrink-0" />}
|
||||
|
||||
{/* Connected PCs */}
|
||||
<div className="space-y-1.5">
|
||||
<p className="text-xs font-medium text-muted-foreground">Windows PC to control</p>
|
||||
{agents.length === 0 ? (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
No DentalAgent.exe is connected yet. Run it on the front-desk PC you want to
|
||||
control — it will show up here.
|
||||
</p>
|
||||
) : (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{agents.map((agent) => (
|
||||
<div
|
||||
key={agent.socketId}
|
||||
className={`flex items-center gap-1.5 rounded-lg border pl-3 pr-1.5 py-1.5 text-sm transition-colors ${
|
||||
selectedAgentIp === agent.ip
|
||||
? "border-violet-300 bg-violet-50 text-violet-700"
|
||||
: "border-input hover:bg-accent"
|
||||
}`}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSelectedAgentIp(agent.ip)}
|
||||
className="flex items-center gap-1.5"
|
||||
>
|
||||
<Monitor className="h-3.5 w-3.5" />
|
||||
{agent.ip}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleDisconnect(agent.ip)}
|
||||
title="Disconnect this PC"
|
||||
className="rounded-full p-0.5 text-muted-foreground hover:bg-destructive/10 hover:text-destructive"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className="flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Maximize2 className="h-3.5 w-3.5 flex-shrink-0" />
|
||||
Maximize {softwareName || "your dental software"} to full screen on the target PC before running an action — the agent reads the whole screen, so a small or floating window makes it harder to find things reliably.
|
||||
</p>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
{ACTIONS.map((action) => (
|
||||
<Button
|
||||
key={action.id}
|
||||
variant="outline"
|
||||
disabled={!canRun}
|
||||
title={!canRun ? "Choose software, a PC, and a patient first" : undefined}
|
||||
onClick={() => setRunAction(action.id)}
|
||||
className="h-auto py-4 justify-start gap-2 whitespace-normal text-left"
|
||||
>
|
||||
{action.icon}
|
||||
{action.label}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground pt-1 border-t">
|
||||
The agent runs as a lightweight local process on your workstation. It receives
|
||||
instructions from this app and uses keyboard automation to type data into whatever
|
||||
window is currently focused in your dental software — no clipboard involved.
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* What it will type */}
|
||||
<div className="space-y-3">
|
||||
<h2 className="text-base font-semibold">What it will type</h2>
|
||||
<div className="grid gap-4 sm:grid-cols-3">
|
||||
{CAPABILITIES.map((cap) => (
|
||||
<Card key={cap.title}>
|
||||
<CardContent className="py-5 space-y-2">
|
||||
<div className="flex items-center gap-2">
|
||||
{cap.icon}
|
||||
<span className="text-sm font-medium">{cap.title}</span>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground leading-relaxed">
|
||||
{cap.description}
|
||||
{/* Patient search */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Patient Records</CardTitle>
|
||||
<CardDescription>
|
||||
Select the patient to use for the action above.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<PatientTable allowCheckbox={true} onSelectPatient={setSelectedPatient} />
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Run panel */}
|
||||
<Dialog open={runAction !== null} onOpenChange={(open) => !open && setRunAction(null)}>
|
||||
<DialogContent className="max-w-lg">
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{activeAction?.label} — {softwareName}
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
{isAutomated
|
||||
? `Running live on the connected PC at ${selectedAgentIp}.`
|
||||
: selectedAgentIp
|
||||
? `Running on the connected PC at ${selectedAgentIp}.`
|
||||
: "Preview only — no PC selected, so these steps won't run automatically."}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
{isAutomated ? (
|
||||
<div className="space-y-2">
|
||||
{runStartError && (
|
||||
<p className="text-sm text-destructive">{runStartError}</p>
|
||||
)}
|
||||
{startCountdown !== null && (
|
||||
<p className="text-sm font-medium text-violet-700">
|
||||
Starting in {startCountdown}s — minimize or switch away from this window now.
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Supported software */}
|
||||
<div className="space-y-3">
|
||||
<h2 className="text-base font-semibold">Planned software support</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{SOFTWARE_TARGETS.map((s) => (
|
||||
<span
|
||||
key={s.name}
|
||||
className={`text-sm font-medium px-3 py-1.5 rounded-lg border ${s.color}`}
|
||||
>
|
||||
{s.name}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Each software has its own field layout. The agent will include pre-built templates
|
||||
for common workflows in each platform.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Why */}
|
||||
<Card className="border-violet-200 bg-violet-50/50">
|
||||
<CardContent className="py-5 flex items-start gap-3">
|
||||
<Zap className="h-5 w-5 text-violet-500 mt-0.5 flex-shrink-0" />
|
||||
<div className="space-y-1">
|
||||
<p className="text-sm font-medium text-violet-900">Why this matters</p>
|
||||
<p className="text-sm text-violet-700 leading-relaxed">
|
||||
Staff currently check eligibility or look up a claim here, then manually retype
|
||||
every value into Open Dental or Eaglesoft. This agent eliminates that step entirely —
|
||||
one click sends the data straight into the right field.
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
{!runStartError && startCountdown === null && liveSteps.length === 0 && (
|
||||
<p className="text-sm text-muted-foreground">Starting run...</p>
|
||||
)}
|
||||
{liveSteps.map((step, i) => (
|
||||
<div key={i} className="flex items-start gap-3 text-sm">
|
||||
<div className="flex-shrink-0 mt-0.5">
|
||||
{step.status === "running" && <Loader2 className="h-4 w-4 animate-spin text-violet-500" />}
|
||||
{step.status === "done" && <CheckCircle2 className="h-4 w-4 text-emerald-500" />}
|
||||
{step.status === "error" && <XCircle className="h-4 w-4 text-destructive" />}
|
||||
</div>
|
||||
<div>
|
||||
<p className="leading-snug pt-0.5">{step.label}</p>
|
||||
{step.error && <p className="text-xs text-destructive mt-0.5">{step.error}</p>}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{steps.map((step, i) => (
|
||||
<div key={i} className="flex items-start gap-3 text-sm">
|
||||
<div className="flex-shrink-0 mt-0.5">
|
||||
<span className="flex h-5 w-5 items-center justify-center rounded-full bg-muted text-xs font-semibold text-muted-foreground">
|
||||
{i + 1}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-muted-foreground leading-snug pt-0.5">{step}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user