GiteadandClaude Sonnet 5
59f064583a
Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend services to locate UI elements via vision and drive an existing-patient appointment flow in Open Dental, wired into the Copy/Type Agent page and socket progress updates. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
59 lines
2.2 KiB
TypeScript
59 lines
2.2 KiB
TypeScript
import { io } from "../socket";
|
|
|
|
const COMMAND_TIMEOUT_MS = 10000;
|
|
|
|
// Picks a connected Windows Agent socket by IP address — falls back to the first connected
|
|
// agent if none is given (fine while offices only ever have one PC connected).
|
|
function getAgentSocket(ip?: string) {
|
|
if (!io) throw new Error("Socket.io not initialized");
|
|
const namespaceSockets = Array.from(io.of("/agent").sockets.values());
|
|
const target = ip
|
|
? namespaceSockets.find((s) => {
|
|
const socketIp = (s.handshake.headers["x-forwarded-for"] as string)?.split(",")[0]?.trim()
|
|
|| s.handshake.address;
|
|
return socketIp === ip;
|
|
})
|
|
: namespaceSockets[0];
|
|
if (!target) throw new Error("No Windows Agent connected");
|
|
return target;
|
|
}
|
|
|
|
async function sendCommand<T>(event: string, payload: unknown, ip?: string): Promise<T> {
|
|
const socket = getAgentSocket(ip);
|
|
return socket.timeout(COMMAND_TIMEOUT_MS).emitWithAck(event, payload) as Promise<T>;
|
|
}
|
|
|
|
// Captures the Windows PC's screen. Returns a base64-encoded PNG for the app's AI to read.
|
|
export function captureScreenshot(ip?: string) {
|
|
return sendCommand<{ image: string }>("cmd:screenshot", {}, ip);
|
|
}
|
|
|
|
export function click(x: number, y: number, ip?: string) {
|
|
return sendCommand<{ ok: boolean }>("cmd:click", { x, y }, ip);
|
|
}
|
|
|
|
export function doubleClick(x: number, y: number, ip?: string) {
|
|
return sendCommand<{ ok: boolean }>("cmd:double_click", { x, y }, ip);
|
|
}
|
|
|
|
// Double-clicks wherever the mouse already is — no coordinates needed. Used for the first
|
|
// step of a run, where staff pre-position the cursor before triggering it from the browser.
|
|
export function doubleClickCurrent(ip?: string) {
|
|
return sendCommand<{ ok: boolean }>("cmd:double_click_current", {}, ip);
|
|
}
|
|
|
|
export function typeText(text: string, ip?: string) {
|
|
return sendCommand<{ ok: boolean }>("cmd:type", { text }, ip);
|
|
}
|
|
|
|
export function pressKey(key: string, ip?: string) {
|
|
return sendCommand<{ ok: boolean }>("cmd:key", { key }, ip);
|
|
}
|
|
|
|
// Forcibly closes a connected agent's socket — used by the "x" on the Type Agent page to
|
|
// let staff stop controlling a PC without touching the exe on that machine.
|
|
export function disconnectAgent(ip: string) {
|
|
const socket = getAgentSocket(ip);
|
|
socket.disconnect(true);
|
|
}
|