Gitead
ad2e15ec62
Replaces fragile whole-screen text matching with a screenshot-diff step that finds each dialog's actual pixel bounds, then restricts every subsequent AI locate/click to that cropped region — eliminating false matches from text elsewhere on screen (title bars, side panels). Adds column-boundary and row-height detection so the patient row and Exam procedure click positions are computed geometrically instead of relying on repeated fuzzy AI guesses for visually similar neighbors. Also adds per-run screenshot/debug-crop backups and a structured run.log for diagnosing failed runs, plus a cmd:move primitive on the Windows agent for pre-click confirmation crops. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
74 lines
2.1 KiB
JavaScript
74 lines
2.1 KiB
JavaScript
// Simulates DentalAgent.exe connecting to the backend's /agent socket.io namespace,
|
|
// so the Type Agent page's "Windows Agent: Connected" status can be tested end-to-end
|
|
// before the real pyautogui-based Windows client exists.
|
|
//
|
|
// Usage: node scripts/fake-windows-agent.js [serverUrl]
|
|
// Requires WINDOWS_AGENT_TOKEN to match the backend's .env value.
|
|
// The server identifies agents by connection IP address, not anything sent here.
|
|
|
|
const { io } = require("socket.io-client");
|
|
|
|
const serverUrl = process.argv[2] || "http://localhost:5000";
|
|
const token = process.env.WINDOWS_AGENT_TOKEN || "dev-windows-agent-token";
|
|
|
|
const socket = io(`${serverUrl}/agent`, {
|
|
auth: { token },
|
|
reconnectionAttempts: 5,
|
|
});
|
|
|
|
socket.on("connect", () => {
|
|
console.log(`✅ Connected as fake Windows Agent, socket id: ${socket.id}`);
|
|
});
|
|
|
|
// 1x1 transparent PNG, base64 — stands in for a real screen capture.
|
|
const FAKE_SCREENSHOT =
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
|
|
|
|
socket.on("cmd:screenshot", (_payload, ack) => {
|
|
console.log("📸 screenshot requested");
|
|
ack({ image: FAKE_SCREENSHOT });
|
|
});
|
|
|
|
socket.on("cmd:move", ({ x, y }, ack) => {
|
|
console.log(`🖱️ move to (${x}, ${y}) (no click)`);
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("cmd:click", ({ x, y }, ack) => {
|
|
console.log(`🖱️ click at (${x}, ${y})`);
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("cmd:double_click", ({ x, y }, ack) => {
|
|
console.log(`🖱️ double-click at (${x}, ${y})`);
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("cmd:double_click_current", (_payload, ack) => {
|
|
console.log("🖱️ double-click at current cursor position");
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("cmd:type", ({ text }, ack) => {
|
|
console.log(`⌨️ type "${text}"`);
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("cmd:key", ({ key }, ack) => {
|
|
console.log(`⌨️ press key "${key}"`);
|
|
ack({ ok: true });
|
|
});
|
|
|
|
socket.on("connect_error", (err) => {
|
|
console.error("❌ Connection failed:", err.message);
|
|
});
|
|
|
|
socket.on("disconnect", (reason) => {
|
|
console.log("🔌 Disconnected:", reason);
|
|
});
|
|
|
|
process.on("SIGINT", () => {
|
|
socket.close();
|
|
process.exit(0);
|
|
});
|