feat: AI Type Agent - vision-based Open Dental automation via Windows agent
Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend services to locate UI elements via vision and drive an existing-patient appointment flow in Open Dental, wired into the Copy/Type Agent page and socket progress updates. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
68
apps/Backend/scripts/fake-windows-agent.js
Normal file
68
apps/Backend/scripts/fake-windows-agent.js
Normal file
@@ -0,0 +1,68 @@
|
||||
// Simulates DentalAgent.exe connecting to the backend's /agent socket.io namespace,
|
||||
// so the Type Agent page's "Windows Agent: Connected" status can be tested end-to-end
|
||||
// before the real pyautogui-based Windows client exists.
|
||||
//
|
||||
// Usage: node scripts/fake-windows-agent.js [serverUrl]
|
||||
// Requires WINDOWS_AGENT_TOKEN to match the backend's .env value.
|
||||
// The server identifies agents by connection IP address, not anything sent here.
|
||||
|
||||
const { io } = require("socket.io-client");
|
||||
|
||||
const serverUrl = process.argv[2] || "http://localhost:5000";
|
||||
const token = process.env.WINDOWS_AGENT_TOKEN || "dev-windows-agent-token";
|
||||
|
||||
const socket = io(`${serverUrl}/agent`, {
|
||||
auth: { token },
|
||||
reconnectionAttempts: 5,
|
||||
});
|
||||
|
||||
socket.on("connect", () => {
|
||||
console.log(`✅ Connected as fake Windows Agent, socket id: ${socket.id}`);
|
||||
});
|
||||
|
||||
// 1x1 transparent PNG, base64 — stands in for a real screen capture.
|
||||
const FAKE_SCREENSHOT =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
|
||||
|
||||
socket.on("cmd:screenshot", (_payload, ack) => {
|
||||
console.log("📸 screenshot requested");
|
||||
ack({ image: FAKE_SCREENSHOT });
|
||||
});
|
||||
|
||||
socket.on("cmd:click", ({ x, y }, ack) => {
|
||||
console.log(`🖱️ click at (${x}, ${y})`);
|
||||
ack({ ok: true });
|
||||
});
|
||||
|
||||
socket.on("cmd:double_click", ({ x, y }, ack) => {
|
||||
console.log(`🖱️ double-click at (${x}, ${y})`);
|
||||
ack({ ok: true });
|
||||
});
|
||||
|
||||
socket.on("cmd:double_click_current", (_payload, ack) => {
|
||||
console.log("🖱️ double-click at current cursor position");
|
||||
ack({ ok: true });
|
||||
});
|
||||
|
||||
socket.on("cmd:type", ({ text }, ack) => {
|
||||
console.log(`⌨️ type "${text}"`);
|
||||
ack({ ok: true });
|
||||
});
|
||||
|
||||
socket.on("cmd:key", ({ key }, ack) => {
|
||||
console.log(`⌨️ press key "${key}"`);
|
||||
ack({ ok: true });
|
||||
});
|
||||
|
||||
socket.on("connect_error", (err) => {
|
||||
console.error("❌ Connection failed:", err.message);
|
||||
});
|
||||
|
||||
socket.on("disconnect", (reason) => {
|
||||
console.log("🔌 Disconnected:", reason);
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
socket.close();
|
||||
process.exit(0);
|
||||
});
|
||||
Reference in New Issue
Block a user