Files
DentalManagementMH07/apps/WindowsAgent/agent.py
Gitead 59f064583a feat: AI Type Agent - vision-based Open Dental automation via Windows agent
Adds a Windows-side agent (screenshot/click/type over HTTP) plus backend
services to locate UI elements via vision and drive an existing-patient
appointment flow in Open Dental, wired into the Copy/Type Agent page and
socket progress updates.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-25 17:24:27 -04:00

219 lines
7.8 KiB
Python

"""
DentalAgent — runs on a staff front-desk Windows PC. Connects to the app's server over
the /agent socket.io namespace and executes low-level mouse/keyboard/screenshot commands
sent by the server. It has no knowledge of dental software, patients, or workflows — the
app's AI decides what to click/type by reading the screenshots this agent sends back.
Configuration (via a small popup on first run, saved to agent_config.json next to the exe):
- Server URL (e.g. http://192.168.0.240:5000) — the only thing staff need to enter.
The server identifies which PC is which by connection IP address (shown on the Type Agent
page), so there's no Office ID to type. The AGENT_TOKEN below authenticates the exe itself
to the server — set it to match the server's WINDOWS_AGENT_TOKEN before distributing this
to real front-desk PCs; it's baked into the build, not something staff ever see or type.
After connecting it has no visible window — just a system tray icon showing
Connected/Connecting/Disconnected. Clicking the icon reopens the same setup window
(pre-filled with the current URL) without restarting the process — a self-relaunch turned
out to be unreliable with PyInstaller --onefile builds on Windows (the freshly spawned copy
can crash trying to reuse the outgoing process's temp extraction folder), so instead a single
hidden Tk window is kept alive for the whole run and just shown/hidden as needed.
"""
AGENT_TOKEN = "dev-windows-agent-token"
import base64
import io
import json
import os
import sys
import threading
import tkinter as tk
from tkinter import messagebox
import pyautogui
import pystray
import socketio
from PIL import Image, ImageDraw, ImageGrab
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "agent_config.json")
pyautogui.FAILSAFE = True # moving mouse to a screen corner aborts an in-progress action
def load_config():
if os.path.exists(CONFIG_PATH):
with open(CONFIG_PATH, "r") as f:
return json.load(f)
return {}
def save_config(config):
with open(CONFIG_PATH, "w") as f:
json.dump(config, f)
def make_status_icon(color):
image = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
draw = ImageDraw.Draw(image)
draw.ellipse((8, 8, 56, 56), fill=color)
return image
ICON_CONNECTED = make_status_icon("#22c55e")
ICON_DISCONNECTED = make_status_icon("#ef4444")
ICON_CONNECTING = make_status_icon("#9ca3af")
ICON_NOT_CONFIGURED = make_status_icon("#9ca3af")
class DentalAgent:
def __init__(self):
self.server_url = None
self.status = "Not configured"
self.tray_icon = None
self.sio = socketio.Client(reconnection=True, reconnection_attempts=0)
self._register_handlers()
def _set_status(self, status, icon_image):
self.status = status
if self.tray_icon:
self.tray_icon.icon = icon_image
self.tray_icon.title = f"Dental Agent - {status}"
self.tray_icon.update_menu()
def _register_handlers(self):
sio = self.sio
@sio.event(namespace="/agent")
def connect():
print(f"Connected to {self.server_url}")
self._set_status("Connected", ICON_CONNECTED)
@sio.event(namespace="/agent")
def connect_error(data):
print(f"Connection failed: {data}")
self._set_status("Connection failed", ICON_DISCONNECTED)
@sio.event(namespace="/agent")
def disconnect():
print("Disconnected")
self._set_status("Disconnected", ICON_DISCONNECTED)
@sio.on("cmd:screenshot", namespace="/agent")
def on_screenshot(_data=None):
image = ImageGrab.grab()
buf = io.BytesIO()
image.save(buf, format="PNG")
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
return {"image": encoded}
@sio.on("cmd:click", namespace="/agent")
def on_click(data):
pyautogui.click(data["x"], data["y"])
return {"ok": True}
@sio.on("cmd:double_click", namespace="/agent")
def on_double_click(data):
pyautogui.doubleClick(data["x"], data["y"])
return {"ok": True}
@sio.on("cmd:double_click_current", namespace="/agent")
def on_double_click_current(_data=None):
# No x/y — clicks wherever the mouse already is. Lets staff pre-position the
# cursor over the target (e.g. an open time slot) before minimizing the browser,
# so this step doesn't need a screenshot/vision lookup at all.
pyautogui.doubleClick()
return {"ok": True}
@sio.on("cmd:type", namespace="/agent")
def on_type(data):
pyautogui.write(data["text"], interval=0.02)
return {"ok": True}
@sio.on("cmd:key", namespace="/agent")
def on_key(data):
pyautogui.press(data["key"])
return {"ok": True}
def connect_to(self, server_url):
"""(Re)connects to server_url, reusing the same socketio.Client for the process's
whole lifetime — safe to call again later with a new URL to switch servers."""
self.server_url = server_url
self._set_status("Connecting...", ICON_CONNECTING)
def _run():
try:
if self.sio.connected:
self.sio.disconnect()
self.sio.connect(
server_url,
namespaces=["/agent"],
auth={"token": AGENT_TOKEN},
wait_timeout=10,
)
self.sio.wait()
except Exception as exc:
print(f"Agent stopped: {exc}")
self._set_status("Connection failed", ICON_DISCONNECTED)
threading.Thread(target=_run, daemon=True).start()
def main():
config = load_config()
agent = DentalAgent()
# --- Settings window: built once, shown/hidden for the process's whole lifetime ---
root = tk.Tk()
root.title("Dental Agent Setup")
root.resizable(False, False)
root.withdraw()
root.protocol("WM_DELETE_WINDOW", root.withdraw) # closing the window just hides it
tk.Label(root, text="Server URL").pack(pady=(15, 0))
server_entry = tk.Entry(root, width=40)
server_entry.pack(padx=15)
def on_connect():
server_url = server_entry.get().strip()
if not server_url:
messagebox.showerror("Dental Agent", "Server URL is required.")
return
save_config({"server_url": server_url})
root.withdraw()
agent.connect_to(server_url)
tk.Button(root, text="Connect", command=on_connect).pack(pady=15)
root.bind("<Return>", lambda _event: on_connect())
def show_settings():
server_entry.delete(0, tk.END)
server_entry.insert(0, agent.server_url or config.get("server_url", "http://"))
root.deiconify()
root.eval("tk::PlaceWindow . center")
root.lift()
root.focus_force()
# --- Tray icon: runs on its own thread so Tk can own the main thread/event loop ---
menu = pystray.Menu(
pystray.MenuItem(lambda _item: f"Status: {agent.status}", None, enabled=False),
pystray.MenuItem(lambda _item: f"Server: {agent.server_url or '(not set)'}", None, enabled=False),
pystray.Menu.SEPARATOR,
pystray.MenuItem("Settings...", lambda: root.after(0, show_settings), default=True),
pystray.MenuItem("Quit", lambda icon: (icon.stop(), root.after(0, root.quit))),
)
icon = pystray.Icon("DentalAgent", ICON_NOT_CONFIGURED, "Dental Agent - Not configured", menu)
agent.tray_icon = icon
threading.Thread(target=icon.run, daemon=True).start()
if "server_url" in config:
agent.connect_to(config["server_url"])
else:
show_settings() # first run — nothing saved yet, ask right away
root.mainloop()
if __name__ == "__main__":
main()