Gitead
ad2e15ec62
Replaces fragile whole-screen text matching with a screenshot-diff step that finds each dialog's actual pixel bounds, then restricts every subsequent AI locate/click to that cropped region — eliminating false matches from text elsewhere on screen (title bars, side panels). Adds column-boundary and row-height detection so the patient row and Exam procedure click positions are computed geometrically instead of relying on repeated fuzzy AI guesses for visually similar neighbors. Also adds per-run screenshot/debug-crop backups and a structured run.log for diagnosing failed runs, plus a cmd:move primitive on the Windows agent for pre-click confirmation crops. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
226 lines
8.1 KiB
Python
226 lines
8.1 KiB
Python
"""
|
|
DentalAgent — runs on a staff front-desk Windows PC. Connects to the app's server over
|
|
the /agent socket.io namespace and executes low-level mouse/keyboard/screenshot commands
|
|
sent by the server. It has no knowledge of dental software, patients, or workflows — the
|
|
app's AI decides what to click/type by reading the screenshots this agent sends back.
|
|
|
|
Configuration (via a small popup on first run, saved to agent_config.json next to the exe):
|
|
- Server URL (e.g. http://192.168.0.240:5000) — the only thing staff need to enter.
|
|
|
|
The server identifies which PC is which by connection IP address (shown on the Type Agent
|
|
page), so there's no Office ID to type. The AGENT_TOKEN below authenticates the exe itself
|
|
to the server — set it to match the server's WINDOWS_AGENT_TOKEN before distributing this
|
|
to real front-desk PCs; it's baked into the build, not something staff ever see or type.
|
|
|
|
After connecting it has no visible window — just a system tray icon showing
|
|
Connected/Connecting/Disconnected. Clicking the icon reopens the same setup window
|
|
(pre-filled with the current URL) without restarting the process — a self-relaunch turned
|
|
out to be unreliable with PyInstaller --onefile builds on Windows (the freshly spawned copy
|
|
can crash trying to reuse the outgoing process's temp extraction folder), so instead a single
|
|
hidden Tk window is kept alive for the whole run and just shown/hidden as needed.
|
|
"""
|
|
|
|
AGENT_TOKEN = "dev-windows-agent-token"
|
|
|
|
import base64
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
import tkinter as tk
|
|
from tkinter import messagebox
|
|
|
|
import pyautogui
|
|
import pystray
|
|
import socketio
|
|
from PIL import Image, ImageDraw, ImageGrab
|
|
|
|
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "agent_config.json")
|
|
|
|
pyautogui.FAILSAFE = True # moving mouse to a screen corner aborts an in-progress action
|
|
|
|
|
|
def load_config():
|
|
if os.path.exists(CONFIG_PATH):
|
|
with open(CONFIG_PATH, "r") as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
|
|
def save_config(config):
|
|
with open(CONFIG_PATH, "w") as f:
|
|
json.dump(config, f)
|
|
|
|
|
|
def make_status_icon(color):
|
|
image = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.ellipse((8, 8, 56, 56), fill=color)
|
|
return image
|
|
|
|
|
|
ICON_CONNECTED = make_status_icon("#22c55e")
|
|
ICON_DISCONNECTED = make_status_icon("#ef4444")
|
|
ICON_CONNECTING = make_status_icon("#9ca3af")
|
|
ICON_NOT_CONFIGURED = make_status_icon("#9ca3af")
|
|
|
|
|
|
class DentalAgent:
|
|
def __init__(self):
|
|
self.server_url = None
|
|
self.status = "Not configured"
|
|
self.tray_icon = None
|
|
self.sio = socketio.Client(reconnection=True, reconnection_attempts=0)
|
|
self._register_handlers()
|
|
|
|
def _set_status(self, status, icon_image):
|
|
self.status = status
|
|
if self.tray_icon:
|
|
self.tray_icon.icon = icon_image
|
|
self.tray_icon.title = f"Dental Agent - {status}"
|
|
self.tray_icon.update_menu()
|
|
|
|
def _register_handlers(self):
|
|
sio = self.sio
|
|
|
|
@sio.event(namespace="/agent")
|
|
def connect():
|
|
print(f"Connected to {self.server_url}")
|
|
self._set_status("Connected", ICON_CONNECTED)
|
|
|
|
@sio.event(namespace="/agent")
|
|
def connect_error(data):
|
|
print(f"Connection failed: {data}")
|
|
self._set_status("Connection failed", ICON_DISCONNECTED)
|
|
|
|
@sio.event(namespace="/agent")
|
|
def disconnect():
|
|
print("Disconnected")
|
|
self._set_status("Disconnected", ICON_DISCONNECTED)
|
|
|
|
@sio.on("cmd:screenshot", namespace="/agent")
|
|
def on_screenshot(_data=None):
|
|
image = ImageGrab.grab()
|
|
buf = io.BytesIO()
|
|
image.save(buf, format="PNG")
|
|
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
|
|
return {"image": encoded}
|
|
|
|
@sio.on("cmd:move", namespace="/agent")
|
|
def on_move(data):
|
|
# Moves the cursor without clicking — used to double-check a location (via a
|
|
# follow-up cropped screenshot) before committing to a click.
|
|
pyautogui.moveTo(data["x"], data["y"])
|
|
return {"ok": True}
|
|
|
|
@sio.on("cmd:click", namespace="/agent")
|
|
def on_click(data):
|
|
pyautogui.click(data["x"], data["y"])
|
|
return {"ok": True}
|
|
|
|
@sio.on("cmd:double_click", namespace="/agent")
|
|
def on_double_click(data):
|
|
pyautogui.doubleClick(data["x"], data["y"])
|
|
return {"ok": True}
|
|
|
|
@sio.on("cmd:double_click_current", namespace="/agent")
|
|
def on_double_click_current(_data=None):
|
|
# No x/y — clicks wherever the mouse already is. Lets staff pre-position the
|
|
# cursor over the target (e.g. an open time slot) before minimizing the browser,
|
|
# so this step doesn't need a screenshot/vision lookup at all.
|
|
pyautogui.doubleClick()
|
|
return {"ok": True}
|
|
|
|
@sio.on("cmd:type", namespace="/agent")
|
|
def on_type(data):
|
|
pyautogui.write(data["text"], interval=0.02)
|
|
return {"ok": True}
|
|
|
|
@sio.on("cmd:key", namespace="/agent")
|
|
def on_key(data):
|
|
pyautogui.press(data["key"])
|
|
return {"ok": True}
|
|
|
|
def connect_to(self, server_url):
|
|
"""(Re)connects to server_url, reusing the same socketio.Client for the process's
|
|
whole lifetime — safe to call again later with a new URL to switch servers."""
|
|
self.server_url = server_url
|
|
self._set_status("Connecting...", ICON_CONNECTING)
|
|
|
|
def _run():
|
|
try:
|
|
if self.sio.connected:
|
|
self.sio.disconnect()
|
|
self.sio.connect(
|
|
server_url,
|
|
namespaces=["/agent"],
|
|
auth={"token": AGENT_TOKEN},
|
|
wait_timeout=10,
|
|
)
|
|
self.sio.wait()
|
|
except Exception as exc:
|
|
print(f"Agent stopped: {exc}")
|
|
self._set_status("Connection failed", ICON_DISCONNECTED)
|
|
|
|
threading.Thread(target=_run, daemon=True).start()
|
|
|
|
|
|
def main():
|
|
config = load_config()
|
|
agent = DentalAgent()
|
|
|
|
# --- Settings window: built once, shown/hidden for the process's whole lifetime ---
|
|
root = tk.Tk()
|
|
root.title("Dental Agent Setup")
|
|
root.resizable(False, False)
|
|
root.withdraw()
|
|
root.protocol("WM_DELETE_WINDOW", root.withdraw) # closing the window just hides it
|
|
|
|
tk.Label(root, text="Server URL").pack(pady=(15, 0))
|
|
server_entry = tk.Entry(root, width=40)
|
|
server_entry.pack(padx=15)
|
|
|
|
def on_connect():
|
|
server_url = server_entry.get().strip()
|
|
if not server_url:
|
|
messagebox.showerror("Dental Agent", "Server URL is required.")
|
|
return
|
|
save_config({"server_url": server_url})
|
|
root.withdraw()
|
|
agent.connect_to(server_url)
|
|
|
|
tk.Button(root, text="Connect", command=on_connect).pack(pady=15)
|
|
root.bind("<Return>", lambda _event: on_connect())
|
|
|
|
def show_settings():
|
|
server_entry.delete(0, tk.END)
|
|
server_entry.insert(0, agent.server_url or config.get("server_url", "http://"))
|
|
root.deiconify()
|
|
root.eval("tk::PlaceWindow . center")
|
|
root.lift()
|
|
root.focus_force()
|
|
|
|
# --- Tray icon: runs on its own thread so Tk can own the main thread/event loop ---
|
|
menu = pystray.Menu(
|
|
pystray.MenuItem(lambda _item: f"Status: {agent.status}", None, enabled=False),
|
|
pystray.MenuItem(lambda _item: f"Server: {agent.server_url or '(not set)'}", None, enabled=False),
|
|
pystray.Menu.SEPARATOR,
|
|
pystray.MenuItem("Settings...", lambda: root.after(0, show_settings), default=True),
|
|
pystray.MenuItem("Quit", lambda icon: (icon.stop(), root.after(0, root.quit))),
|
|
)
|
|
icon = pystray.Icon("DentalAgent", ICON_NOT_CONFIGURED, "Dental Agent - Not configured", menu)
|
|
agent.tray_icon = icon
|
|
threading.Thread(target=icon.run, daemon=True).start()
|
|
|
|
if "server_url" in config:
|
|
agent.connect_to(config["server_url"])
|
|
else:
|
|
show_settings() # first run — nothing saved yet, ask right away
|
|
|
|
root.mainloop()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|