Gitead/DentalManagementMH07
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: window-diff-based Type Agent locating for Open Dental appointment flow

Browse Source 
Replaces fragile whole-screen text matching with a screenshot-diff step that
finds each dialog's actual pixel bounds, then restricts every subsequent AI
locate/click to that cropped region — eliminating false matches from text
elsewhere on screen (title bars, side panels). Adds column-boundary and
row-height detection so the patient row and Exam procedure click positions
are computed geometrically instead of relying on repeated fuzzy AI guesses
for visually similar neighbors. Also adds per-run screenshot/debug-crop
backups and a structured run.log for diagnosing failed runs, plus a
cmd:move primitive on the Windows agent for pre-click confirmation crops.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Gitead
2026-07-27 08:50:47 -04:00
parent 59f064583a
commit ad2e15ec62
8 changed files with 817 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
apps/Backend/src/utils/screenshotBackup.ts
View File

@@ -19,3 +19,38 @@ export function backupScreenshots(files: { originalname: string; buffer: Buffer
console.error("[screenshotBackup] failed to write backup:", err);
}
}
function getTypeAgentRunDir(runId: string): string {
const dateDir = new Date().toISOString().slice(0, 10); // YYYY-MM-DD
return path.join(BACKUP_ROOT, "type-agent", dateDir, runId);
}
// Same convention as backupScreenshots, but grouped under a "type-agent/<date>/<runId>"
// subfolder so every screenshot from one Type Agent run stays together for debugging
// (e.g. "locate" and "confirm" images per step, in click order via the timestamp prefix).
export function backupTypeAgentScreenshot(runId: string, label: string, imageBase64: string): void {
try {
const dir = getTypeAgentRunDir(runId);
fs.mkdirSync(dir, { recursive: true });
const safeLabel = label.replace(/[/\\?%*:|"<>]/g, "-").slice(0, 80);
const fileName = `${Date.now()}_${safeLabel}.png`;
fs.writeFileSync(path.join(dir, fileName), Buffer.from(imageBase64, "base64"));
} catch (err) {
console.error("[screenshotBackup] failed to write type-agent backup:", err);
}
}
// One human-readable line per event (step start/attempt/locate/confirm/click/error), appended
// to run.log in the same per-run folder as that run's screenshots — so opening one folder
// shows both what was clicked and what it looked like at the time, including the exact
// AI-reported ratio for every locate attempt.
export function logTypeAgentStep(runId: string, entry: Record<string, unknown>): void {
try {
const dir = getTypeAgentRunDir(runId);
fs.mkdirSync(dir, { recursive: true });
const line = `[${new Date().toISOString()}] ${JSON.stringify(entry)}\n`;
fs.appendFileSync(path.join(dir, "run.log"), line);
} catch (err) {
console.error("[screenshotBackup] failed to write type-agent log:", err);
}
}