Gitead/DentalManagementMH07
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use Google Vision OCR instead of AI-vision estimates for Windows Type Agent clicks

Browse Source 
Claude vision's self-reported text coordinates had a small but consistent bias,
causing the patient-row double-click to land above the name and the Exam-column
divider count to occasionally miscount and hard-fail. Swap patient-row, Exam,
and Save button targeting to exact OCR bounding boxes (Google Vision via the
existing PaymentOCRService) instead of AI-estimated ratios.
This commit is contained in:
Gitead
2026-07-30 23:14:02 -04:00
parent 3f83794ac5
commit 5cdd3452f2
3 changed files with 141 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
apps/Backend/src/services/ocrLocate.ts Normal file
View File

@@ -0,0 +1,61 @@
import axios from "axios";
import FormData from "form-data";
interface OcrWord {
text: string;
left: number;
top: number;
w: number;
h: number;
cx: number;
cy: number;
}
// Finds every occurrence of exact text in a screenshot using real OCR (Google Vision via
// PaymentOCRService, already running for payment-document extraction) instead of asking an AI
// to visually estimate coordinates. Google Vision's word bounding boxes give an exact pixel
// center, so unlike locateAllOnScreenshot's AI-estimated top/bottom edges, there's no systematic
// bias to correct for.
//
// `text` may be multiple words (e.g. "Single click") — Google Vision reports one bounding box
// per individual word, not per phrase, so a multi-word target is matched as a run of consecutive
// words in the OCR result (which preserves reading order) and their boxes are merged into one.
export async function locateAllViaOcr(imageBase64: string, text: string): Promise<{ x: number; y: number }[]> {
const form = new FormData();
form.append("file", Buffer.from(imageBase64, "base64"), {
filename: "screenshot.png",
contentType: "image/png",
});
let words: OcrWord[];
try {
const resp = await axios.post<{ words: OcrWord[] }>("http://localhost:5003/extract/words", form, {
headers: form.getHeaders(),
maxBodyLength: Infinity,
maxContentLength: Infinity,
timeout: 30000,
});
words = resp.data?.words ?? [];
} catch (err: any) {
const status = err?.response?.status;
const detail = err?.response?.data?.detail || err?.message || "Unknown error";
throw new Error(`OCR request failed${status ? ` (${status})` : ""}: ${detail}`);
}
const targetTokens = text.trim().toLowerCase().split(/\s+/).filter(Boolean);
if (targetTokens.length === 0) return [];
const matches: { x: number; y: number }[] = [];
for (let i = 0; i <= words.length - targetTokens.length; i++) {
const span = words.slice(i, i + targetTokens.length);
const matchesHere = span.every((w, j) => w.text.trim().toLowerCase() === targetTokens[j]);
if (!matchesHere) continue;
const left = Math.min(...span.map((w) => w.left));
const top = Math.min(...span.map((w) => w.top));
const right = Math.max(...span.map((w) => w.left + w.w));
const bottom = Math.max(...span.map((w) => w.top + w.h));
matches.push({ x: Math.round((left + right) / 2), y: Math.round((top + bottom) / 2) });
}
return matches;
}

94
apps/Backend/src/services/typeAgentRunner.ts
View File

@@ -4,13 +4,12 @@ import {
confirmTextTyped,
cropAroundPoint,
cropToRegion,
detectColumnBoundaries,
detectRowHeightRatio,
diffBoundingBox,
locateAllOnScreenshot,
locateOnScreenshot,
WindowBounds,
} from "./visionLocate";
import { locateAllViaOcr } from "./ocrLocate";
import { backupTypeAgentScreenshot, logTypeAgentStep } from "../utils/screenshotBackup";
export type StepStatus = "running" | "done" | "error";
@@ -196,19 +195,20 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
const { region } = await captureWindowScreenshot(ctx);
backupTypeAgentScreenshot(ctx.runId, "patient_row_locate", region);
const headerMatches = await locateAllOnScreenshot(ctx.userId, region, "PatNum");
const headerMatches = await locateAllViaOcr(region, "PatNum");
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "PatNum header", matches: toScreenPoints(ctx, headerMatches) });
if (headerMatches.length === 0) throw new Error('Could not find the "PatNum" column header');
const header = headerMatches.reduce((a, b) => (a.y < b.y ? a : b));
// Locate the target patient's first and last name directly, rather than computing the row's
// y arithmetically (headerY + rowHeight * rowIndex) — a previous version did that and the
// click drifted further off-row the more rows down the target was, since any small error in
// the estimated row height got multiplied by the row index. Two real, independently-detected
// text positions and a same-row pairing between them is grounded in what's actually on
// screen instead of compounding an estimate.
const lastNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientLastName);
const firstNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientFirstName);
// Locate the target patient's first and last name directly via OCR (real pixel bounding
// boxes from Google Vision), rather than computing the row's y arithmetically (headerY +
// rowHeight * rowIndex) — a previous version did that and the click drifted further off-row
// the more rows down the target was. An earlier AI-vision version of this step (asking
// Claude to estimate each word's top/bottom edge) also had a small but consistent upward
// bias since it was an LLM's visual estimate, not a measurement. OCR bounding-box centers
// are exact, so the same-row pairing below needs no bias correction.
const lastNameMatches = await locateAllViaOcr(region, ctx.patientLastName);
const firstNameMatches = await locateAllViaOcr(region, ctx.patientFirstName);
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "last name", matches: toScreenPoints(ctx, lastNameMatches) });
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "first name", matches: toScreenPoints(ctx, firstNameMatches) });
if (lastNameMatches.length === 0) throw new Error(`Could not find "${ctx.patientLastName}" in the patient list`);
@@ -268,49 +268,37 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
// clickable single-click procedures list (middle), and the already-added procedures grid
// plus comm log (right). "Exam" appears both as a clickable item in the middle list AND
// as an already-added row in the right grid — telling them apart by y-position alone isn't
// reliable since both can land at a similar height. Finding the actual column dividers and
// keeping only the "Exam" match inside the middle column resolves this structurally.
const dividers = await detectColumnBoundaries(ctx.userId, region);
logTypeAgentStep(ctx.runId, { event: "column_boundaries", dividers: dividers.map((x) => toScreenPoint(ctx, { x, y: 0 }).x) });
if (dividers.length !== 2) {
throw new Error(`Expected 2 column dividers (3 columns), found ${dividers.length}`);
}
const [middleLeft, middleRight] = dividers as [number, number];
const examMatches = await locateAllOnScreenshot(ctx.userId, region, "Exam");
const singleClickMatches = await locateAllOnScreenshot(ctx.userId, region, "Single click");
// reliable since both can land at a similar height. Column-divider detection (asking AI to
// find the panel-separator lines) used to disambiguate this, but that's a visual-layout
// judgment call prone to over/under-counting dividers (seen in practice: 3 found instead of
// 2, hard-failing the whole step). "Single click" is a hint label that only ever appears
// once, directly above the correct clickable list — so the "Exam" occurrence nearest it in
// x is structurally the right one, without needing to know where the panel boundaries are.
const examMatches = await locateAllViaOcr(region, "Exam");
const singleClickMatches = await locateAllViaOcr(region, "Single click");
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Exam", matches: toScreenPoints(ctx, examMatches) });
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Single click", matches: toScreenPoints(ctx, singleClickMatches) });
if (examMatches.length === 0) throw new Error('Could not find "Exam" in the Edit Appointment window');
if (singleClickMatches.length === 0) throw new Error('Could not find "Single click" in the Edit Appointment window');
const inMiddleColumn = examMatches.filter((m) => m.x > middleLeft && m.x < middleRight);
if (inMiddleColumn.length === 0) {
throw new Error('Found "Exam", but none were in the middle column of the window');
}
// "Upper" — the topmost match in that column, in case more than one qualifies.
const examLoc = inMiddleColumn.reduce((a, b) => (a.y < b.y ? a : b));
// Topmost "Single click" occurrence, in case OCR ever reports more than one.
const singleClick = singleClickMatches.reduce((a, b) => (a.y < b.y ? a : b));
// "Single click" is the hint text right above this list — same row-height/grid-average
// technique used for the patient list, anchored on that text instead of a column header,
// so the click lands toward the middle of the Exam row rather than its top edge.
const singleClickY = Math.min(...singleClickMatches.map((m) => m.y));
const rowHeightRatio = await detectRowHeightRatio(ctx.userId, region);
const rowHeightPx = Math.round(rowHeightRatio * (ctx.windowBounds?.height ?? 0));
if (rowHeightPx <= 0) throw new Error(`AI reported an invalid row height ratio: ${rowHeightRatio}`);
const rowIndex = Math.max(1, Math.round((examLoc.y - singleClickY) / rowHeightPx));
const gridY = singleClickY + rowHeightPx * rowIndex;
const finalY = Math.round((examLoc.y + gridY) / 2);
// Nearest "Exam" match to that x — the one in the unrelated right-side grid sits in a
// different column, far enough in x not to compete with the real target.
const examLoc = examMatches.reduce((a, b) =>
Math.abs(a.x - singleClick.x) <= Math.abs(b.x - singleClick.x) ? a : b
);
const screen = toScreenPoint(ctx, { x: examLoc.x, y: finalY });
// OCR's bounding-box center is exact, so the click lands directly on it — no row-height
// grid estimate needed to nudge toward a row's middle.
const screen = toScreenPoint(ctx, examLoc);
logTypeAgentStep(ctx.runId, {
event: "exam_match",
pixelX: screen.x,
directY: toScreenPoint(ctx, { x: 0, y: examLoc.y }).y,
gridY: toScreenPoint(ctx, { x: 0, y: gridY }).y,
finalY: screen.y,
rowHeightPx,
singleClickMatch: toScreenPoint(ctx, singleClick),
});
await debugClickAt(ctx, screen.x, screen.y, "exam");
@@ -322,16 +310,18 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
execute: async (ctx) => {
const { region } = await captureWindowScreenshot(ctx);
backupTypeAgentScreenshot(ctx.runId, "save_locate", region);
const loc = await locateOnScreenshot(
ctx.userId,
region,
"the Save button in the Edit Appointment window — it's in the LOWER part of the " +
"RIGHTMOST of the window's three big columns (the one with the already-added " +
"procedures grid and comm log above it), below the other action buttons in that column " +
'(e.g. "Delete", "To Task List", "Audit Trail")'
);
if (!loc) throw new Error("Could not find the Save button in the Edit Appointment window");
const screen = toScreenPoint(ctx, loc);
const saveMatches = await locateAllViaOcr(region, "Save");
logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Save", matches: toScreenPoints(ctx, saveMatches) });
if (saveMatches.length === 0) throw new Error("Could not find the Save button in the Edit Appointment window");
// "Save" only ever appears once as a button label in this window, but if OCR ever reports
// more than one, the real one is in the LOWER part of the RIGHTMOST column (below the other
// action buttons there, e.g. "Delete", "To Task List", "Audit Trail") — so the bottom-right-
// most occurrence is the correct disambiguation, same reasoning the old AI-vision prompt used.
const saveLoc = saveMatches.reduce((a, b) => (a.x + a.y >= b.x + b.y ? a : b));
const screen = toScreenPoint(ctx, saveLoc);
logTypeAgentStep(ctx.runId, { event: "locate", label: "Save", pixelX: screen.x, pixelY: screen.y });
await debugClickAt(ctx, screen.x, screen.y, "save");

37
apps/PaymentOCRService/main.py
View File

@@ -5,11 +5,13 @@ from typing import List, Optional
import io
import os
import asyncio
import tempfile
import uvicorn
from dotenv import load_dotenv
load_dotenv()
from complete_pipeline_adapter import process_images_to_rows,rows_to_csv_bytes
from complete_pipeline import extract_words_and_text
from pdf_extractor import extract_ra_pdf
app = FastAPI(
@@ -98,6 +100,41 @@ async def extract_json(files: List[UploadFile] = File(...)):
async with lock:
active_jobs -= 1
@app.post("/extract/words")
async def extract_words(file: UploadFile = File(...)):
"""Raw OCR only — no deskew/line-grouping/payment-domain logic, just the
flat word list with pixel bounding boxes. Used by the Windows Type Agent
to locate exact click targets (e.g. a patient name) instead of relying
on an AI-vision estimate of where text is."""
_validate_files([file])
async with lock:
global waiting_jobs
waiting_jobs += 1
async with semaphore:
async with lock:
waiting_jobs -= 1
global active_jobs
active_jobs += 1
tmp_path = None
try:
suffix = os.path.splitext(file.filename or "")[1] or ".png"
blob = await file.read()
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(blob)
tmp_path = tmp.name
words, _ = extract_words_and_text(tmp_path)
return JSONResponse(content={"words": words})
except Exception as e:
raise HTTPException(status_code=500, detail=f"OCR error: {e}")
finally:
if tmp_path and os.path.exists(tmp_path):
os.remove(tmp_path)
async with lock:
active_jobs -= 1
@app.post("/extract/csvtext", response_class=PlainTextResponse)
async def extract_csvtext(files: List[UploadFile] = File(...)):
_validate_files(files)