From 5cdd3452f2a7f40939c9c440528b9bce74233517 Mon Sep 17 00:00:00 2001 From: Gitead Date: Thu, 30 Jul 2026 23:14:02 -0400 Subject: [PATCH] fix: use Google Vision OCR instead of AI-vision estimates for Windows Type Agent clicks Claude vision's self-reported text coordinates had a small but consistent bias, causing the patient-row double-click to land above the name and the Exam-column divider count to occasionally miscount and hard-fail. Swap patient-row, Exam, and Save button targeting to exact OCR bounding boxes (Google Vision via the existing PaymentOCRService) instead of AI-estimated ratios. --- apps/Backend/src/services/ocrLocate.ts | 61 +++++++++++++ apps/Backend/src/services/typeAgentRunner.ts | 94 +++++++++----------- apps/PaymentOCRService/main.py | 39 +++++++- 3 files changed, 141 insertions(+), 53 deletions(-) create mode 100644 apps/Backend/src/services/ocrLocate.ts diff --git a/apps/Backend/src/services/ocrLocate.ts b/apps/Backend/src/services/ocrLocate.ts new file mode 100644 index 00000000..140cdfb5 --- /dev/null +++ b/apps/Backend/src/services/ocrLocate.ts @@ -0,0 +1,61 @@ +import axios from "axios"; +import FormData from "form-data"; + +interface OcrWord { + text: string; + left: number; + top: number; + w: number; + h: number; + cx: number; + cy: number; +} + +// Finds every occurrence of exact text in a screenshot using real OCR (Google Vision via +// PaymentOCRService, already running for payment-document extraction) instead of asking an AI +// to visually estimate coordinates. Google Vision's word bounding boxes give an exact pixel +// center, so unlike locateAllOnScreenshot's AI-estimated top/bottom edges, there's no systematic +// bias to correct for. +// +// `text` may be multiple words (e.g. "Single click") — Google Vision reports one bounding box +// per individual word, not per phrase, so a multi-word target is matched as a run of consecutive +// words in the OCR result (which preserves reading order) and their boxes are merged into one. +export async function locateAllViaOcr(imageBase64: string, text: string): Promise<{ x: number; y: number }[]> { + const form = new FormData(); + form.append("file", Buffer.from(imageBase64, "base64"), { + filename: "screenshot.png", + contentType: "image/png", + }); + + let words: OcrWord[]; + try { + const resp = await axios.post<{ words: OcrWord[] }>("http://localhost:5003/extract/words", form, { + headers: form.getHeaders(), + maxBodyLength: Infinity, + maxContentLength: Infinity, + timeout: 30000, + }); + words = resp.data?.words ?? []; + } catch (err: any) { + const status = err?.response?.status; + const detail = err?.response?.data?.detail || err?.message || "Unknown error"; + throw new Error(`OCR request failed${status ? ` (${status})` : ""}: ${detail}`); + } + + const targetTokens = text.trim().toLowerCase().split(/\s+/).filter(Boolean); + if (targetTokens.length === 0) return []; + + const matches: { x: number; y: number }[] = []; + for (let i = 0; i <= words.length - targetTokens.length; i++) { + const span = words.slice(i, i + targetTokens.length); + const matchesHere = span.every((w, j) => w.text.trim().toLowerCase() === targetTokens[j]); + if (!matchesHere) continue; + + const left = Math.min(...span.map((w) => w.left)); + const top = Math.min(...span.map((w) => w.top)); + const right = Math.max(...span.map((w) => w.left + w.w)); + const bottom = Math.max(...span.map((w) => w.top + w.h)); + matches.push({ x: Math.round((left + right) / 2), y: Math.round((top + bottom) / 2) }); + } + return matches; +} diff --git a/apps/Backend/src/services/typeAgentRunner.ts b/apps/Backend/src/services/typeAgentRunner.ts index 8b6755f8..b5ff64b1 100644 --- a/apps/Backend/src/services/typeAgentRunner.ts +++ b/apps/Backend/src/services/typeAgentRunner.ts @@ -4,13 +4,12 @@ import { confirmTextTyped, cropAroundPoint, cropToRegion, - detectColumnBoundaries, detectRowHeightRatio, diffBoundingBox, - locateAllOnScreenshot, locateOnScreenshot, WindowBounds, } from "./visionLocate"; +import { locateAllViaOcr } from "./ocrLocate"; import { backupTypeAgentScreenshot, logTypeAgentStep } from "../utils/screenshotBackup"; export type StepStatus = "running" | "done" | "error"; @@ -196,19 +195,20 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ const { region } = await captureWindowScreenshot(ctx); backupTypeAgentScreenshot(ctx.runId, "patient_row_locate", region); - const headerMatches = await locateAllOnScreenshot(ctx.userId, region, "PatNum"); + const headerMatches = await locateAllViaOcr(region, "PatNum"); logTypeAgentStep(ctx.runId, { event: "locate_all", label: "PatNum header", matches: toScreenPoints(ctx, headerMatches) }); if (headerMatches.length === 0) throw new Error('Could not find the "PatNum" column header'); const header = headerMatches.reduce((a, b) => (a.y < b.y ? a : b)); - // Locate the target patient's first and last name directly, rather than computing the row's - // y arithmetically (headerY + rowHeight * rowIndex) — a previous version did that and the - // click drifted further off-row the more rows down the target was, since any small error in - // the estimated row height got multiplied by the row index. Two real, independently-detected - // text positions and a same-row pairing between them is grounded in what's actually on - // screen instead of compounding an estimate. - const lastNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientLastName); - const firstNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientFirstName); + // Locate the target patient's first and last name directly via OCR (real pixel bounding + // boxes from Google Vision), rather than computing the row's y arithmetically (headerY + + // rowHeight * rowIndex) — a previous version did that and the click drifted further off-row + // the more rows down the target was. An earlier AI-vision version of this step (asking + // Claude to estimate each word's top/bottom edge) also had a small but consistent upward + // bias since it was an LLM's visual estimate, not a measurement. OCR bounding-box centers + // are exact, so the same-row pairing below needs no bias correction. + const lastNameMatches = await locateAllViaOcr(region, ctx.patientLastName); + const firstNameMatches = await locateAllViaOcr(region, ctx.patientFirstName); logTypeAgentStep(ctx.runId, { event: "locate_all", label: "last name", matches: toScreenPoints(ctx, lastNameMatches) }); logTypeAgentStep(ctx.runId, { event: "locate_all", label: "first name", matches: toScreenPoints(ctx, firstNameMatches) }); if (lastNameMatches.length === 0) throw new Error(`Could not find "${ctx.patientLastName}" in the patient list`); @@ -268,49 +268,37 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ // clickable single-click procedures list (middle), and the already-added procedures grid // plus comm log (right). "Exam" appears both as a clickable item in the middle list AND // as an already-added row in the right grid — telling them apart by y-position alone isn't - // reliable since both can land at a similar height. Finding the actual column dividers and - // keeping only the "Exam" match inside the middle column resolves this structurally. - const dividers = await detectColumnBoundaries(ctx.userId, region); - logTypeAgentStep(ctx.runId, { event: "column_boundaries", dividers: dividers.map((x) => toScreenPoint(ctx, { x, y: 0 }).x) }); - if (dividers.length !== 2) { - throw new Error(`Expected 2 column dividers (3 columns), found ${dividers.length}`); - } - const [middleLeft, middleRight] = dividers as [number, number]; - - const examMatches = await locateAllOnScreenshot(ctx.userId, region, "Exam"); - const singleClickMatches = await locateAllOnScreenshot(ctx.userId, region, "Single click"); + // reliable since both can land at a similar height. Column-divider detection (asking AI to + // find the panel-separator lines) used to disambiguate this, but that's a visual-layout + // judgment call prone to over/under-counting dividers (seen in practice: 3 found instead of + // 2, hard-failing the whole step). "Single click" is a hint label that only ever appears + // once, directly above the correct clickable list — so the "Exam" occurrence nearest it in + // x is structurally the right one, without needing to know where the panel boundaries are. + const examMatches = await locateAllViaOcr(region, "Exam"); + const singleClickMatches = await locateAllViaOcr(region, "Single click"); logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Exam", matches: toScreenPoints(ctx, examMatches) }); logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Single click", matches: toScreenPoints(ctx, singleClickMatches) }); if (examMatches.length === 0) throw new Error('Could not find "Exam" in the Edit Appointment window'); if (singleClickMatches.length === 0) throw new Error('Could not find "Single click" in the Edit Appointment window'); - const inMiddleColumn = examMatches.filter((m) => m.x > middleLeft && m.x < middleRight); - if (inMiddleColumn.length === 0) { - throw new Error('Found "Exam", but none were in the middle column of the window'); - } - // "Upper" — the topmost match in that column, in case more than one qualifies. - const examLoc = inMiddleColumn.reduce((a, b) => (a.y < b.y ? a : b)); + // Topmost "Single click" occurrence, in case OCR ever reports more than one. + const singleClick = singleClickMatches.reduce((a, b) => (a.y < b.y ? a : b)); - // "Single click" is the hint text right above this list — same row-height/grid-average - // technique used for the patient list, anchored on that text instead of a column header, - // so the click lands toward the middle of the Exam row rather than its top edge. - const singleClickY = Math.min(...singleClickMatches.map((m) => m.y)); - const rowHeightRatio = await detectRowHeightRatio(ctx.userId, region); - const rowHeightPx = Math.round(rowHeightRatio * (ctx.windowBounds?.height ?? 0)); - if (rowHeightPx <= 0) throw new Error(`AI reported an invalid row height ratio: ${rowHeightRatio}`); - const rowIndex = Math.max(1, Math.round((examLoc.y - singleClickY) / rowHeightPx)); - const gridY = singleClickY + rowHeightPx * rowIndex; - const finalY = Math.round((examLoc.y + gridY) / 2); + // Nearest "Exam" match to that x — the one in the unrelated right-side grid sits in a + // different column, far enough in x not to compete with the real target. + const examLoc = examMatches.reduce((a, b) => + Math.abs(a.x - singleClick.x) <= Math.abs(b.x - singleClick.x) ? a : b + ); - const screen = toScreenPoint(ctx, { x: examLoc.x, y: finalY }); + // OCR's bounding-box center is exact, so the click lands directly on it — no row-height + // grid estimate needed to nudge toward a row's middle. + const screen = toScreenPoint(ctx, examLoc); logTypeAgentStep(ctx.runId, { event: "exam_match", pixelX: screen.x, - directY: toScreenPoint(ctx, { x: 0, y: examLoc.y }).y, - gridY: toScreenPoint(ctx, { x: 0, y: gridY }).y, finalY: screen.y, - rowHeightPx, + singleClickMatch: toScreenPoint(ctx, singleClick), }); await debugClickAt(ctx, screen.x, screen.y, "exam"); @@ -322,16 +310,18 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [ execute: async (ctx) => { const { region } = await captureWindowScreenshot(ctx); backupTypeAgentScreenshot(ctx.runId, "save_locate", region); - const loc = await locateOnScreenshot( - ctx.userId, - region, - "the Save button in the Edit Appointment window — it's in the LOWER part of the " + - "RIGHTMOST of the window's three big columns (the one with the already-added " + - "procedures grid and comm log above it), below the other action buttons in that column " + - '(e.g. "Delete", "To Task List", "Audit Trail")' - ); - if (!loc) throw new Error("Could not find the Save button in the Edit Appointment window"); - const screen = toScreenPoint(ctx, loc); + + const saveMatches = await locateAllViaOcr(region, "Save"); + logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Save", matches: toScreenPoints(ctx, saveMatches) }); + if (saveMatches.length === 0) throw new Error("Could not find the Save button in the Edit Appointment window"); + + // "Save" only ever appears once as a button label in this window, but if OCR ever reports + // more than one, the real one is in the LOWER part of the RIGHTMOST column (below the other + // action buttons there, e.g. "Delete", "To Task List", "Audit Trail") — so the bottom-right- + // most occurrence is the correct disambiguation, same reasoning the old AI-vision prompt used. + const saveLoc = saveMatches.reduce((a, b) => (a.x + a.y >= b.x + b.y ? a : b)); + + const screen = toScreenPoint(ctx, saveLoc); logTypeAgentStep(ctx.runId, { event: "locate", label: "Save", pixelX: screen.x, pixelY: screen.y }); await debugClickAt(ctx, screen.x, screen.y, "save"); diff --git a/apps/PaymentOCRService/main.py b/apps/PaymentOCRService/main.py index 4281d2ee..21102d07 100755 --- a/apps/PaymentOCRService/main.py +++ b/apps/PaymentOCRService/main.py @@ -5,11 +5,13 @@ from typing import List, Optional import io import os import asyncio +import tempfile import uvicorn from dotenv import load_dotenv -load_dotenv() +load_dotenv() from complete_pipeline_adapter import process_images_to_rows,rows_to_csv_bytes +from complete_pipeline import extract_words_and_text from pdf_extractor import extract_ra_pdf app = FastAPI( @@ -98,6 +100,41 @@ async def extract_json(files: List[UploadFile] = File(...)): async with lock: active_jobs -= 1 +@app.post("/extract/words") +async def extract_words(file: UploadFile = File(...)): + """Raw OCR only — no deskew/line-grouping/payment-domain logic, just the + flat word list with pixel bounding boxes. Used by the Windows Type Agent + to locate exact click targets (e.g. a patient name) instead of relying + on an AI-vision estimate of where text is.""" + _validate_files([file]) + + async with lock: + global waiting_jobs + waiting_jobs += 1 + + async with semaphore: + async with lock: + waiting_jobs -= 1 + global active_jobs + active_jobs += 1 + + tmp_path = None + try: + suffix = os.path.splitext(file.filename or "")[1] or ".png" + blob = await file.read() + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(blob) + tmp_path = tmp.name + words, _ = extract_words_and_text(tmp_path) + return JSONResponse(content={"words": words}) + except Exception as e: + raise HTTPException(status_code=500, detail=f"OCR error: {e}") + finally: + if tmp_path and os.path.exists(tmp_path): + os.remove(tmp_path) + async with lock: + active_jobs -= 1 + @app.post("/extract/csvtext", response_class=PlainTextResponse) async def extract_csvtext(files: List[UploadFile] = File(...)): _validate_files(files)