fix: use Google Vision OCR instead of AI-vision estimates for Windows Type Agent clicks

Claude vision's self-reported text coordinates had a small but consistent bias,
causing the patient-row double-click to land above the name and the Exam-column
divider count to occasionally miscount and hard-fail. Swap patient-row, Exam,
and Save button targeting to exact OCR bounding boxes (Google Vision via the
existing PaymentOCRService) instead of AI-estimated ratios.

apps/Backend/src/services/ocrLocate.ts

@@ -0,0 +1,61 @@
+import axios from "axios";
+import FormData from "form-data";
+
+interface OcrWord {
+  text: string;
+  left: number;
+  top: number;
+  w: number;
+  h: number;
+  cx: number;
+  cy: number;
+}
+
+// Finds every occurrence of exact text in a screenshot using real OCR (Google Vision via
+// PaymentOCRService, already running for payment-document extraction) instead of asking an AI
+// to visually estimate coordinates. Google Vision's word bounding boxes give an exact pixel
+// center, so unlike locateAllOnScreenshot's AI-estimated top/bottom edges, there's no systematic
+// bias to correct for.
+//
+// `text` may be multiple words (e.g. "Single click") — Google Vision reports one bounding box
+// per individual word, not per phrase, so a multi-word target is matched as a run of consecutive
+// words in the OCR result (which preserves reading order) and their boxes are merged into one.
+export async function locateAllViaOcr(imageBase64: string, text: string): Promise<{ x: number; y: number }[]> {
+  const form = new FormData();
+  form.append("file", Buffer.from(imageBase64, "base64"), {
+    filename: "screenshot.png",
+    contentType: "image/png",
+  });
+
+  let words: OcrWord[];
+  try {
+    const resp = await axios.post<{ words: OcrWord[] }>("http://localhost:5003/extract/words", form, {
+      headers: form.getHeaders(),
+      maxBodyLength: Infinity,
+      maxContentLength: Infinity,
+      timeout: 30000,
+    });
+    words = resp.data?.words ?? [];
+  } catch (err: any) {
+    const status = err?.response?.status;
+    const detail = err?.response?.data?.detail || err?.message || "Unknown error";
+    throw new Error(`OCR request failed${status ? ` (${status})` : ""}: ${detail}`);
+  }
+
+  const targetTokens = text.trim().toLowerCase().split(/\s+/).filter(Boolean);
+  if (targetTokens.length === 0) return [];
+
+  const matches: { x: number; y: number }[] = [];
+  for (let i = 0; i <= words.length - targetTokens.length; i++) {
+    const span = words.slice(i, i + targetTokens.length);
+    const matchesHere = span.every((w, j) => w.text.trim().toLowerCase() === targetTokens[j]);
+    if (!matchesHere) continue;
+
+    const left = Math.min(...span.map((w) => w.left));
+    const top = Math.min(...span.map((w) => w.top));
+    const right = Math.max(...span.map((w) => w.left + w.w));
+    const bottom = Math.max(...span.map((w) => w.top + w.h));
+    matches.push({ x: Math.round((left + right) / 2), y: Math.round((top + bottom) / 2) });
+  }
+  return matches;
+}

apps/Backend/src/services/typeAgentRunner.ts

@@ -4,13 +4,12 @@ import {
   confirmTextTyped,
   cropAroundPoint,
   cropToRegion,
-  detectColumnBoundaries,
   detectRowHeightRatio,
   diffBoundingBox,
-  locateAllOnScreenshot,
   locateOnScreenshot,
   WindowBounds,
 } from "./visionLocate";
+import { locateAllViaOcr } from "./ocrLocate";
 import { backupTypeAgentScreenshot, logTypeAgentStep } from "../utils/screenshotBackup";
 
 export type StepStatus = "running" | "done" | "error";
@@ -196,19 +195,20 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
       const { region } = await captureWindowScreenshot(ctx);
       backupTypeAgentScreenshot(ctx.runId, "patient_row_locate", region);
 
-      const headerMatches = await locateAllOnScreenshot(ctx.userId, region, "PatNum");
+      const headerMatches = await locateAllViaOcr(region, "PatNum");
       logTypeAgentStep(ctx.runId, { event: "locate_all", label: "PatNum header", matches: toScreenPoints(ctx, headerMatches) });
       if (headerMatches.length === 0) throw new Error('Could not find the "PatNum" column header');
       const header = headerMatches.reduce((a, b) => (a.y < b.y ? a : b));
 
-      // Locate the target patient's first and last name directly, rather than computing the row's
+      // Locate the target patient's first and last name directly via OCR (real pixel bounding
-      // y arithmetically (headerY + rowHeight * rowIndex) — a previous version did that and the
+      // boxes from Google Vision), rather than computing the row's y arithmetically (headerY +
-      // click drifted further off-row the more rows down the target was, since any small error in
+      // rowHeight * rowIndex) — a previous version did that and the click drifted further off-row
-      // the estimated row height got multiplied by the row index. Two real, independently-detected
+      // the more rows down the target was. An earlier AI-vision version of this step (asking
-      // text positions and a same-row pairing between them is grounded in what's actually on
+      // Claude to estimate each word's top/bottom edge) also had a small but consistent upward
-      // screen instead of compounding an estimate.
+      // bias since it was an LLM's visual estimate, not a measurement. OCR bounding-box centers
-      const lastNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientLastName);
+      // are exact, so the same-row pairing below needs no bias correction.
-      const firstNameMatches = await locateAllOnScreenshot(ctx.userId, region, ctx.patientFirstName);
+      const lastNameMatches = await locateAllViaOcr(region, ctx.patientLastName);
+      const firstNameMatches = await locateAllViaOcr(region, ctx.patientFirstName);
       logTypeAgentStep(ctx.runId, { event: "locate_all", label: "last name", matches: toScreenPoints(ctx, lastNameMatches) });
       logTypeAgentStep(ctx.runId, { event: "locate_all", label: "first name", matches: toScreenPoints(ctx, firstNameMatches) });
       if (lastNameMatches.length === 0) throw new Error(`Could not find "${ctx.patientLastName}" in the patient list`);
@@ -268,49 +268,37 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
       // clickable single-click procedures list (middle), and the already-added procedures grid
       // plus comm log (right). "Exam" appears both as a clickable item in the middle list AND
       // as an already-added row in the right grid — telling them apart by y-position alone isn't
-      // reliable since both can land at a similar height. Finding the actual column dividers and
+      // reliable since both can land at a similar height. Column-divider detection (asking AI to
-      // keeping only the "Exam" match inside the middle column resolves this structurally.
+      // find the panel-separator lines) used to disambiguate this, but that's a visual-layout
-      const dividers = await detectColumnBoundaries(ctx.userId, region);
+      // judgment call prone to over/under-counting dividers (seen in practice: 3 found instead of
-      logTypeAgentStep(ctx.runId, { event: "column_boundaries", dividers: dividers.map((x) => toScreenPoint(ctx, { x, y: 0 }).x) });
+      // 2, hard-failing the whole step). "Single click" is a hint label that only ever appears
-      if (dividers.length !== 2) {
+      // once, directly above the correct clickable list — so the "Exam" occurrence nearest it in
-        throw new Error(`Expected 2 column dividers (3 columns), found ${dividers.length}`);
+      // x is structurally the right one, without needing to know where the panel boundaries are.
-      }
+      const examMatches = await locateAllViaOcr(region, "Exam");
-      const [middleLeft, middleRight] = dividers as [number, number];
+      const singleClickMatches = await locateAllViaOcr(region, "Single click");
-
-      const examMatches = await locateAllOnScreenshot(ctx.userId, region, "Exam");
-      const singleClickMatches = await locateAllOnScreenshot(ctx.userId, region, "Single click");
       logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Exam", matches: toScreenPoints(ctx, examMatches) });
       logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Single click", matches: toScreenPoints(ctx, singleClickMatches) });
 
       if (examMatches.length === 0) throw new Error('Could not find "Exam" in the Edit Appointment window');
       if (singleClickMatches.length === 0) throw new Error('Could not find "Single click" in the Edit Appointment window');
 
-      const inMiddleColumn = examMatches.filter((m) => m.x > middleLeft && m.x < middleRight);
+      // Topmost "Single click" occurrence, in case OCR ever reports more than one.
-      if (inMiddleColumn.length === 0) {
+      const singleClick = singleClickMatches.reduce((a, b) => (a.y < b.y ? a : b));
-        throw new Error('Found "Exam", but none were in the middle column of the window');
-      }
-      // "Upper" — the topmost match in that column, in case more than one qualifies.
-      const examLoc = inMiddleColumn.reduce((a, b) => (a.y < b.y ? a : b));
 
-      // "Single click" is the hint text right above this list — same row-height/grid-average
+      // Nearest "Exam" match to that x — the one in the unrelated right-side grid sits in a
-      // technique used for the patient list, anchored on that text instead of a column header,
+      // different column, far enough in x not to compete with the real target.
-      // so the click lands toward the middle of the Exam row rather than its top edge.
+      const examLoc = examMatches.reduce((a, b) =>
-      const singleClickY = Math.min(...singleClickMatches.map((m) => m.y));
+        Math.abs(a.x - singleClick.x) <= Math.abs(b.x - singleClick.x) ? a : b
-      const rowHeightRatio = await detectRowHeightRatio(ctx.userId, region);
+      );
-      const rowHeightPx = Math.round(rowHeightRatio * (ctx.windowBounds?.height ?? 0));
-      if (rowHeightPx <= 0) throw new Error(`AI reported an invalid row height ratio: ${rowHeightRatio}`);
-      const rowIndex = Math.max(1, Math.round((examLoc.y - singleClickY) / rowHeightPx));
-      const gridY = singleClickY + rowHeightPx * rowIndex;
-      const finalY = Math.round((examLoc.y + gridY) / 2);
 
-      const screen = toScreenPoint(ctx, { x: examLoc.x, y: finalY });
+      // OCR's bounding-box center is exact, so the click lands directly on it — no row-height
+      // grid estimate needed to nudge toward a row's middle.
+      const screen = toScreenPoint(ctx, examLoc);
       logTypeAgentStep(ctx.runId, {
         event: "exam_match",
         pixelX: screen.x,
-        directY: toScreenPoint(ctx, { x: 0, y: examLoc.y }).y,
-        gridY: toScreenPoint(ctx, { x: 0, y: gridY }).y,
         finalY: screen.y,
-        rowHeightPx,
+        singleClickMatch: toScreenPoint(ctx, singleClick),
       });
 
       await debugClickAt(ctx, screen.x, screen.y, "exam");
@@ -322,16 +310,18 @@ const OPEN_DENTAL_EXISTING_PATIENT: RunStep[] = [
     execute: async (ctx) => {
       const { region } = await captureWindowScreenshot(ctx);
       backupTypeAgentScreenshot(ctx.runId, "save_locate", region);
-      const loc = await locateOnScreenshot(
+
-        ctx.userId,
+      const saveMatches = await locateAllViaOcr(region, "Save");
-        region,
+      logTypeAgentStep(ctx.runId, { event: "locate_all", label: "Save", matches: toScreenPoints(ctx, saveMatches) });
-        "the Save button in the Edit Appointment window — it's in the LOWER part of the " +
+      if (saveMatches.length === 0) throw new Error("Could not find the Save button in the Edit Appointment window");
-          "RIGHTMOST of the window's three big columns (the one with the already-added " +
+
-          "procedures grid and comm log above it), below the other action buttons in that column " +
+      // "Save" only ever appears once as a button label in this window, but if OCR ever reports
-          '(e.g. "Delete", "To Task List", "Audit Trail")'
+      // more than one, the real one is in the LOWER part of the RIGHTMOST column (below the other
-      );
+      // action buttons there, e.g. "Delete", "To Task List", "Audit Trail") — so the bottom-right-
-      if (!loc) throw new Error("Could not find the Save button in the Edit Appointment window");
+      // most occurrence is the correct disambiguation, same reasoning the old AI-vision prompt used.
-      const screen = toScreenPoint(ctx, loc);
+      const saveLoc = saveMatches.reduce((a, b) => (a.x + a.y >= b.x + b.y ? a : b));
+
+      const screen = toScreenPoint(ctx, saveLoc);
       logTypeAgentStep(ctx.runId, { event: "locate", label: "Save", pixelX: screen.x, pixelY: screen.y });
 
       await debugClickAt(ctx, screen.x, screen.y, "save");

apps/PaymentOCRService/main.py

@@ -5,11 +5,13 @@ from typing import List, Optional
 import io
 import os
 import asyncio
+import tempfile
 import uvicorn
 from dotenv import load_dotenv
 load_dotenv()
 
 from complete_pipeline_adapter import process_images_to_rows,rows_to_csv_bytes
+from complete_pipeline import extract_words_and_text
 from pdf_extractor import extract_ra_pdf
 
 app = FastAPI(
@@ -98,6 +100,41 @@ async def extract_json(files: List[UploadFile] = File(...)):
             async with lock:
                 active_jobs -= 1
 
+@app.post("/extract/words")
+async def extract_words(file: UploadFile = File(...)):
+    """Raw OCR only — no deskew/line-grouping/payment-domain logic, just the
+    flat word list with pixel bounding boxes. Used by the Windows Type Agent
+    to locate exact click targets (e.g. a patient name) instead of relying
+    on an AI-vision estimate of where text is."""
+    _validate_files([file])
+
+    async with lock:
+        global waiting_jobs
+        waiting_jobs += 1
+
+    async with semaphore:
+        async with lock:
+            waiting_jobs -= 1
+            global active_jobs
+            active_jobs += 1
+
+        tmp_path = None
+        try:
+            suffix = os.path.splitext(file.filename or "")[1] or ".png"
+            blob = await file.read()
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                tmp.write(blob)
+                tmp_path = tmp.name
+            words, _ = extract_words_and_text(tmp_path)
+            return JSONResponse(content={"words": words})
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"OCR error: {e}")
+        finally:
+            if tmp_path and os.path.exists(tmp_path):
+                os.remove(tmp_path)
+            async with lock:
+                active_jobs -= 1
+
 @app.post("/extract/csvtext", response_class=PlainTextResponse)
 async def extract_csvtext(files: List[UploadFile] = File(...)):
     _validate_files(files)