fix: use Google Vision OCR instead of AI-vision estimates for Windows Type Agent clicks

Claude vision's self-reported text coordinates had a small but consistent bias,
causing the patient-row double-click to land above the name and the Exam-column
divider count to occasionally miscount and hard-fail. Swap patient-row, Exam,
and Save button targeting to exact OCR bounding boxes (Google Vision via the
existing PaymentOCRService) instead of AI-estimated ratios.
This commit is contained in:
2026-07-30 23:14:02 -04:00
parent 3f83794ac5
commit 5cdd3452f2
3 changed files with 141 additions and 53 deletions
+38 -1
View File
@@ -5,11 +5,13 @@ from typing import List, Optional
import io
import os
import asyncio
import tempfile
import uvicorn
from dotenv import load_dotenv
load_dotenv()
load_dotenv()
from complete_pipeline_adapter import process_images_to_rows,rows_to_csv_bytes
from complete_pipeline import extract_words_and_text
from pdf_extractor import extract_ra_pdf
app = FastAPI(
@@ -98,6 +100,41 @@ async def extract_json(files: List[UploadFile] = File(...)):
async with lock:
active_jobs -= 1
@app.post("/extract/words")
async def extract_words(file: UploadFile = File(...)):
"""Raw OCR only — no deskew/line-grouping/payment-domain logic, just the
flat word list with pixel bounding boxes. Used by the Windows Type Agent
to locate exact click targets (e.g. a patient name) instead of relying
on an AI-vision estimate of where text is."""
_validate_files([file])
async with lock:
global waiting_jobs
waiting_jobs += 1
async with semaphore:
async with lock:
waiting_jobs -= 1
global active_jobs
active_jobs += 1
tmp_path = None
try:
suffix = os.path.splitext(file.filename or "")[1] or ".png"
blob = await file.read()
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(blob)
tmp_path = tmp.name
words, _ = extract_words_and_text(tmp_path)
return JSONResponse(content={"words": words})
except Exception as e:
raise HTTPException(status_code=500, detail=f"OCR error: {e}")
finally:
if tmp_path and os.path.exists(tmp_path):
os.remove(tmp_path)
async with lock:
active_jobs -= 1
@app.post("/extract/csvtext", response_class=PlainTextResponse)
async def extract_csvtext(files: List[UploadFile] = File(...)):
_validate_files(files)