import sharp from "sharp"; import { storage } from "../storage"; import { resolveAiProvider, getLlm } from "../ai/llm-factory"; // Side length (px) of the debug crop saved around a click point — big enough to show // surrounding context (e.g. neighboring grid rows) when reviewing what a run actually clicked. const DEBUG_CROP_SIZE = 440; interface ParsedLocation { found: boolean; xRatio?: number; yRatio?: number; } // Parses the model's response into {found, xRatio, yRatio}, tolerating near-miss JSON rather // than only accepting strictly valid JSON. Seen in practice with the old pixel-based format: // {"found": true, "x": 1023, 187} — a dropped key with the value still present positionally. // Since this response should only ever contain two numbers (the coordinates), falling back to // "first two numbers after found" is safe and recovers without burning another AI call. function parseLocationResponse(raw: string): ParsedLocation | null { const match = raw.match(/\{[\s\S]*\}/); if (!match) return null; const block = match[0]; try { return JSON.parse(block) as ParsedLocation; } catch { // fall through to lenient repair below } const foundMatch = block.match(/"found"\s*:\s*(true|false)/i); if (!foundMatch) return null; const found = foundMatch[1]!.toLowerCase() === "true"; if (!found) return { found: false }; const afterFound = block.slice(foundMatch.index! + foundMatch[0].length); const numbers = afterFound.match(/-?\d+(\.\d+)?/g); if (!numbers || numbers.length < 2) return null; return { found: true, xRatio: Number(numbers[0]), yRatio: Number(numbers[1]) }; } // Asks the configured vision AI to find a described UI element in a screenshot and return // its position — the core primitive the Type Agent step runner clicks/types through. // Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info. // // The AI reports position as a 0-1 ratio of image width/height rather than absolute pixels. // Vision models are more reliable at "this is 4/10 of the way across, 3/10 down" than at // naming an exact pixel — asking for pixels directly produced coordinates that were close but // occasionally off by enough to land on a neighboring row or the title bar above it. A ratio // is resolution-independent by construction, so it's converted to real screen pixels here // using the actual screenshot dimensions rather than trusting the model to know the image size. export async function locateOnScreenshot( userId: number, imageBase64: string, goal: string ): Promise<{ x: number; y: number; xRatio: number; yRatio: number } | null> { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { throw new Error("AI is not configured. Add an API key in AI Settings."); } if (activeAi.provider !== "claude") { throw new Error("Vision-guided steps require Claude to be the active AI provider."); } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); const content = [ { type: "text", text: `This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` + "Respond with strict JSON only, no prose, no markdown fences: " + '{"found": true, "xRatio": <0 to 1>, "yRatio": <0 to 1>} if you can locate it, or ' + '{"found": false} if it isn\'t visible in this screenshot. xRatio is how far across the ' + "image the element's center is, as a fraction of the TOTAL image width (0 = left edge, " + "0.5 = horizontal center, 1 = right edge). yRatio is how far down the element's center is, " + "as a fraction of the TOTAL image height (0 = top edge, 1 = bottom edge). Use two decimal " + "places of precision — do not report pixel coordinates.", }, { type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` }, }, ]; // A response that's fully unparseable even after the lenient repair above is worth one // retry (rare model hiccup); anything the repair can salvage is used immediately, no // retry needed. const MAX_ATTEMPTS = 2; let lastError: Error | null = null; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { const response = await llm.invoke([{ role: "user", content }] as any); const raw = String(response.content).trim(); const parsed = parseLocationResponse(raw); if (!parsed) { lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); continue; } if (!parsed.found || typeof parsed.xRatio !== "number" || typeof parsed.yRatio !== "number") { return null; } return { x: Math.round(parsed.xRatio * width), y: Math.round(parsed.yRatio * height), xRatio: parsed.xRatio, yRatio: parsed.yRatio, }; } throw lastError ?? new Error("AI returned an unparseable response"); } // Crops a small square around (x, y) and draws a red crosshair marker at that exact point — // a debug artifact saved before each click so a person can see afterward exactly where the // run was about to click. A real screen capture won't reliably include the OS mouse cursor // (BitBlt-based grabs typically omit it), so this synthetic marker stands in for it. export async function cropAroundPoint(imageBase64: string, x: number, y: number): Promise { const source = sharp(Buffer.from(imageBase64, "base64")); const { width = 0, height = 0 } = await source.metadata(); const cropWidth = Math.min(DEBUG_CROP_SIZE, width); const cropHeight = Math.min(DEBUG_CROP_SIZE, height); const left = Math.round(Math.max(0, Math.min(x - cropWidth / 2, width - cropWidth))); const top = Math.round(Math.max(0, Math.min(y - cropHeight / 2, height - cropHeight))); const markerX = x - left; const markerY = y - top; const crosshair = Buffer.from( `` + `` + `` + `` + `` ); const cropped = await source .extract({ left, top, width: cropWidth, height: cropHeight }) .composite([{ input: crosshair }]) .png() .toBuffer(); return cropped.toString("base64"); } interface RawMatch { xRatio: number; yTopRatio: number; yBottomRatio: number; } // Parses {"matches": [{"xRatio":.., "yTopRatio":.., "yBottomRatio":..}, ...]}, tolerating // near-miss JSON the same way parseLocationResponse does — falls back to regex-extracting every // triple in the response if strict parsing fails, rather than treating a malformed-but-salvageable // response as zero matches. function parseMatchesResponse(raw: string): RawMatch[] | null { const block = raw.match(/\{[\s\S]*\}/)?.[0]; if (!block) return null; try { const parsed = JSON.parse(block); if (Array.isArray(parsed.matches)) return parsed.matches; } catch { // fall through to lenient repair below } const triples: RawMatch[] = []; const tripleRegex = /"xRatio"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"yTopRatio"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"yBottomRatio"\s*:\s*(-?\d+(?:\.\d+)?)/g; let match: RegExpExecArray | null; while ((match = tripleRegex.exec(block))) { triples.push({ xRatio: Number(match[1]), yTopRatio: Number(match[2]), yBottomRatio: Number(match[3]) }); } return triples.length > 0 ? triples : null; } // Like locateOnScreenshot, but finds every occurrence of an exact piece of text rather than // one best guess for a described element. Used to disambiguate by position instead of by // asking the AI to judge which occurrence is "the right one" — e.g. a last name shows up once // in the search box and once per matching patient row, so the caller pairs it against another // text's occurrences (a first name) that share the same row instead of relying on a single // fuzzy vision judgment call. // // Reports each match's TOP and BOTTOM pixel edge rather than asking the model to self-estimate a // "center" — a center is an abstract judgment call, whereas the top/bottom of a glyph is a // concrete, visible thing to point at. The vertical midpoint used for x/y below is then computed // here in code from those two real edges, not trusted as a direct AI estimate. export async function locateAllOnScreenshot( userId: number, imageBase64: string, text: string ): Promise<{ x: number; y: number }[]> { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { throw new Error("AI is not configured. Add an API key in AI Settings."); } if (activeAi.provider !== "claude") { throw new Error("Vision-guided steps require Claude to be the active AI provider."); } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); const content = [ { type: "text", text: `This is a screenshot of a Windows desktop application. Find EVERY occurrence of the text ` + `"${text}" visible anywhere in this screenshot (case-insensitive) — there may be zero, one, ` + "or several. Respond with strict JSON only, no prose, no markdown fences: " + '{"matches": [{"xRatio": <0 to 1>, "yTopRatio": <0 to 1>, "yBottomRatio": <0 to 1>}, ...]} — ' + "one entry per occurrence. xRatio is how far across the image the occurrence's horizontal " + "center is, as a fraction of the TOTAL image width. yTopRatio is where the TOP edge of that " + "occurrence's text (the top of its tallest letters) is, and yBottomRatio is where its BOTTOM " + "edge (the bottom of its lowest letters, including descenders) is — both as a fraction of " + "the TOTAL image height. Use an empty array if there are no occurrences.", }, { type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` }, }, ]; const MAX_ATTEMPTS = 2; let lastError: Error | null = null; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { const response = await llm.invoke([{ role: "user", content }] as any); const raw = String(response.content).trim(); const matches = parseMatchesResponse(raw); if (!matches) { lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); continue; } return matches.map((m) => ({ x: Math.round(m.xRatio * width), y: Math.round(((m.yTopRatio + m.yBottomRatio) / 2) * height), })); } throw lastError ?? new Error("AI returned an unparseable response"); } // Asks the AI to measure the height of one row in a results-grid list, as a ratio of the // image's total height — used to build a grid of valid row-center y-positions so a click can be // snapped toward the middle of whichever row it's closest to, rather than trusting a single // direct locate call's y-coordinate on its own (which tends to land near a row's top edge). export async function detectRowHeightRatio(userId: number, imageBase64: string): Promise { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { throw new Error("AI is not configured. Add an API key in AI Settings."); } if (activeAi.provider !== "claude") { throw new Error("Vision-guided steps require Claude to be the active AI provider."); } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); const content = [ { type: "text", text: "This is a screenshot containing a results-grid list of rows (e.g. a patient list) with a " + "header row followed by data rows, all data rows the same height. IGNORE the header row — " + "it is taller than a data row and would skew the measurement. Look at ALL the visible data " + "rows (not just one) and measure their AVERAGE height. Respond with strict JSON only, no " + 'prose, no markdown fences: {"rowHeightRatio": <0 to 1>} — the average data row height as ' + "a fraction of the TOTAL image height. Use three decimal places of precision.", }, { type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` }, }, ]; const response = await llm.invoke([{ role: "user", content }] as any); const raw = String(response.content).trim(); const match = raw.match(/"rowHeightRatio"\s*:\s*(-?\d+(?:\.\d+)?)/); if (!match) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); return Number(match[1]); } // Asks the AI to find the x-positions of the MAJOR vertical dividers that split a window into // its big panels/columns (e.g. a "Patient Info" panel, a middle fields+list panel, a right-side // grid panel) — as opposed to minor lines inside a single table's own columns. Used to tell // apart same-text matches that live in different panels (e.g. "Exam" appearing both in a // clickable procedures list and, separately, in an already-added-procedures grid) by which // column they actually fall in, rather than guessing from y-position alone. export async function detectColumnBoundaries(userId: number, imageBase64: string): Promise { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { throw new Error("AI is not configured. Add an API key in AI Settings."); } if (activeAi.provider !== "claude") { throw new Error("Vision-guided steps require Claude to be the active AI provider."); } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); const { width = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata(); const content = [ { type: "text", text: "This screenshot shows a Windows dialog laid out in a small number of large vertical " + "sections (columns) — e.g. a distinct panel on the left, a middle section with fields " + 'and/or a list, and another panel or grid on the right. Find the x-position of each MAJOR ' + "vertical divider between these big sections (a visible line, border, or clear gap — NOT " + "the minor column lines inside a single table). Respond with strict JSON only, no prose, " + 'no markdown fences: {"dividerXRatios": [<0 to 1>, ...]} — one entry per major divider, ' + "sorted left to right, as a fraction of the TOTAL image width. Use two decimal places of " + "precision.", }, { type: "image_url", image_url: { url: `data:image/png;base64,${imageBase64}` }, }, ]; const response = await llm.invoke([{ role: "user", content }] as any); const raw = String(response.content).trim(); const block = raw.match(/\{[\s\S]*\}/)?.[0]; if (!block) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); let ratios: number[]; try { const parsed = JSON.parse(block); if (!Array.isArray(parsed.dividerXRatios)) throw new Error("not an array"); ratios = parsed.dividerXRatios; } catch { const arrayMatch = block.match(/"dividerXRatios"\s*:\s*\[([^\]]*)\]/); if (!arrayMatch) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`); ratios = arrayMatch[1]! .split(",") .map((s) => Number(s.trim())) .filter((n) => !Number.isNaN(n)); } return ratios.map((r) => Math.round(r * width)).sort((a, b) => a - b); } export interface WindowBounds { left: number; top: number; width: number; height: number; } // Compares two full screenshots and returns the bounding box of whatever changed // significantly between them — used to find a newly-opened dialog's exact position/size with // pixel math instead of an AI call, so later steps can crop every screenshot down to just that // window and never even show the AI anything outside it (the earlier bug where a name prefix // happened to also match unrelated text elsewhere on screen, e.g. "All Received" in a side // panel, is structurally impossible once matching only ever sees the cropped window). // // Rather than a raw min/max over any differing pixel, this counts changed pixels per row and // per column and only counts a row/column as part of the new window once a meaningful fraction // of it changed — a real window opening changes a large contiguous block, whereas incidental // noise elsewhere (a blinking clock, a flickering tray icon) only touches a handful of pixels // and would otherwise blow the bounding box out to include it. export async function diffBoundingBox( beforeBase64: string, afterBase64: string ): Promise { const [before, after] = await Promise.all([ sharp(Buffer.from(beforeBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }), sharp(Buffer.from(afterBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }), ]); const { data: b, info } = before; const { data: a, info: infoAfter } = after; if (info.width !== infoAfter.width || info.height !== infoAfter.height) return null; const { width, height, channels } = info; const PIXEL_DIFF_THRESHOLD = 40; // sum of |dR| + |dG| + |dB| to count a pixel as "changed" const rowCounts = new Uint32Array(height); const colCounts = new Uint32Array(width); for (let y = 0; y < height; y++) { const rowBase = y * width * channels; for (let x = 0; x < width; x++) { const idx = rowBase + x * channels; const diff = Math.abs(a[idx]! - b[idx]!) + Math.abs(a[idx + 1]! - b[idx + 1]!) + Math.abs(a[idx + 2]! - b[idx + 2]!); if (diff > PIXEL_DIFF_THRESHOLD) { rowCounts[y] = (rowCounts[y] ?? 0) + 1; colCounts[x] = (colCounts[x] ?? 0) + 1; } } } const rowThreshold = width * 0.05; const colThreshold = height * 0.05; let top = -1; let bottom = -1; for (let y = 0; y < height; y++) { if (rowCounts[y]! > rowThreshold) { if (top === -1) top = y; bottom = y; } } let left = -1; let right = -1; for (let x = 0; x < width; x++) { if (colCounts[x]! > colThreshold) { if (left === -1) left = x; right = x; } } if (top === -1 || left === -1) return null; return { left, top, width: right - left + 1, height: bottom - top + 1 }; } // Plain crop, no marker — used to restrict a screenshot to a previously-detected window's // bounds before handing it to any locate call. export async function cropToRegion(imageBase64: string, region: WindowBounds): Promise { const cropped = await sharp(Buffer.from(imageBase64, "base64")) .extract({ left: region.left, top: region.top, width: region.width, height: region.height }) .png() .toBuffer(); return cropped.toString("base64"); } // Sanity check for steps that type without first vision-clicking into a field (relying on // whatever already has focus) — confirms the text actually landed somewhere visible on // screen before moving on, catching the case where focus wasn't where it was assumed to be. export async function confirmTextTyped( userId: number, fullScreenshotBase64: string, expectedText: string ): Promise { const aiSettings = await storage.getAiSettings(userId); const activeAi = resolveAiProvider(aiSettings ?? {}); if (!activeAi) { throw new Error("AI is not configured. Add an API key in AI Settings."); } if (activeAi.provider !== "claude") { throw new Error("Vision-guided steps require Claude to be the active AI provider."); } const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model); const content = [ { type: "text", text: `This is a screenshot of a Windows desktop application. Is the text "${expectedText}" ` + "visible typed into some text input field on screen? Respond with strict JSON only, no " + 'prose, no markdown fences: {"confirmed": true} if that text is visible in a field, or ' + '{"confirmed": false} if it is not visible anywhere.', }, { type: "image_url", image_url: { url: `data:image/png;base64,${fullScreenshotBase64}` }, }, ]; const response = await llm.invoke([{ role: "user", content }] as any); const raw = String(response.content).trim(); const match = raw.match(/"confirmed"\s*:\s*(true|false)/i); return match?.[1]?.toLowerCase() === "true"; }