Gitead
65dc795eb2
Row selection previously computed the target row's y arithmetically (header.y + rowHeight * rowIndex), which drifted further off-row the deeper the target patient was in the filtered list, and an AI "onTarget" verification step added to catch drift instead confirmed a click point that was visibly in blank space below the grid. locateAllOnScreenshot now reports each text match's top/bottom edges instead of a self-estimated center, and the patient row step locates the target's first and last name directly, pairing whichever occurrences land on the same row instead of trusting arithmetic or a qualitative AI judgment call. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
472 lines
20 KiB
TypeScript
472 lines
20 KiB
TypeScript
import sharp from "sharp";
|
|
import { storage } from "../storage";
|
|
import { resolveAiProvider, getLlm } from "../ai/llm-factory";
|
|
|
|
// Side length (px) of the debug crop saved around a click point — big enough to show
|
|
// surrounding context (e.g. neighboring grid rows) when reviewing what a run actually clicked.
|
|
const DEBUG_CROP_SIZE = 440;
|
|
|
|
interface ParsedLocation {
|
|
found: boolean;
|
|
xRatio?: number;
|
|
yRatio?: number;
|
|
}
|
|
|
|
// Parses the model's response into {found, xRatio, yRatio}, tolerating near-miss JSON rather
|
|
// than only accepting strictly valid JSON. Seen in practice with the old pixel-based format:
|
|
// {"found": true, "x": 1023, 187} — a dropped key with the value still present positionally.
|
|
// Since this response should only ever contain two numbers (the coordinates), falling back to
|
|
// "first two numbers after found" is safe and recovers without burning another AI call.
|
|
function parseLocationResponse(raw: string): ParsedLocation | null {
|
|
const match = raw.match(/\{[\s\S]*\}/);
|
|
if (!match) return null;
|
|
const block = match[0];
|
|
|
|
try {
|
|
return JSON.parse(block) as ParsedLocation;
|
|
} catch {
|
|
// fall through to lenient repair below
|
|
}
|
|
|
|
const foundMatch = block.match(/"found"\s*:\s*(true|false)/i);
|
|
if (!foundMatch) return null;
|
|
const found = foundMatch[1]!.toLowerCase() === "true";
|
|
if (!found) return { found: false };
|
|
|
|
const afterFound = block.slice(foundMatch.index! + foundMatch[0].length);
|
|
const numbers = afterFound.match(/-?\d+(\.\d+)?/g);
|
|
if (!numbers || numbers.length < 2) return null;
|
|
|
|
return { found: true, xRatio: Number(numbers[0]), yRatio: Number(numbers[1]) };
|
|
}
|
|
|
|
// Asks the configured vision AI to find a described UI element in a screenshot and return
|
|
// its position — the core primitive the Type Agent step runner clicks/types through.
|
|
// Reuses the same Claude-vision pattern as /api/ai/detect-eligibility-info.
|
|
//
|
|
// The AI reports position as a 0-1 ratio of image width/height rather than absolute pixels.
|
|
// Vision models are more reliable at "this is 4/10 of the way across, 3/10 down" than at
|
|
// naming an exact pixel — asking for pixels directly produced coordinates that were close but
|
|
// occasionally off by enough to land on a neighboring row or the title bar above it. A ratio
|
|
// is resolution-independent by construction, so it's converted to real screen pixels here
|
|
// using the actual screenshot dimensions rather than trusting the model to know the image size.
|
|
export async function locateOnScreenshot(
|
|
userId: number,
|
|
imageBase64: string,
|
|
goal: string
|
|
): Promise<{ x: number; y: number; xRatio: number; yRatio: number } | null> {
|
|
const aiSettings = await storage.getAiSettings(userId);
|
|
const activeAi = resolveAiProvider(aiSettings ?? {});
|
|
if (!activeAi) {
|
|
throw new Error("AI is not configured. Add an API key in AI Settings.");
|
|
}
|
|
if (activeAi.provider !== "claude") {
|
|
throw new Error("Vision-guided steps require Claude to be the active AI provider.");
|
|
}
|
|
|
|
const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
|
|
const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata();
|
|
|
|
const content = [
|
|
{
|
|
type: "text",
|
|
text:
|
|
`This is a screenshot of a Windows desktop application. Find this UI element: "${goal}". ` +
|
|
"Respond with strict JSON only, no prose, no markdown fences: " +
|
|
'{"found": true, "xRatio": <0 to 1>, "yRatio": <0 to 1>} if you can locate it, or ' +
|
|
'{"found": false} if it isn\'t visible in this screenshot. xRatio is how far across the ' +
|
|
"image the element's center is, as a fraction of the TOTAL image width (0 = left edge, " +
|
|
"0.5 = horizontal center, 1 = right edge). yRatio is how far down the element's center is, " +
|
|
"as a fraction of the TOTAL image height (0 = top edge, 1 = bottom edge). Use two decimal " +
|
|
"places of precision — do not report pixel coordinates.",
|
|
},
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
},
|
|
];
|
|
|
|
// A response that's fully unparseable even after the lenient repair above is worth one
|
|
// retry (rare model hiccup); anything the repair can salvage is used immediately, no
|
|
// retry needed.
|
|
const MAX_ATTEMPTS = 2;
|
|
let lastError: Error | null = null;
|
|
|
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
const response = await llm.invoke([{ role: "user", content }] as any);
|
|
const raw = String(response.content).trim();
|
|
|
|
const parsed = parseLocationResponse(raw);
|
|
if (!parsed) {
|
|
lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
|
|
continue;
|
|
}
|
|
|
|
if (!parsed.found || typeof parsed.xRatio !== "number" || typeof parsed.yRatio !== "number") {
|
|
return null;
|
|
}
|
|
return {
|
|
x: Math.round(parsed.xRatio * width),
|
|
y: Math.round(parsed.yRatio * height),
|
|
xRatio: parsed.xRatio,
|
|
yRatio: parsed.yRatio,
|
|
};
|
|
}
|
|
|
|
throw lastError ?? new Error("AI returned an unparseable response");
|
|
}
|
|
|
|
// Crops a small square around (x, y) and draws a red crosshair marker at that exact point —
|
|
// a debug artifact saved before each click so a person can see afterward exactly where the
|
|
// run was about to click. A real screen capture won't reliably include the OS mouse cursor
|
|
// (BitBlt-based grabs typically omit it), so this synthetic marker stands in for it.
|
|
export async function cropAroundPoint(imageBase64: string, x: number, y: number): Promise<string> {
|
|
const source = sharp(Buffer.from(imageBase64, "base64"));
|
|
const { width = 0, height = 0 } = await source.metadata();
|
|
|
|
const cropWidth = Math.min(DEBUG_CROP_SIZE, width);
|
|
const cropHeight = Math.min(DEBUG_CROP_SIZE, height);
|
|
const left = Math.round(Math.max(0, Math.min(x - cropWidth / 2, width - cropWidth)));
|
|
const top = Math.round(Math.max(0, Math.min(y - cropHeight / 2, height - cropHeight)));
|
|
const markerX = x - left;
|
|
const markerY = y - top;
|
|
|
|
const crosshair = Buffer.from(
|
|
`<svg width="${cropWidth}" height="${cropHeight}">` +
|
|
`<line x1="${markerX - 12}" y1="${markerY}" x2="${markerX + 12}" y2="${markerY}" stroke="red" stroke-width="2"/>` +
|
|
`<line x1="${markerX}" y1="${markerY - 12}" x2="${markerX}" y2="${markerY + 12}" stroke="red" stroke-width="2"/>` +
|
|
`<circle cx="${markerX}" cy="${markerY}" r="7" fill="none" stroke="red" stroke-width="2"/>` +
|
|
`</svg>`
|
|
);
|
|
|
|
const cropped = await source
|
|
.extract({ left, top, width: cropWidth, height: cropHeight })
|
|
.composite([{ input: crosshair }])
|
|
.png()
|
|
.toBuffer();
|
|
|
|
return cropped.toString("base64");
|
|
}
|
|
|
|
interface RawMatch {
|
|
xRatio: number;
|
|
yTopRatio: number;
|
|
yBottomRatio: number;
|
|
}
|
|
|
|
// Parses {"matches": [{"xRatio":.., "yTopRatio":.., "yBottomRatio":..}, ...]}, tolerating
|
|
// near-miss JSON the same way parseLocationResponse does — falls back to regex-extracting every
|
|
// triple in the response if strict parsing fails, rather than treating a malformed-but-salvageable
|
|
// response as zero matches.
|
|
function parseMatchesResponse(raw: string): RawMatch[] | null {
|
|
const block = raw.match(/\{[\s\S]*\}/)?.[0];
|
|
if (!block) return null;
|
|
|
|
try {
|
|
const parsed = JSON.parse(block);
|
|
if (Array.isArray(parsed.matches)) return parsed.matches;
|
|
} catch {
|
|
// fall through to lenient repair below
|
|
}
|
|
|
|
const triples: RawMatch[] = [];
|
|
const tripleRegex =
|
|
/"xRatio"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"yTopRatio"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"yBottomRatio"\s*:\s*(-?\d+(?:\.\d+)?)/g;
|
|
let match: RegExpExecArray | null;
|
|
while ((match = tripleRegex.exec(block))) {
|
|
triples.push({ xRatio: Number(match[1]), yTopRatio: Number(match[2]), yBottomRatio: Number(match[3]) });
|
|
}
|
|
return triples.length > 0 ? triples : null;
|
|
}
|
|
|
|
// Like locateOnScreenshot, but finds every occurrence of an exact piece of text rather than
|
|
// one best guess for a described element. Used to disambiguate by position instead of by
|
|
// asking the AI to judge which occurrence is "the right one" — e.g. a last name shows up once
|
|
// in the search box and once per matching patient row, so the caller pairs it against another
|
|
// text's occurrences (a first name) that share the same row instead of relying on a single
|
|
// fuzzy vision judgment call.
|
|
//
|
|
// Reports each match's TOP and BOTTOM pixel edge rather than asking the model to self-estimate a
|
|
// "center" — a center is an abstract judgment call, whereas the top/bottom of a glyph is a
|
|
// concrete, visible thing to point at. The vertical midpoint used for x/y below is then computed
|
|
// here in code from those two real edges, not trusted as a direct AI estimate.
|
|
export async function locateAllOnScreenshot(
|
|
userId: number,
|
|
imageBase64: string,
|
|
text: string
|
|
): Promise<{ x: number; y: number }[]> {
|
|
const aiSettings = await storage.getAiSettings(userId);
|
|
const activeAi = resolveAiProvider(aiSettings ?? {});
|
|
if (!activeAi) {
|
|
throw new Error("AI is not configured. Add an API key in AI Settings.");
|
|
}
|
|
if (activeAi.provider !== "claude") {
|
|
throw new Error("Vision-guided steps require Claude to be the active AI provider.");
|
|
}
|
|
|
|
const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
|
|
const { width = 0, height = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata();
|
|
|
|
const content = [
|
|
{
|
|
type: "text",
|
|
text:
|
|
`This is a screenshot of a Windows desktop application. Find EVERY occurrence of the text ` +
|
|
`"${text}" visible anywhere in this screenshot (case-insensitive) — there may be zero, one, ` +
|
|
"or several. Respond with strict JSON only, no prose, no markdown fences: " +
|
|
'{"matches": [{"xRatio": <0 to 1>, "yTopRatio": <0 to 1>, "yBottomRatio": <0 to 1>}, ...]} — ' +
|
|
"one entry per occurrence. xRatio is how far across the image the occurrence's horizontal " +
|
|
"center is, as a fraction of the TOTAL image width. yTopRatio is where the TOP edge of that " +
|
|
"occurrence's text (the top of its tallest letters) is, and yBottomRatio is where its BOTTOM " +
|
|
"edge (the bottom of its lowest letters, including descenders) is — both as a fraction of " +
|
|
"the TOTAL image height. Use an empty array if there are no occurrences.",
|
|
},
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
},
|
|
];
|
|
|
|
const MAX_ATTEMPTS = 2;
|
|
let lastError: Error | null = null;
|
|
|
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
const response = await llm.invoke([{ role: "user", content }] as any);
|
|
const raw = String(response.content).trim();
|
|
|
|
const matches = parseMatchesResponse(raw);
|
|
if (!matches) {
|
|
lastError = new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
|
|
continue;
|
|
}
|
|
return matches.map((m) => ({
|
|
x: Math.round(m.xRatio * width),
|
|
y: Math.round(((m.yTopRatio + m.yBottomRatio) / 2) * height),
|
|
}));
|
|
}
|
|
|
|
throw lastError ?? new Error("AI returned an unparseable response");
|
|
}
|
|
|
|
// Asks the AI to measure the height of one row in a results-grid list, as a ratio of the
|
|
// image's total height — used to build a grid of valid row-center y-positions so a click can be
|
|
// snapped toward the middle of whichever row it's closest to, rather than trusting a single
|
|
// direct locate call's y-coordinate on its own (which tends to land near a row's top edge).
|
|
export async function detectRowHeightRatio(userId: number, imageBase64: string): Promise<number> {
|
|
const aiSettings = await storage.getAiSettings(userId);
|
|
const activeAi = resolveAiProvider(aiSettings ?? {});
|
|
if (!activeAi) {
|
|
throw new Error("AI is not configured. Add an API key in AI Settings.");
|
|
}
|
|
if (activeAi.provider !== "claude") {
|
|
throw new Error("Vision-guided steps require Claude to be the active AI provider.");
|
|
}
|
|
|
|
const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
|
|
|
|
const content = [
|
|
{
|
|
type: "text",
|
|
text:
|
|
"This is a screenshot containing a results-grid list of rows (e.g. a patient list) with a " +
|
|
"header row followed by data rows, all data rows the same height. IGNORE the header row — " +
|
|
"it is taller than a data row and would skew the measurement. Look at ALL the visible data " +
|
|
"rows (not just one) and measure their AVERAGE height. Respond with strict JSON only, no " +
|
|
'prose, no markdown fences: {"rowHeightRatio": <0 to 1>} — the average data row height as ' +
|
|
"a fraction of the TOTAL image height. Use three decimal places of precision.",
|
|
},
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
},
|
|
];
|
|
|
|
const response = await llm.invoke([{ role: "user", content }] as any);
|
|
const raw = String(response.content).trim();
|
|
const match = raw.match(/"rowHeightRatio"\s*:\s*(-?\d+(?:\.\d+)?)/);
|
|
if (!match) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
|
|
return Number(match[1]);
|
|
}
|
|
|
|
// Asks the AI to find the x-positions of the MAJOR vertical dividers that split a window into
|
|
// its big panels/columns (e.g. a "Patient Info" panel, a middle fields+list panel, a right-side
|
|
// grid panel) — as opposed to minor lines inside a single table's own columns. Used to tell
|
|
// apart same-text matches that live in different panels (e.g. "Exam" appearing both in a
|
|
// clickable procedures list and, separately, in an already-added-procedures grid) by which
|
|
// column they actually fall in, rather than guessing from y-position alone.
|
|
export async function detectColumnBoundaries(userId: number, imageBase64: string): Promise<number[]> {
|
|
const aiSettings = await storage.getAiSettings(userId);
|
|
const activeAi = resolveAiProvider(aiSettings ?? {});
|
|
if (!activeAi) {
|
|
throw new Error("AI is not configured. Add an API key in AI Settings.");
|
|
}
|
|
if (activeAi.provider !== "claude") {
|
|
throw new Error("Vision-guided steps require Claude to be the active AI provider.");
|
|
}
|
|
|
|
const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
|
|
const { width = 0 } = await sharp(Buffer.from(imageBase64, "base64")).metadata();
|
|
|
|
const content = [
|
|
{
|
|
type: "text",
|
|
text:
|
|
"This screenshot shows a Windows dialog laid out in a small number of large vertical " +
|
|
"sections (columns) — e.g. a distinct panel on the left, a middle section with fields " +
|
|
'and/or a list, and another panel or grid on the right. Find the x-position of each MAJOR ' +
|
|
"vertical divider between these big sections (a visible line, border, or clear gap — NOT " +
|
|
"the minor column lines inside a single table). Respond with strict JSON only, no prose, " +
|
|
'no markdown fences: {"dividerXRatios": [<0 to 1>, ...]} — one entry per major divider, ' +
|
|
"sorted left to right, as a fraction of the TOTAL image width. Use two decimal places of " +
|
|
"precision.",
|
|
},
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: `data:image/png;base64,${imageBase64}` },
|
|
},
|
|
];
|
|
|
|
const response = await llm.invoke([{ role: "user", content }] as any);
|
|
const raw = String(response.content).trim();
|
|
const block = raw.match(/\{[\s\S]*\}/)?.[0];
|
|
if (!block) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
|
|
|
|
let ratios: number[];
|
|
try {
|
|
const parsed = JSON.parse(block);
|
|
if (!Array.isArray(parsed.dividerXRatios)) throw new Error("not an array");
|
|
ratios = parsed.dividerXRatios;
|
|
} catch {
|
|
const arrayMatch = block.match(/"dividerXRatios"\s*:\s*\[([^\]]*)\]/);
|
|
if (!arrayMatch) throw new Error(`AI returned an unparseable response: ${raw.slice(0, 200)}`);
|
|
ratios = arrayMatch[1]!
|
|
.split(",")
|
|
.map((s) => Number(s.trim()))
|
|
.filter((n) => !Number.isNaN(n));
|
|
}
|
|
|
|
return ratios.map((r) => Math.round(r * width)).sort((a, b) => a - b);
|
|
}
|
|
|
|
export interface WindowBounds {
|
|
left: number;
|
|
top: number;
|
|
width: number;
|
|
height: number;
|
|
}
|
|
|
|
// Compares two full screenshots and returns the bounding box of whatever changed
|
|
// significantly between them — used to find a newly-opened dialog's exact position/size with
|
|
// pixel math instead of an AI call, so later steps can crop every screenshot down to just that
|
|
// window and never even show the AI anything outside it (the earlier bug where a name prefix
|
|
// happened to also match unrelated text elsewhere on screen, e.g. "All Received" in a side
|
|
// panel, is structurally impossible once matching only ever sees the cropped window).
|
|
//
|
|
// Rather than a raw min/max over any differing pixel, this counts changed pixels per row and
|
|
// per column and only counts a row/column as part of the new window once a meaningful fraction
|
|
// of it changed — a real window opening changes a large contiguous block, whereas incidental
|
|
// noise elsewhere (a blinking clock, a flickering tray icon) only touches a handful of pixels
|
|
// and would otherwise blow the bounding box out to include it.
|
|
export async function diffBoundingBox(
|
|
beforeBase64: string,
|
|
afterBase64: string
|
|
): Promise<WindowBounds | null> {
|
|
const [before, after] = await Promise.all([
|
|
sharp(Buffer.from(beforeBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }),
|
|
sharp(Buffer.from(afterBase64, "base64")).ensureAlpha().raw().toBuffer({ resolveWithObject: true }),
|
|
]);
|
|
const { data: b, info } = before;
|
|
const { data: a, info: infoAfter } = after;
|
|
if (info.width !== infoAfter.width || info.height !== infoAfter.height) return null;
|
|
|
|
const { width, height, channels } = info;
|
|
const PIXEL_DIFF_THRESHOLD = 40; // sum of |dR| + |dG| + |dB| to count a pixel as "changed"
|
|
const rowCounts = new Uint32Array(height);
|
|
const colCounts = new Uint32Array(width);
|
|
|
|
for (let y = 0; y < height; y++) {
|
|
const rowBase = y * width * channels;
|
|
for (let x = 0; x < width; x++) {
|
|
const idx = rowBase + x * channels;
|
|
const diff = Math.abs(a[idx]! - b[idx]!) + Math.abs(a[idx + 1]! - b[idx + 1]!) + Math.abs(a[idx + 2]! - b[idx + 2]!);
|
|
if (diff > PIXEL_DIFF_THRESHOLD) {
|
|
rowCounts[y] = (rowCounts[y] ?? 0) + 1;
|
|
colCounts[x] = (colCounts[x] ?? 0) + 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
const rowThreshold = width * 0.05;
|
|
const colThreshold = height * 0.05;
|
|
|
|
let top = -1;
|
|
let bottom = -1;
|
|
for (let y = 0; y < height; y++) {
|
|
if (rowCounts[y]! > rowThreshold) {
|
|
if (top === -1) top = y;
|
|
bottom = y;
|
|
}
|
|
}
|
|
let left = -1;
|
|
let right = -1;
|
|
for (let x = 0; x < width; x++) {
|
|
if (colCounts[x]! > colThreshold) {
|
|
if (left === -1) left = x;
|
|
right = x;
|
|
}
|
|
}
|
|
|
|
if (top === -1 || left === -1) return null;
|
|
return { left, top, width: right - left + 1, height: bottom - top + 1 };
|
|
}
|
|
|
|
// Plain crop, no marker — used to restrict a screenshot to a previously-detected window's
|
|
// bounds before handing it to any locate call.
|
|
export async function cropToRegion(imageBase64: string, region: WindowBounds): Promise<string> {
|
|
const cropped = await sharp(Buffer.from(imageBase64, "base64"))
|
|
.extract({ left: region.left, top: region.top, width: region.width, height: region.height })
|
|
.png()
|
|
.toBuffer();
|
|
return cropped.toString("base64");
|
|
}
|
|
|
|
// Sanity check for steps that type without first vision-clicking into a field (relying on
|
|
// whatever already has focus) — confirms the text actually landed somewhere visible on
|
|
// screen before moving on, catching the case where focus wasn't where it was assumed to be.
|
|
export async function confirmTextTyped(
|
|
userId: number,
|
|
fullScreenshotBase64: string,
|
|
expectedText: string
|
|
): Promise<boolean> {
|
|
const aiSettings = await storage.getAiSettings(userId);
|
|
const activeAi = resolveAiProvider(aiSettings ?? {});
|
|
if (!activeAi) {
|
|
throw new Error("AI is not configured. Add an API key in AI Settings.");
|
|
}
|
|
if (activeAi.provider !== "claude") {
|
|
throw new Error("Vision-guided steps require Claude to be the active AI provider.");
|
|
}
|
|
|
|
const llm = getLlm(activeAi.provider, activeAi.key, activeAi.model);
|
|
|
|
const content = [
|
|
{
|
|
type: "text",
|
|
text:
|
|
`This is a screenshot of a Windows desktop application. Is the text "${expectedText}" ` +
|
|
"visible typed into some text input field on screen? Respond with strict JSON only, no " +
|
|
'prose, no markdown fences: {"confirmed": true} if that text is visible in a field, or ' +
|
|
'{"confirmed": false} if it is not visible anywhere.',
|
|
},
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: `data:image/png;base64,${fullScreenshotBase64}` },
|
|
},
|
|
];
|
|
|
|
const response = await llm.invoke([{ role: "user", content }] as any);
|
|
const raw = String(response.content).trim();
|
|
const match = raw.match(/"confirmed"\s*:\s*(true|false)/i);
|
|
return match?.[1]?.toLowerCase() === "true";
|
|
}
|