feat: integrate DDMA eligibility into BullMQ queue with persistent session
- Route DDMA eligibility through InProcessQueue (concurrency=1) so it queues behind other selenium jobs instead of running concurrently - New ddmaEligibilityProcessor: starts Python session, polls for OTP/ completion via socket events, saves PDF and updates patient DB - Frontend ddma-buton-modal now uses shared app socket + job:update pattern (drops private socket connection) - SeleniumService: upgrade ddma_browser_manager with credential hash tracking, anti-detection options, and startup session clearing; upgrade DDMA worker with firstName/lastName support, PDF via printToPDF, force-logout on credential change; upgrade helpers with dual OTP strategy (app API + browser polling); add /clear-ddma-session endpoint; reduce fixed sleeps with smart WebDriverWait Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ff
@@ -11,6 +11,7 @@ import { runEligibilityProcessor } from "./processors/eligibilityProcessor";
|
||||
import { runClaimStatusProcessor } from "./processors/claimStatusProcessor";
|
||||
import { runClaimSubmitProcessor } from "./processors/claimSubmitProcessor";
|
||||
import { runOcrProcessor } from "./processors/ocrProcessor";
|
||||
import { runDdmaEligibilityProcessor } from "./processors/ddmaEligibilityProcessor";
|
||||
import type { SeleniumJobData, OcrJobData } from "./queues";
|
||||
|
||||
// ── Queue instances ──────────────────────────────────────────────────────────
|
||||
@@ -68,6 +69,20 @@ export function enqueueSeleniumJob(data: SeleniumJobData): string {
|
||||
variant: jobType === "claim-pre-auth" ? "claim-pre-auth" : "claimsubmit",
|
||||
});
|
||||
}
|
||||
if (jobType === "ddma-eligibility-check") {
|
||||
return runDdmaEligibilityProcessor(
|
||||
{
|
||||
enrichedPayload: data.enrichedPayload,
|
||||
userId: data.userId,
|
||||
insuranceId: data.insuranceId!,
|
||||
formFirstName: data.formFirstName,
|
||||
formLastName: data.formLastName,
|
||||
formDob: data.formDob,
|
||||
socketId: data.socketId,
|
||||
},
|
||||
job.id
|
||||
);
|
||||
}
|
||||
throw new Error(`Unknown selenium jobType: ${jobType}`);
|
||||
});
|
||||
|
||||
|
||||
359
apps/Backend/src/queue/processors/ddmaEligibilityProcessor.ts
Normal file
359
apps/Backend/src/queue/processors/ddmaEligibilityProcessor.ts
Normal file
@@ -0,0 +1,359 @@
|
||||
/**
|
||||
* Processor for "ddma-eligibility-check" jobs.
|
||||
*
|
||||
* Integrates the full DDMA persistent-session flow into the InProcessQueue:
|
||||
* 1. Start a session on the Python agent (POST /ddma-eligibility)
|
||||
* 2. Emit selenium:ddma_session_started → frontend stores session_id for OTP
|
||||
* 3. Poll agent status, emitting selenium:otp_required when OTP is needed
|
||||
* 4. On completion: save PDF, create/update patient, update eligibility status
|
||||
* 5. Return { pdfFileId, pdfFilename, patientUpdateStatus, pdfUploadStatus }
|
||||
*
|
||||
* The OTP submission endpoint (/api/insurance-status-ddma/selenium/submit-otp)
|
||||
* continues to forward OTPs directly to the Python agent — it does NOT go
|
||||
* through the queue.
|
||||
*/
|
||||
import fs from "fs/promises";
|
||||
import fsSync from "fs";
|
||||
import path from "path";
|
||||
import { storage } from "../../storage";
|
||||
import { emptyFolderContainingFile } from "../../utils/emptyTempFolder";
|
||||
import {
|
||||
forwardToSeleniumDdmaEligibilityAgent,
|
||||
getSeleniumDdmaSessionStatus,
|
||||
} from "../../services/seleniumDdmaInsuranceEligibilityClient";
|
||||
import {
|
||||
splitName,
|
||||
createOrUpdatePatientByInsuranceId,
|
||||
imageToPdfBuffer,
|
||||
} from "./_shared";
|
||||
import { io } from "../../socket";
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function now() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function log(tag: string, msg: string, ctx?: any) {
|
||||
console.log(`${now()} [${tag}] ${msg}`, ctx ?? "");
|
||||
}
|
||||
|
||||
function emitToSocket(socketId: string | undefined, event: string, payload: any) {
|
||||
if (!socketId || !io) return;
|
||||
try {
|
||||
const socket = io.sockets.sockets.get(socketId);
|
||||
if (socket) {
|
||||
socket.emit(event, payload);
|
||||
log("ddma-processor", `emitted ${event}`, { socketId });
|
||||
}
|
||||
} catch (err: any) {
|
||||
log("ddma-processor", `emit failed for ${event}`, { err: err?.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Types ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface DdmaEligibilityProcessorInput {
|
||||
enrichedPayload: any;
|
||||
userId: number;
|
||||
insuranceId: string;
|
||||
formFirstName?: string;
|
||||
formLastName?: string;
|
||||
formDob?: string;
|
||||
socketId?: string;
|
||||
}
|
||||
|
||||
export interface DdmaEligibilityProcessorResult {
|
||||
patientUpdateStatus?: string;
|
||||
pdfUploadStatus?: string;
|
||||
pdfFileId?: number | null;
|
||||
pdfFilename?: string | null;
|
||||
}
|
||||
|
||||
// ─── Core DB processing (mirrors handleDdmaCompletedJob in the route) ─────────
|
||||
|
||||
async function processDdmaResult(
|
||||
userId: number,
|
||||
insuranceId: string,
|
||||
formFirstName: string | undefined,
|
||||
formLastName: string | undefined,
|
||||
formDob: string | undefined,
|
||||
seleniumResult: any
|
||||
): Promise<DdmaEligibilityProcessorResult> {
|
||||
const output: DdmaEligibilityProcessorResult = {};
|
||||
let createdPdfFileId: number | null = null;
|
||||
|
||||
try {
|
||||
// 1) Resolve patient name (prefer selenium extraction → form data)
|
||||
const rawName =
|
||||
typeof seleniumResult?.patientName === "string"
|
||||
? seleniumResult.patientName.trim()
|
||||
: null;
|
||||
|
||||
const { firstName, lastName } = rawName
|
||||
? splitName(rawName)
|
||||
: { firstName: formFirstName ?? "", lastName: formLastName ?? "" };
|
||||
|
||||
// 2) Create / update patient
|
||||
await createOrUpdatePatientByInsuranceId({
|
||||
insuranceId,
|
||||
firstName,
|
||||
lastName,
|
||||
dob: formDob,
|
||||
userId,
|
||||
});
|
||||
|
||||
// 3) Fetch patient (needed for ID)
|
||||
const patient = await storage.getPatientByInsuranceId(insuranceId);
|
||||
if (!patient?.id) {
|
||||
output.patientUpdateStatus = "Patient not found; no update performed";
|
||||
return output;
|
||||
}
|
||||
|
||||
// 4) Determine and update eligibility status + insurance provider name
|
||||
const eligStatus = (seleniumResult?.eligibility ?? "").toLowerCase();
|
||||
const newStatus =
|
||||
eligStatus === "active" || eligStatus === "y" ? "ACTIVE" : "INACTIVE";
|
||||
await storage.updatePatient(patient.id, {
|
||||
status: newStatus,
|
||||
insuranceProvider: "Delta Dental MA",
|
||||
});
|
||||
output.patientUpdateStatus = `Patient status updated to ${newStatus}`;
|
||||
|
||||
// 5) Resolve PDF buffer
|
||||
// New DDMA worker returns a real PDF via pdf_path / ss_path (.pdf).
|
||||
// Old worker returned a screenshot (.png) via ss_path.
|
||||
let pdfBuffer: Buffer | null = null;
|
||||
let pdfFilename: string | null = null;
|
||||
|
||||
const pdfPath: string | null =
|
||||
seleniumResult?.pdf_path ?? seleniumResult?.ss_path ?? null;
|
||||
|
||||
if (pdfPath && fsSync.existsSync(pdfPath)) {
|
||||
if (pdfPath.endsWith(".pdf")) {
|
||||
// Already a PDF — read directly
|
||||
try {
|
||||
pdfBuffer = await fs.readFile(pdfPath);
|
||||
pdfFilename = path.basename(pdfPath);
|
||||
log("ddma-processor", "read PDF directly", { pdfPath });
|
||||
} catch (e: any) {
|
||||
output.pdfUploadStatus = `Failed to read PDF: ${e.message}`;
|
||||
}
|
||||
} else if (
|
||||
pdfPath.endsWith(".png") ||
|
||||
pdfPath.endsWith(".jpg") ||
|
||||
pdfPath.endsWith(".jpeg")
|
||||
) {
|
||||
// Legacy screenshot → convert to PDF
|
||||
try {
|
||||
pdfBuffer = await imageToPdfBuffer(pdfPath);
|
||||
pdfFilename = `ddma_eligibility_${insuranceId}_${Date.now()}.pdf`;
|
||||
log("ddma-processor", "converted screenshot to PDF", { pdfPath });
|
||||
} catch (e: any) {
|
||||
output.pdfUploadStatus = `Failed to convert screenshot to PDF: ${e.message}`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
output.pdfUploadStatus = "No valid file path from Selenium; nothing uploaded.";
|
||||
}
|
||||
|
||||
// 6) Save PDF to patient document group
|
||||
if (pdfBuffer && pdfFilename) {
|
||||
const groupTitleKey = "ELIGIBILITY_STATUS";
|
||||
const groupTitle = "Eligibility Status";
|
||||
|
||||
let group = await storage.findPdfGroupByPatientTitleKey(patient.id, groupTitleKey);
|
||||
if (!group) group = await storage.createPdfGroup(patient.id, groupTitle, groupTitleKey);
|
||||
if (!group?.id) throw new Error("PDF group creation failed");
|
||||
|
||||
const created = await storage.createPdfFile(group.id, pdfFilename, pdfBuffer);
|
||||
if (created && typeof created === "object" && "id" in created) {
|
||||
createdPdfFileId = Number(created.id);
|
||||
}
|
||||
output.pdfUploadStatus = `PDF saved to group: ${group.title}`;
|
||||
output.pdfFilename = pdfFilename;
|
||||
}
|
||||
|
||||
output.pdfFileId = createdPdfFileId;
|
||||
return output;
|
||||
} catch (err: any) {
|
||||
return {
|
||||
...output,
|
||||
pdfUploadStatus:
|
||||
output.pdfUploadStatus ?? `Processing failed: ${err?.message ?? String(err)}`,
|
||||
pdfFileId: createdPdfFileId,
|
||||
};
|
||||
} finally {
|
||||
// Always clean up temp files
|
||||
const cleanupPath =
|
||||
seleniumResult?.pdf_path ?? seleniumResult?.ss_path ?? null;
|
||||
if (cleanupPath) {
|
||||
try {
|
||||
await emptyFolderContainingFile(cleanupPath);
|
||||
} catch (e) {
|
||||
log("ddma-processor", "cleanup failed", { cleanupPath });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Polling loop ────────────────────────────────────────────────────────────
|
||||
|
||||
async function pollUntilDone(
|
||||
sessionId: string,
|
||||
socketId: string | undefined,
|
||||
jobId: string,
|
||||
pollTimeoutMs = 5 * 60 * 1000 // 5 min total (includes OTP wait)
|
||||
): Promise<any> {
|
||||
const maxAttempts = 600; // 600 × 500ms = 5 min
|
||||
const pollIntervalMs = 500;
|
||||
const maxTransientErrors = 12;
|
||||
const noProgressLimit = 120; // 60 s of same status → abort
|
||||
|
||||
let transientErrors = 0;
|
||||
let consecutiveNoProgress = 0;
|
||||
let lastStatus: string | null = null;
|
||||
const deadline = Date.now() + pollTimeoutMs;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error(
|
||||
`DDMA polling timeout (${Math.round(pollTimeoutMs / 1000)}s) for session ${sessionId}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const st = await getSeleniumDdmaSessionStatus(sessionId);
|
||||
const status: string = st?.status ?? "unknown";
|
||||
|
||||
log("ddma-processor", `poll attempt=${attempt}`, { sessionId, status });
|
||||
|
||||
transientErrors = 0; // reset on success
|
||||
|
||||
// Track no-progress
|
||||
const isTerminal =
|
||||
status === "completed" || status === "error" || status === "not_found";
|
||||
if (status === lastStatus && !isTerminal) {
|
||||
consecutiveNoProgress++;
|
||||
} else {
|
||||
consecutiveNoProgress = 0;
|
||||
}
|
||||
lastStatus = status;
|
||||
|
||||
if (consecutiveNoProgress >= noProgressLimit) {
|
||||
throw new Error(
|
||||
`No progress from Python agent (status="${status}") after ${consecutiveNoProgress} polls`
|
||||
);
|
||||
}
|
||||
|
||||
// OTP required — notify frontend and keep polling
|
||||
if (status === "waiting_for_otp") {
|
||||
emitToSocket(socketId, "selenium:otp_required", {
|
||||
session_id: sessionId,
|
||||
jobId,
|
||||
message: "OTP required. Please enter the OTP shown by the DDMA portal.",
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, pollIntervalMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (status === "completed") {
|
||||
log("ddma-processor", "session completed", { sessionId });
|
||||
return st.result;
|
||||
}
|
||||
|
||||
if (status === "error" || status === "not_found") {
|
||||
throw new Error(
|
||||
st?.message || `DDMA session ended with status: ${status}`
|
||||
);
|
||||
}
|
||||
|
||||
// Still running / otp_submitted / created — keep polling
|
||||
await new Promise((r) => setTimeout(r, pollIntervalMs));
|
||||
} catch (err: any) {
|
||||
// Propagate terminal errors immediately
|
||||
const isTerminal =
|
||||
err?.response?.status === 404 ||
|
||||
(typeof err?.message === "string" &&
|
||||
(err.message.includes("not_found") ||
|
||||
err.message.includes("polling timeout") ||
|
||||
err.message.includes("No progress")));
|
||||
|
||||
if (isTerminal) throw err;
|
||||
|
||||
// Transient network errors — back off
|
||||
transientErrors++;
|
||||
if (transientErrors > maxTransientErrors) {
|
||||
throw new Error(
|
||||
`Too many transient network errors polling DDMA session ${sessionId}`
|
||||
);
|
||||
}
|
||||
const backoff = Math.min(30_000, 500 * Math.pow(2, transientErrors - 1));
|
||||
log("ddma-processor", `transient error #${transientErrors}, backoff ${backoff}ms`, {
|
||||
err: err?.message,
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, backoff));
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`DDMA polling exhausted all attempts for session ${sessionId}`);
|
||||
}
|
||||
|
||||
// ─── Main processor entry point ───────────────────────────────────────────────
|
||||
|
||||
export async function runDdmaEligibilityProcessor(
|
||||
input: DdmaEligibilityProcessorInput,
|
||||
jobId: string
|
||||
): Promise<DdmaEligibilityProcessorResult> {
|
||||
const {
|
||||
enrichedPayload,
|
||||
userId,
|
||||
insuranceId,
|
||||
formFirstName,
|
||||
formLastName,
|
||||
formDob,
|
||||
socketId,
|
||||
} = input;
|
||||
|
||||
// 1) Tell Python agent to start a DDMA session
|
||||
log("ddma-processor", "starting Python agent session", { insuranceId });
|
||||
const agentResp = await forwardToSeleniumDdmaEligibilityAgent(enrichedPayload);
|
||||
|
||||
if (!agentResp?.session_id) {
|
||||
throw new Error(
|
||||
"Python agent did not return a session_id for DDMA eligibility"
|
||||
);
|
||||
}
|
||||
|
||||
const sessionId = agentResp.session_id as string;
|
||||
log("ddma-processor", "got session_id", { sessionId });
|
||||
|
||||
// 2) Emit session started so frontend can store session_id for OTP submission
|
||||
emitToSocket(socketId, "selenium:ddma_session_started", {
|
||||
session_id: sessionId,
|
||||
jobId,
|
||||
});
|
||||
|
||||
// 3) Poll until done (handles OTP events internally)
|
||||
const seleniumResult = await pollUntilDone(sessionId, socketId, jobId);
|
||||
|
||||
if (!seleniumResult || seleniumResult.status === "error") {
|
||||
throw new Error(
|
||||
seleniumResult?.message ?? "DDMA session returned an error result"
|
||||
);
|
||||
}
|
||||
|
||||
// 4) Process DB writes and PDF upload
|
||||
log("ddma-processor", "processing DB result", { insuranceId });
|
||||
const result = await processDdmaResult(
|
||||
userId,
|
||||
insuranceId,
|
||||
formFirstName,
|
||||
formLastName,
|
||||
formDob,
|
||||
seleniumResult
|
||||
);
|
||||
|
||||
log("ddma-processor", "done", { result });
|
||||
return result;
|
||||
}
|
||||
@@ -6,7 +6,8 @@ export type SeleniumJobType =
|
||||
| "eligibility-check"
|
||||
| "claim-status-check"
|
||||
| "claim-submit"
|
||||
| "claim-pre-auth";
|
||||
| "claim-pre-auth"
|
||||
| "ddma-eligibility-check";
|
||||
|
||||
export interface SeleniumJobData {
|
||||
jobType: SeleniumJobType;
|
||||
|
||||
Reference in New Issue
Block a user