feat: integrate DDMA eligibility into BullMQ queue with persistent session

- Route DDMA eligibility through InProcessQueue (concurrency=1) so it
  queues behind other selenium jobs instead of running concurrently
- New ddmaEligibilityProcessor: starts Python session, polls for OTP/
  completion via socket events, saves PDF and updates patient DB
- Frontend ddma-buton-modal now uses shared app socket + job:update
  pattern (drops private socket connection)
- SeleniumService: upgrade ddma_browser_manager with credential hash
  tracking, anti-detection options, and startup session clearing;
  upgrade DDMA worker with firstName/lastName support, PDF via
  printToPDF, force-logout on credential change; upgrade helpers with
  dual OTP strategy (app API + browser polling); add /clear-ddma-session
  endpoint; reduce fixed sleeps with smart WebDriverWait

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ff
2026-04-16 09:21:47 -04:00
parent a1cccc8716
commit 289ea426d3
9 changed files with 1429 additions and 1239 deletions

View File

@@ -1,5 +1,4 @@
import { useEffect, useRef, useState } from "react";
import { io as ioClient, Socket } from "socket.io-client";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
@@ -9,13 +8,11 @@ import { apiRequest, queryClient } from "@/lib/queryClient";
import { useAppDispatch } from "@/redux/hooks";
import { setTaskStatus } from "@/redux/slices/seleniumTaskSlice";
import { formatLocalDate } from "@/utils/dateUtils";
import { socket } from "@/lib/socket";
import { QK_PATIENTS_BASE } from "@/components/patients/patient-table";
const SOCKET_URL =
import.meta.env.VITE_API_BASE_URL_BACKEND ||
(typeof window !== "undefined" ? window.location.origin : "");
// ─── OTP Modal ────────────────────────────────────────────────────────────────
// ---------- OTP Modal component ----------
interface DdmaOtpModalProps {
open: boolean;
onClose: () => void;
@@ -23,12 +20,7 @@ interface DdmaOtpModalProps {
isSubmitting: boolean;
}
function DdmaOtpModal({
open,
onClose,
onSubmit,
isSubmitting,
}: DdmaOtpModalProps) {
function DdmaOtpModal({ open, onClose, onSubmit, isSubmitting }: DdmaOtpModalProps) {
const [otp, setOtp] = useState("");
useEffect(() => {
@@ -48,17 +40,13 @@ function DdmaOtpModal({
<div className="bg-white dark:bg-slate-900 rounded-xl shadow-lg w-full max-w-md p-6">
<div className="flex items-center justify-between mb-4">
<h2 className="text-lg font-semibold">Enter OTP</h2>
<button
type="button"
onClick={onClose}
className="text-slate-500 hover:text-slate-800"
>
<button type="button" onClick={onClose} className="text-slate-500 hover:text-slate-800">
<X className="w-4 h-4" />
</button>
</div>
<p className="text-sm text-slate-500 mb-4">
We need the one-time password (OTP) sent by the Delta Dental MA portal
to complete this eligibility check.
We need the one-time password (OTP) sent by the Delta Dental MA portal to complete this
eligibility check.
</p>
<form onSubmit={handleSubmit} className="space-y-4">
<div className="space-y-2">
@@ -72,12 +60,7 @@ function DdmaOtpModal({
/>
</div>
<div className="flex justify-end gap-3">
<Button
type="button"
variant="outline"
onClick={onClose}
disabled={isSubmitting}
>
<Button type="button" variant="outline" onClick={onClose} disabled={isSubmitting}>
Cancel
</Button>
<Button type="submit" disabled={isSubmitting || !otp.trim()}>
@@ -97,14 +80,14 @@ function DdmaOtpModal({
);
}
// ---------- Main DDMA Eligibility button component ----------
// ─── Main component ───────────────────────────────────────────────────────────
interface DdmaEligibilityButtonProps {
memberId: string;
dateOfBirth: Date | null;
firstName?: string;
lastName?: string;
isFormIncomplete: boolean;
/** Called when backend has finished and PDF is ready */
onPdfReady: (pdfId: number, fallbackFilename: string | null) => void;
}
@@ -119,267 +102,18 @@ export function DdmaEligibilityButton({
const { toast } = useToast();
const dispatch = useAppDispatch();
const socketRef = useRef<Socket | null>(null);
const connectingRef = useRef<Promise<void> | null>(null);
// session_id is provided by the backend once the Python agent starts the
// browser session. We receive it via the selenium:ddma_session_started event
// and need it to forward the OTP back.
const sessionIdRef = useRef<string | null>(null);
const [sessionId, setSessionId] = useState<string | null>(null);
const [otpModalOpen, setOtpModalOpen] = useState(false);
const [isStarting, setIsStarting] = useState(false);
const [isSubmittingOtp, setIsSubmittingOtp] = useState(false);
// Clean up socket on unmount
useEffect(() => {
return () => {
if (socketRef.current) {
socketRef.current.removeAllListeners();
socketRef.current.disconnect();
socketRef.current = null;
}
connectingRef.current = null;
};
}, []);
// ── Socket event handlers ─────────────────────────────────────────────────
const closeSocket = () => {
try {
socketRef.current?.removeAllListeners();
socketRef.current?.disconnect();
} catch (e) {
// ignore
} finally {
socketRef.current = null;
}
};
// Lazy socket setup: called only when we actually need it (first click)
const ensureSocketConnected = async () => {
// If already connected, nothing to do
if (socketRef.current && socketRef.current.connected) {
return;
}
// If a connection is in progress, reuse that promise
if (connectingRef.current) {
return connectingRef.current;
}
const promise = new Promise<void>((resolve, reject) => {
const socket = ioClient(SOCKET_URL, {
withCredentials: true,
});
socketRef.current = socket;
socket.on("connect", () => {
console.log("DDMA socket connected:", socket.id);
resolve();
});
// connection error when first connecting (or later)
socket.on("connect_error", (err: any) => {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: "Connection failed",
})
);
toast({
title: "Realtime connection failed",
description:
"Could not connect to realtime server. Retrying automatically...",
variant: "destructive",
});
// do not reject here because socket.io will attempt reconnection
});
// socket.io will emit 'reconnect_attempt' for retries
socket.on("reconnect_attempt", (attempt: number) => {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: `Realtime reconnect attempt #${attempt}`,
})
);
});
// when reconnection failed after configured attempts
socket.on("reconnect_failed", () => {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: "Reconnect failed",
})
);
toast({
title: "Realtime reconnect failed",
description:
"Connection to realtime server could not be re-established. Please try again later.",
variant: "destructive",
});
// terminal failure — cleanup and reject so caller can stop start flow
closeSocket();
reject(new Error("Realtime reconnect failed"));
});
socket.on("disconnect", (reason: any) => {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: "Connection disconnected",
})
);
toast({
title: "Connection Disconnected",
description:
"Connection to the server was lost. If a DDMA job was running it may have failed.",
variant: "destructive",
});
// clear sessionId/OTP modal
setSessionId(null);
setOtpModalOpen(false);
});
// OTP required
socket.on("selenium:otp_required", (payload: any) => {
if (!payload?.session_id) return;
setSessionId(payload.session_id);
setOtpModalOpen(true);
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "OTP required for DDMA eligibility. Please enter the OTP.",
})
);
});
// OTP submitted (optional UX)
socket.on("selenium:otp_submitted", (payload: any) => {
if (!payload?.session_id) return;
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "OTP submitted. Finishing DDMA eligibility check...",
})
);
});
// Session update
socket.on("selenium:session_update", (payload: any) => {
const { session_id, status, final } = payload || {};
if (!session_id) return;
if (status === "completed") {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "success",
message:
"DDMA eligibility updated and PDF attached to patient documents.",
})
);
toast({
title: "DDMA eligibility complete",
description:
"Patient status was updated and the eligibility PDF was saved.",
variant: "default",
});
const pdfId = final?.pdfFileId;
if (pdfId) {
const filename =
final?.pdfFilename ?? `eligibility_ddma_${memberId}.pdf`;
onPdfReady(Number(pdfId), filename);
}
setSessionId(null);
setOtpModalOpen(false);
} else if (status === "error") {
const msg =
payload?.message ||
final?.error ||
"DDMA eligibility session failed.";
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: msg,
})
);
toast({
title: "DDMA selenium error",
description: msg,
variant: "destructive",
});
// Ensure socket is torn down for this session (stop receiving stale events)
try {
closeSocket();
} catch (e) {}
setSessionId(null);
setOtpModalOpen(false);
}
queryClient.invalidateQueries({ queryKey: QK_PATIENTS_BASE });
});
// explicit session error event (helpful)
socket.on("selenium:session_error", (payload: any) => {
const msg = payload?.message || "Selenium session error";
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: msg,
})
);
toast({
title: "Selenium session error",
description: msg,
variant: "destructive",
});
// tear down socket to avoid stale updates
try {
closeSocket();
} catch (e) {}
setSessionId(null);
setOtpModalOpen(false);
});
// If socket.io initial connection fails permanently (very rare: client-level)
// set a longer timeout to reject the first attempt to connect.
const initialConnectTimeout = setTimeout(() => {
if (!socket.connected) {
// if still not connected after 8s, treat as failure and reject so caller can handle it
closeSocket();
reject(new Error("Realtime initial connection timeout"));
}
}, 8000);
// When the connect resolves we should clear this timer
socket.once("connect", () => {
clearTimeout(initialConnectTimeout);
});
});
// store promise to prevent multiple concurrent connections
connectingRef.current = promise;
try {
await promise;
} finally {
connectingRef.current = null;
}
};
const startDdmaEligibility = async () => {
const handleDdmaStart = async () => {
if (!memberId || !dateOfBirth) {
toast({
title: "Missing fields",
@@ -389,107 +123,220 @@ export function DdmaEligibilityButton({
return;
}
const formattedDob = dateOfBirth ? formatLocalDate(dateOfBirth) : "";
const formattedDob = formatLocalDate(dateOfBirth);
const payload = {
memberId,
dateOfBirth: formattedDob,
firstName,
lastName,
insuranceSiteKey: "DDMA", // make sure this matches backend credential key
insuranceSiteKey: "DDMA",
};
setIsStarting(true);
try {
setIsStarting(true);
// 1) Ensure socket is connected (lazy)
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "Opening realtime channel for DDMA eligibility...",
})
);
await ensureSocketConnected();
const socket = socketRef.current;
if (!socket || !socket.connected) {
throw new Error("Socket connection failed");
}
const socketId = socket.id;
// 2) Start the selenium job via backend
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "Starting DDMA eligibility check via selenium...",
message: "Starting DDMA eligibility check…",
})
);
// 1) POST to backend — returns { status: "queued", jobId }
const response = await apiRequest(
"POST",
"/api/insurance-status-ddma/ddma-eligibility",
{
data: JSON.stringify(payload),
socketId,
}
{ data: JSON.stringify(payload), socketId: socket.id }
);
// If apiRequest threw, we would have caught above; but just in case it returns.
let result: any = null;
let backendError: string | null = null;
try {
// attempt JSON first
result = await response.clone().json();
backendError =
result?.error || result?.message || result?.detail || null;
} catch {
// fallback to text response
try {
const text = await response.clone().text();
backendError = text?.trim() || null;
} catch {
backendError = null;
}
const result = await response.json();
if (!response.ok || result.error) {
throw new Error(result.error || `Server error (${response.status})`);
}
if (!response.ok) {
throw new Error(
backendError ||
`DDMA selenium start failed (status ${response.status})`
);
}
const jobId: string = result.jobId;
if (!jobId) throw new Error("No jobId returned from server");
// Normal success path: optional: if backend returns non-error shape still check for result.error
if (result?.error) {
throw new Error(result.error);
}
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "DDMA job queued. Waiting for browser session to start…",
})
);
if (result.status === "started" && result.session_id) {
setSessionId(result.session_id as string);
// 2) Listen for job-lifecycle and DDMA-specific socket events.
// All events come through the shared app socket.
// Handler: Python agent started a browser session → we now have session_id
const onSessionStarted = (data: any) => {
if (String(data?.jobId) !== String(jobId)) return;
sessionIdRef.current = data.session_id ?? null;
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message:
"DDMA eligibility job started. Waiting for OTP or final result...",
message: "Browser session started. Waiting for OTP or result…",
})
);
} else {
// fallback if backend returns immediate result
};
// Handler: OTP is required by the DDMA portal
const onOtpRequired = (data: any) => {
if (String(data?.jobId) !== String(jobId)) return;
// Update sessionId in case it arrives here first
if (data.session_id) sessionIdRef.current = data.session_id;
setOtpModalOpen(true);
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "success",
message: "DDMA eligibility completed.",
status: "pending",
message: "OTP required for Delta Dental MA. Please enter the code.",
})
);
};
// Handler: OTP accepted by Python agent (optional UX feedback)
const onOtpSubmitted = (data: any) => {
if (data?.session_id && data.session_id !== sessionIdRef.current) return;
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: "OTP submitted. Finishing DDMA eligibility check…",
})
);
};
// Handler: job completed or failed (from InProcessQueue)
const onJobUpdate = (data: any) => {
if (String(data?.jobId) !== String(jobId)) return;
if (data.status === "active") {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: data.message ?? "Selenium browser starting…",
})
);
return;
}
// Terminal states
cleanup();
if (data.status === "completed") {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "success",
message: "DDMA eligibility updated and PDF attached to patient documents.",
})
);
toast({
title: "DDMA eligibility complete",
description: "Patient status was updated and the eligibility PDF was saved.",
});
queryClient.invalidateQueries({ queryKey: QK_PATIENTS_BASE });
const pdfId = data.result?.pdfFileId;
if (pdfId) {
const filename =
data.result?.pdfFilename ?? `eligibility_ddma_${memberId}.pdf`;
onPdfReady(Number(pdfId), filename);
}
} else if (data.status === "failed") {
const msg = data.error ?? "DDMA eligibility job failed.";
dispatch(
setTaskStatus({ key: "eligibilityCheck", status: "error", message: msg })
);
toast({ title: "DDMA selenium error", description: msg, variant: "destructive" });
}
setIsStarting(false);
setOtpModalOpen(false);
};
// Attach listeners
socket.on("selenium:ddma_session_started", onSessionStarted);
socket.on("selenium:otp_required", onOtpRequired);
socket.on("selenium:otp_submitted", onOtpSubmitted);
socket.on("job:update", onJobUpdate);
// Cleanup helper removes all listeners for this job
function cleanup() {
socket.off("selenium:ddma_session_started", onSessionStarted);
socket.off("selenium:otp_required", onOtpRequired);
socket.off("selenium:otp_submitted", onOtpSubmitted);
socket.off("job:update", onJobUpdate);
}
// Safety timeout — clean up listeners if no terminal event in 6 min
const safetyTimer = setTimeout(() => {
cleanup();
setIsStarting(false);
setOtpModalOpen(false);
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "error",
message: "DDMA job timed out waiting for completion.",
})
);
}, 6 * 60 * 1000);
// Patch cleanup to also clear the timer
const originalCleanup = cleanup;
function cleanupWithTimer() {
clearTimeout(safetyTimer);
originalCleanup();
}
// Override the onJobUpdate cleanup reference
socket.off("job:update", onJobUpdate);
socket.on("job:update", (data: any) => {
if (String(data?.jobId) !== String(jobId)) return;
if (data.status === "active") {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "pending",
message: data.message ?? "Selenium browser starting…",
})
);
return;
}
cleanupWithTimer();
if (data.status === "completed") {
dispatch(
setTaskStatus({
key: "eligibilityCheck",
status: "success",
message: "DDMA eligibility updated and PDF attached to patient documents.",
})
);
toast({
title: "DDMA eligibility complete",
description: "Patient status was updated and the eligibility PDF was saved.",
});
queryClient.invalidateQueries({ queryKey: QK_PATIENTS_BASE });
const pdfId = data.result?.pdfFileId;
if (pdfId) {
onPdfReady(Number(pdfId), data.result?.pdfFilename ?? `eligibility_ddma_${memberId}.pdf`);
}
} else if (data.status === "failed") {
const msg = data.error ?? "DDMA eligibility job failed.";
dispatch(setTaskStatus({ key: "eligibilityCheck", status: "error", message: msg }));
toast({ title: "DDMA selenium error", description: msg, variant: "destructive" });
}
setIsStarting(false);
setOtpModalOpen(false);
});
} catch (err: any) {
console.error("startDdmaEligibility error:", err);
console.error("DdmaEligibilityButton error:", err);
dispatch(
setTaskStatus({
key: "eligibilityCheck",
@@ -502,17 +349,18 @@ export function DdmaEligibilityButton({
description: err?.message || "Failed to start DDMA eligibility",
variant: "destructive",
});
} finally {
setIsStarting(false);
}
};
// ── OTP submission ────────────────────────────────────────────────────────
const handleSubmitOtp = async (otp: string) => {
if (!sessionId || !socketRef.current || !socketRef.current.connected) {
const sessionId = sessionIdRef.current;
if (!sessionId) {
toast({
title: "Session not ready",
description:
"Could not submit OTP because the DDMA session or socket is not ready.",
description: "Cannot submit OTP — DDMA session ID is not available yet.",
variant: "destructive",
});
return;
@@ -520,21 +368,15 @@ export function DdmaEligibilityButton({
try {
setIsSubmittingOtp(true);
const resp = await apiRequest(
"POST",
"/api/insurance-status-ddma/selenium/submit-otp",
{
session_id: sessionId,
otp,
socketId: socketRef.current.id,
}
);
const resp = await apiRequest("POST", "/api/insurance-status-ddma/selenium/submit-otp", {
session_id: sessionId,
otp,
socketId: socket.id,
});
const data = await resp.json();
if (!resp.ok || data.error) {
throw new Error(data.error || "Failed to submit OTP");
}
// from here we rely on websocket events (otp_submitted + session_update)
setOtpModalOpen(false);
} catch (err: any) {
console.error("handleSubmitOtp error:", err);
@@ -548,13 +390,15 @@ export function DdmaEligibilityButton({
}
};
// ── Render ────────────────────────────────────────────────────────────────
return (
<>
<Button
className="w-full"
variant="default"
disabled={isFormIncomplete || isStarting}
onClick={startDdmaEligibility}
onClick={handleDdmaStart}
>
{isStarting ? (
<>