From ebc5fba2c9b21894ae11b3ac5bcac4b1727e2635 Mon Sep 17 00:00:00 2001 From: Gitead Date: Fri, 30 Oct 2026 15:53:34 -0400 Subject: [PATCH] feat: MassHealth member detail demographics, wait for name/plan before PDF - Extract gender, address (street/city/state/zip) and home phone from the MassHealth Member Detail page and save to the patient; phone is only filled when the patient has none - Shared extract_eligibility_row polls up to 30s until the matching results row has both name and plan before the PDF is printed (AngularJS fills cell text after the rows appear); used by all three MH eligibility workers - Fall back to the Member Detail heading for the name when the results table never filled it in Co-Authored-By: Claude Opus 5.5 --- apps/Backend/src/queue/processors/_shared.ts | 27 ++++ ...mspEligibilityHistoryRemainingProcessor.ts | 2 + .../processors/eligibilityHistoryProcessor.ts | 2 + .../SeleniumService/mh_eligibility_results.py | 80 ++++++++++++ apps/SeleniumService/mh_member_details.py | 74 +++++++++++ ..._eligibilityHistoryRemainingCheckWorker.py | 70 +++------- ...lenium_MH_eligibilityHistoryCheckWorker.py | 75 +++-------- .../selenium_eligibilityCheckWorker.py | 121 +----------------- 8 files changed, 219 insertions(+), 232 deletions(-) create mode 100644 apps/SeleniumService/mh_eligibility_results.py create mode 100644 apps/SeleniumService/mh_member_details.py diff --git a/apps/Backend/src/queue/processors/_shared.ts b/apps/Backend/src/queue/processors/_shared.ts index 7e18aa6c..6b9ff084 100644 --- a/apps/Backend/src/queue/processors/_shared.ts +++ b/apps/Backend/src/queue/processors/_shared.ts @@ -60,6 +60,33 @@ export function splitName(fullName?: string | null) { return { firstName, lastName }; } +/** + * Patient updates from the MassHealth Member Detail block (gender, address, + * home phone) — only fields the portal actually returned. Phone is only + * filled when the patient has none, so a number the office entered is kept. + */ +export function memberDetailDemographics( + seleniumResult: any, + existingPhone?: string | null +): Record { + const updates: Record = {}; + for (const key of ["address", "city", "state", "zipCode"] as const) { + const val = seleniumResult?.[key]; + if (typeof val === "string" && val.trim()) updates[key] = val.trim(); + } + + // "Female"/"Male" → patient form values + const rawGender = String(seleniumResult?.gender ?? "").trim().toUpperCase(); + const gender = rawGender.startsWith("M") ? "male" : rawGender.startsWith("F") ? "female" : ""; + if (gender) updates.gender = gender; + + // Stored like the patient form: "1" + 10 digits + const digits = String(seleniumResult?.phone ?? "").replace(/\D/g, ""); + if (digits.length === 10 && !(existingPhone ?? "").trim()) updates.phone = "1" + digits; + + return updates; +} + export async function imageToPdfBuffer(imagePath: string): Promise { return new Promise((resolve, reject) => { try { diff --git a/apps/Backend/src/queue/processors/cmspEligibilityHistoryRemainingProcessor.ts b/apps/Backend/src/queue/processors/cmspEligibilityHistoryRemainingProcessor.ts index b902fae9..fbac5308 100644 --- a/apps/Backend/src/queue/processors/cmspEligibilityHistoryRemainingProcessor.ts +++ b/apps/Backend/src/queue/processors/cmspEligibilityHistoryRemainingProcessor.ts @@ -12,6 +12,7 @@ import { callPythonSync, splitName, createOrUpdatePatientByInsuranceId, + memberDetailDemographics, } from "./_shared"; export interface CmspEligibilityHistoryRemainingProcessorInput { @@ -112,6 +113,7 @@ export async function runCmspEligibilityHistoryRemainingProcessor( const updates: any = {}; if (newStatus) updates.status = newStatus; if (seleniumInsurance) updates.insuranceProvider = seleniumInsurance; + Object.assign(updates, memberDetailDemographics(seleniumResult, patient.phone)); const remainingBenefit = seleniumResult.remaining_amount?.trim() || null; if (remainingBenefit) updates.remainingBenefit = remainingBenefit; diff --git a/apps/Backend/src/queue/processors/eligibilityHistoryProcessor.ts b/apps/Backend/src/queue/processors/eligibilityHistoryProcessor.ts index e888e24b..5ac83901 100644 --- a/apps/Backend/src/queue/processors/eligibilityHistoryProcessor.ts +++ b/apps/Backend/src/queue/processors/eligibilityHistoryProcessor.ts @@ -12,6 +12,7 @@ import { callPythonSync, splitName, createOrUpdatePatientByInsuranceId, + memberDetailDemographics, } from "./_shared"; export interface EligibilityHistoryProcessorInput { @@ -103,6 +104,7 @@ export async function runEligibilityHistoryProcessor( const updates: any = { status: newStatus }; if (seleniumInsurance) updates.insuranceProvider = seleniumInsurance; + Object.assign(updates, memberDetailDemographics(seleniumResult, patient.phone)); await storage.updatePatient(patient.id, updates); outputResult.patientUpdateStatus = `Patient status updated to ${newStatus}`; diff --git a/apps/SeleniumService/mh_eligibility_results.py b/apps/SeleniumService/mh_eligibility_results.py new file mode 100644 index 00000000..9e041b5f --- /dev/null +++ b/apps/SeleniumService/mh_eligibility_results.py @@ -0,0 +1,80 @@ +import time + +from selenium.webdriver.common.by import By + +# Cell texts of every row in the Eligible / Ineligible results tables. +# innerText (not .text) so cells rendered off-screen still return their text. +_ROWS_JS = """ +const out = []; +for (const label of ['Eligible', 'Ineligible']) { + const rows = document.evaluate( + `//h4[normalize-space(text())='${label}']/following::table[1]/tbody/tr`, + document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); + for (let i = 0; i < rows.snapshotLength; i++) { + const cells = Array.from(rows.snapshotItem(i).querySelectorAll('td')) + .map(td => (td.innerText || td.textContent || '').trim()); + out.push([label, cells]); + } +} +return out; +""" + + +def _normalize_id(s): + return "".join(c for c in str(s) if c.isalnum()).lower() + + +def extract_eligibility_row(driver, member_id, timeout=30): + """Wait for the MassHealth results row matching member_id and return + {eligibility, firstName, lastName, insurance}. + + AngularJS adds the rows first and fills the cell text afterwards — the + member ID can appear before the name and plan do. Keep polling until the + matching row has both a name and a plan, so callers never print the PDF + before the data is captured. After the timeout, returns whatever was + found ({"eligibility": None} if no row matched). + """ + norm_self = _normalize_id(member_id) + best = {"eligibility": None} + deadline = time.time() + timeout + attempt = 0 + + while True: + attempt += 1 + try: + rows = driver.execute_script(_ROWS_JS) or [] + except Exception as e: + print(f"[extraction] reading rows failed: {e}") + rows = [] + + for label, cells in rows: + if len(cells) < 3: + continue + norm_cell = _normalize_id(cells[2]) + if not (norm_self and norm_cell and (norm_self in norm_cell or norm_cell in norm_self)): + continue + + # name in cells[4], plan in cells[6] (fallback to last cell) + full_name = cells[4] if len(cells) > 4 else "" + plan_name = cells[6] if len(cells) > 6 else (cells[-1] if len(cells) > 4 else "") + name_parts = full_name.split() + best = { + "eligibility": "Y" if label == "Eligible" else "N", + "firstName": name_parts[0] if name_parts else "", + "lastName": " ".join(name_parts[1:]), + "insurance": plan_name, + } + if full_name and plan_name: + print(f"[extraction] MATCHED {label} → name='{full_name}' plan='{plan_name}' (attempt {attempt})") + return best + break + + if time.time() >= deadline: + break + if attempt % 5 == 1: + state = "name/plan still empty" if best["eligibility"] else "no matching row yet" + print(f"[extraction] {state} for memberId='{member_id}' (attempt {attempt}), waiting...") + time.sleep(1) + + print(f"[extraction] gave up after {timeout}s for memberId='{member_id}' → {best}") + return best diff --git a/apps/SeleniumService/mh_member_details.py b/apps/SeleniumService/mh_member_details.py new file mode 100644 index 00000000..1e39f5d9 --- /dev/null +++ b/apps/SeleniumService/mh_member_details.py @@ -0,0 +1,74 @@ +import re + +from selenium.webdriver.common.by import By + +# Labels shown in the MassHealth Member Detail block, e.g. +# Date of Birth: 4/21/1958 +# Gender: Female +# Address: 169 CYPRESS ST, NEWTON, MA 02459 +# Home Phone: 6036789376 +# Registered Not Registered in Member Community +_LABELS = ("Date of Birth", "Gender", "Address", "Home Phone", "Phone", "Registered") + + +def _label_value(text, label): + """Value next to "Label:" — same line or the next line (innerText of a + two-column layout can put the value on its own line). Empty when the + portal leaves the field blank.""" + m = re.search(rf"(?:^|\n)[ \t]*{re.escape(label)}:[ \t]*\n?[ \t]*([^\n]*)", text) + if not m: + return "" + value = m.group(1).strip() + # Blank field — we grabbed the next row's label instead of a value + if value.endswith(":") or any(value.startswith(f"{l}:") or value == l for l in _LABELS): + return "" + return value + + +def extract_member_demographics(driver): + """Read gender, address and phone from the MassHealth Member Detail page. + + Returns only the fields that were found: + gender — "Female" / "Male" as shown by the portal + address, city, state, zipCode — split from "STREET, CITY, ST 12345" + phone — 10 digits + memberName — the heading above "Date of Birth:" (e.g. "OLHA SVYSHCH") + """ + result = {} + try: + text = driver.find_element(By.TAG_NAME, "body").get_attribute("innerText") or "" + + # Name is the last non-empty line before the "Date of Birth:" row + dob = re.search(r"(?:^|\n)[ \t]*Date of Birth:", text) + if dob: + lines = [l.strip() for l in text[:dob.start()].split("\n") if l.strip()] + if lines and ":" not in lines[-1]: + result["memberName"] = lines[-1] + + gender = _label_value(text, "Gender") + if gender: + result["gender"] = gender + + address = _label_value(text, "Address") + if address: + # "169 CYPRESS ST, NEWTON, MA 02459" — street may itself contain commas (APT 2) + m = re.match(r"^(.*),\s*([^,]+?),?\s+([A-Z]{2})\s+(\d{5})(?:-?\d{4})?$", address) + if m: + result["address"] = m.group(1).strip() + result["city"] = m.group(2).strip() + result["state"] = m.group(3) + result["zipCode"] = m.group(4) + else: + result["address"] = address + + phone = _label_value(text, "Home Phone") or _label_value(text, "Phone") + digits = re.sub(r"\D", "", phone) + if len(digits) == 11 and digits.startswith("1"): + digits = digits[1:] + if len(digits) == 10: + result["phone"] = digits + + print(f"[extract_member_demographics] {result}") + except Exception as e: + print(f"[extract_member_demographics] failed: {e}") + return result diff --git a/apps/SeleniumService/selenium_CMSP_eligibilityHistoryRemainingCheckWorker.py b/apps/SeleniumService/selenium_CMSP_eligibilityHistoryRemainingCheckWorker.py index 349856b0..8c333a86 100644 --- a/apps/SeleniumService/selenium_CMSP_eligibilityHistoryRemainingCheckWorker.py +++ b/apps/SeleniumService/selenium_CMSP_eligibilityHistoryRemainingCheckWorker.py @@ -9,6 +9,9 @@ import time import os import base64 +from mh_eligibility_results import extract_eligibility_row +from mh_member_details import extract_member_demographics + class AutomationCMSPEligibilityHistoryRemainingCheck: def __init__(self, data): @@ -161,62 +164,9 @@ class AutomationCMSPEligibilityHistoryRemainingCheck: # ── helpers ───────────────────────────────────────────────────────────────── - def _cell_text(self, cell): - text = cell.text.strip() - if not text: - try: - text = (self.driver.execute_script("return arguments[0].innerText;", cell) or "").strip() - except Exception: - pass - return text - - def _normalize_id(self, s): - return "".join(c for c in str(s) if c.isalnum()).lower() - def _extract_data_from_page(self): - wait = WebDriverWait(self.driver, 15) - try: - wait.until( - EC.presence_of_element_located( - (By.XPATH, "//h4[text()='Eligible' or text()='Ineligible']/following::table[1]/tbody/tr") - ) - ) - # AngularJS populates cell text after elements appear in the DOM. - # Retry a few times until a row with non-empty cell[2] is found. - for attempt in range(4): - for status_label, elig_flag in [("Eligible", "Y"), ("Ineligible", "N")]: - rows = self.driver.find_elements( - By.XPATH, - f"//h4[text()='{status_label}']/following::table[1]/tbody/tr" - ) - for row in rows: - cells = row.find_elements(By.TAG_NAME, "td") - if len(cells) < 3: - continue - norm_cell = self._normalize_id(self._cell_text(cells[2])) - norm_self = self._normalize_id(self.memberId) - if norm_self and norm_cell and (norm_self in norm_cell or norm_cell in norm_self): - full_name = self._cell_text(cells[4]) if len(cells) > 4 else "" - plan_name = ( - self._cell_text(cells[6]) if len(cells) > 6 - else (self._cell_text(cells[-1]) if len(cells) > 4 else "") - ) - name_parts = full_name.split() - return { - "eligibility": elig_flag, - "firstName": name_parts[0] if name_parts else "", - "lastName": " ".join(name_parts[1:]) if len(name_parts) > 1 else "", - "insurance": plan_name, - } - if attempt < 3: - print(f"[extraction] cells not yet populated (attempt {attempt + 1}), retrying...") - time.sleep(1.5) - - print(f"[extraction] No matching row for memberId='{self.memberId}'") - return {"eligibility": None} - except Exception as e: - print("Extraction error:", e) - return {"eligibility": None} + # Waits until the matching row's name and plan are filled in + return extract_eligibility_row(self.driver, self.memberId) # ── step 2 — print eligibility PDF, return to results tab ─────────────────── @@ -336,6 +286,16 @@ class AutomationCMSPEligibilityHistoryRemainingCheck: self.extract_third_party_insurance() + # Gender, address and phone from the Member Detail block + details = extract_member_demographics(self.driver) + member_name = details.pop("memberName", "") + # Results table never filled the name in — use the Member Detail heading + if member_name and not self.extracted_data.get("firstName"): + parts = member_name.split() + details["firstName"] = parts[0] + details["lastName"] = " ".join(parts[1:]) + self.extracted_data.update(details) + safe_member = "".join(c for c in str(self.memberId) if c.isalnum() or c in "-_.") pdf_filename = f"mh_member_details_{safe_member}.pdf" pdf_data = self.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True}) diff --git a/apps/SeleniumService/selenium_MH_eligibilityHistoryCheckWorker.py b/apps/SeleniumService/selenium_MH_eligibilityHistoryCheckWorker.py index 7613b13e..b6e5628d 100644 --- a/apps/SeleniumService/selenium_MH_eligibilityHistoryCheckWorker.py +++ b/apps/SeleniumService/selenium_MH_eligibilityHistoryCheckWorker.py @@ -9,6 +9,9 @@ import time import os import base64 +from mh_eligibility_results import extract_eligibility_row +from mh_member_details import extract_member_demographics + class AutomationMassHealthEligibilityHistoryCheck: def __init__(self, data): @@ -191,67 +194,9 @@ class AutomationMassHealthEligibilityHistoryCheck: # ── helpers ───────────────────────────────────────────────────────────────── - def _cell_text(self, cell): - text = cell.text.strip() - if not text: - try: - text = (self.driver.execute_script("return arguments[0].innerText;", cell) or "").strip() - except Exception: - pass - return text - - def _normalize_id(self, s): - return "".join(c for c in str(s) if c.isalnum()).lower() - def _extract_data_from_page(self): - wait = WebDriverWait(self.driver, 15) - extracted = {} - try: - wait.until( - EC.presence_of_element_located( - (By.XPATH, "//h4[text()='Eligible' or text()='Ineligible']/following::table[1]/tbody/tr") - ) - ) - - # AngularJS populates cell text after elements appear in the DOM. - # Retry a few times until a row with non-empty cell[2] is found. - for attempt in range(4): - for status_label, elig_flag in [("Eligible", "Y"), ("Ineligible", "N")]: - rows = self.driver.find_elements( - By.XPATH, - f"//h4[text()='{status_label}']/following::table[1]/tbody/tr" - ) - for row in rows: - cells = row.find_elements(By.TAG_NAME, "td") - if len(cells) < 3: - continue - member_number = self._cell_text(cells[2]) - norm_cell = self._normalize_id(member_number) - norm_self = self._normalize_id(self.memberId) - if norm_self and norm_cell and (norm_self in norm_cell or norm_cell in norm_self): - full_name = self._cell_text(cells[4]) if len(cells) > 4 else "" - plan_name = ( - self._cell_text(cells[6]) if len(cells) > 6 - else (self._cell_text(cells[-1]) if len(cells) > 4 else "") - ) - name_parts = full_name.split() - extracted = { - "eligibility": elig_flag, - "firstName": name_parts[0] if name_parts else "", - "lastName": " ".join(name_parts[1:]) if len(name_parts) > 1 else "", - "insurance": plan_name, - } - print(f"[extraction] MATCHED {status_label} → name='{full_name}' plan='{plan_name}'") - return extracted - if attempt < 3: - print(f"[extraction] cells not yet populated (attempt {attempt + 1}), retrying...") - time.sleep(1.5) - - print(f"[extraction] No matching row for memberId='{self.memberId}'") - return {"eligibility": None} - except Exception as e: - print("Extraction error:", e) - return {"eligibility": None} + # Waits until the matching row's name and plan are filled in + return extract_eligibility_row(self.driver, self.memberId) # ── step 2 — print eligibility PDF, stay on results tab ───────────────────── @@ -333,6 +278,16 @@ class AutomationMassHealthEligibilityHistoryCheck: wait.until(lambda d: d.execute_script("return document.readyState") == "complete") time.sleep(2) + # Gender, address and phone from the Member Detail block + details = extract_member_demographics(self.driver) + member_name = details.pop("memberName", "") + # Results table never filled the name in — use the Member Detail heading + if member_name and not self.extracted_data.get("firstName"): + parts = member_name.split() + details["firstName"] = parts[0] + details["lastName"] = " ".join(parts[1:]) + self.extracted_data.update(details) + safe_member = "".join(c for c in str(self.memberId) if c.isalnum() or c in "-_.") pdf_filename = f"eligibility_member_details_{safe_member}.pdf" pdf_data = self.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True}) diff --git a/apps/SeleniumService/selenium_eligibilityCheckWorker.py b/apps/SeleniumService/selenium_eligibilityCheckWorker.py index 019d664b..1ac1a442 100755 --- a/apps/SeleniumService/selenium_eligibilityCheckWorker.py +++ b/apps/SeleniumService/selenium_eligibilityCheckWorker.py @@ -9,6 +9,8 @@ import time import os import base64 +from mh_eligibility_results import extract_eligibility_row + class AutomationMassHealthEligibilityCheck: def __init__(self, data): self.headless = False @@ -245,124 +247,9 @@ class AutomationMassHealthEligibilityCheck: print(f"[step1] Could not save screenshot: {ss_err}") return f"ERROR:STEP1:{substep}" - def _cell_text(self, cell): - """Get text from a cell, falling back to JS innerText if .text is empty.""" - text = cell.text.strip() - if not text: - try: - text = (self.driver.execute_script("return arguments[0].innerText;", cell) or "").strip() - except Exception: - pass - return text - - def _normalize_id(self, s): - """Strip all non-alphanumeric characters and lowercase for robust ID matching.""" - return ''.join(c for c in str(s) if c.isalnum()).lower() - def _extract_data_from_page(self): - wait = WebDriverWait(self.driver, 15) - extracted = {} - - try: - # Wait until at least one table row appears under Eligible or Ineligible — - # the h4 header renders before the rows are populated, so we must wait for rows. - wait.until( - EC.presence_of_element_located( - (By.XPATH, "//h4[text()='Eligible' or text()='Ineligible']/following::table[1]/tbody/tr") - ) - ) - - eligible_rows = self.driver.find_elements( - By.XPATH, - "//h4[text()='Eligible']/following::table[1]/tbody/tr" - ) - - if eligible_rows: - for row in eligible_rows: - cells = row.find_elements(By.TAG_NAME, "td") - - if len(cells) < 3: - continue - - member_number = self._cell_text(cells[2]) - norm_cell = self._normalize_id(member_number) - norm_self = self._normalize_id(self.memberId) - print(f"[eligible] cells count={len(cells)}, memberId check: '{norm_self}' vs '{norm_cell}' (raw: '{member_number}')") - if len(cells) >= 5: - print(f" cells[3]='{self._cell_text(cells[3])}' cells[4]='{self._cell_text(cells[4])}'", end="") - if len(cells) > 5: - print(f" cells[5]='{self._cell_text(cells[5])}'", end="") - if len(cells) > 6: - print(f" cells[6]='{self._cell_text(cells[6])}'", end="") - print() - - if norm_self and norm_cell and (norm_self in norm_cell or norm_cell in norm_self): - # name is in cells[4], insurance in cells[6] (fallback to last cell) - full_name = self._cell_text(cells[4]) if len(cells) > 4 else "" - plan_name = self._cell_text(cells[6]) if len(cells) > 6 else (self._cell_text(cells[-1]) if len(cells) > 4 else "") - - name_parts = full_name.split() - first_name = name_parts[0] if name_parts else "" - last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else "" - - print(f"[eligible] MATCHED → name='{full_name}' plan='{plan_name}'") - - extracted = { - "eligibility": "Y", - "firstName": first_name, - "lastName": last_name, - "insurance": plan_name - } - - return extracted - - ineligible_rows = self.driver.find_elements( - By.XPATH, - "//h4[text()='Ineligible']/following::table[1]/tbody/tr" - ) - - if ineligible_rows: - for row in ineligible_rows: - cells = row.find_elements(By.TAG_NAME, "td") - - if len(cells) < 3: - continue - - member_number = self._cell_text(cells[2]) - norm_cell = self._normalize_id(member_number) - norm_self = self._normalize_id(self.memberId) - print(f"[ineligible] cells count={len(cells)}, memberId check: '{norm_self}' vs '{norm_cell}' (raw: '{member_number}')") - if len(cells) >= 5: - print(f" cells[3]='{self._cell_text(cells[3])}' cells[4]='{self._cell_text(cells[4])}'", end="") - if len(cells) > 5: - print(f" cells[5]='{self._cell_text(cells[5])}'", end="") - print() - - if norm_self and norm_cell and (norm_self in norm_cell or norm_cell in norm_self): - full_name = self._cell_text(cells[4]) if len(cells) > 4 else "" - plan_name = self._cell_text(cells[5]) if len(cells) > 5 else (self._cell_text(cells[-1]) if len(cells) > 4 else "") - - name_parts = full_name.split() - first_name = name_parts[0] if name_parts else "" - last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else "" - - print(f"[ineligible] MATCHED → name='{full_name}' plan='{plan_name}'") - - extracted = { - "eligibility": "N", - "firstName": first_name, - "lastName": last_name, - "insurance": plan_name - } - - return extracted - - print(f"[extraction] No matching row found for memberId='{self.memberId}'") - return {"eligibility": None} - - except Exception as e: - print("Extraction error:", e) - return {"eligibility": None} + # Waits until the matching row's name and plan are filled in + return extract_eligibility_row(self.driver, self.memberId) def _print_current_page_as_pdf(self): """Generate PDF from the currently active tab using CDP."""