From 0cd095330aabef6669d152d4dd445864406777d5 Mon Sep 17 00:00:00 2001 From: ff Date: Thu, 6 Aug 2026 09:35:48 -0400 Subject: [PATCH] fix: AI recall-call flow, speech-to-text time parsing, and stale conversation state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reset conversation stage to a fresh start on every new voice call (recall and reminder alike), and on every hangup/timeout/goodbye, so an abandoned or completed call never leaves a patient stuck mid-flow for their next call or text. - Add a staleness check to the SMS flow: after a long idle gap, check whether the patient's reply still reads as a continuation before resuming the stored stage, falling back to a friendly "it's been a while" re-greeting when unsure. - Fix parseTime() to handle Twilio's speech-to-text output, which renders "pm"/"am" with periods and often drops the colon in a spoken time (e.g. "100 p.m." -> 1:00 PM) — this was silently failing and causing the AI to loop asking for a time it had already been given. - Extend the voice Gather timeout so the AI doesn't hang up on a patient who just needs a few seconds to respond. - Reword the recall-call opening line and rework the post-"yes" prompt to ask for day and time together. - Fix rclone WebDAV server to serve only the backups folder instead of the entire backend app directory (was exposing .env and source). Co-Authored-By: Claude Sonnet 5 --- .../src/ai/patient-conversation-router.ts | 16 ++- apps/Backend/src/ai/recall-graph.ts | 34 ++++-- apps/Backend/src/ai/reschedule-graph.ts | 23 +++- apps/Backend/src/routes/twilio-webhooks.ts | 115 ++++++++++++++++-- apps/Backend/src/services/rcloneService.ts | 9 +- apps/Backend/src/storage/twilio-storage.ts | 2 +- .../settings/ai-chat-settings-card.tsx | 2 +- 7 files changed, 167 insertions(+), 34 deletions(-) diff --git a/apps/Backend/src/ai/patient-conversation-router.ts b/apps/Backend/src/ai/patient-conversation-router.ts index 05efec51..d0055619 100644 --- a/apps/Backend/src/ai/patient-conversation-router.ts +++ b/apps/Backend/src/ai/patient-conversation-router.ts @@ -2,7 +2,7 @@ import twilio from "twilio"; import { prisma as db } from "@repo/db/client"; import { storage } from "../storage"; import { runReminderGraph } from "./reminder-graph"; -import { runRecallGraph, runRecallBookingStep } from "./recall-graph"; +import { runRecallGraph, runRecallBookingStep, hasDateOrTimeSignal } from "./recall-graph"; import { runNewPatientStep } from "./new-patient-graph"; import { runRescheduleStep, @@ -290,10 +290,16 @@ export async function routePatientTurn(params: { if (intent === "yes") { // Patient may have already given a date (or date + time) in this same // reply ("Tuesday 1pm") — try to book right away instead of asking again. - const { reply: bookingReply, nextStage: bookingNextStage } = await runRecallBookingStep( - message, "recall_asked_datetime", language, patient.id, patient.userId, activeAi.key, activeAi.provider, activeAi.model - ); - return finish(patient.userId, patient.id, [bookingReply], bookingNextStage); + // A bare "yes"/"ok" has nothing to parse, so skip straight to asking for + // day + time instead of wasting a parse call that used to come back + // empty and send a mismatched "Sorry, I didn't catch a day" reply. + if (hasDateOrTimeSignal(message)) { + const { reply: bookingReply, nextStage: bookingNextStage } = await runRecallBookingStep( + message, "recall_asked_datetime", language, patient.id, patient.userId, activeAi.key, activeAi.provider, activeAi.model + ); + return finish(patient.userId, patient.id, [bookingReply], bookingNextStage); + } + return finish(patient.userId, patient.id, [intentReply], "recall_asked_datetime"); } if (intent === "no") { return finish(patient.userId, patient.id, [intentReply], "done"); diff --git a/apps/Backend/src/ai/recall-graph.ts b/apps/Backend/src/ai/recall-graph.ts index 17a2240c..ca6d5686 100644 --- a/apps/Backend/src/ai/recall-graph.ts +++ b/apps/Backend/src/ai/recall-graph.ts @@ -23,6 +23,19 @@ type GraphStateType = typeof GraphState.State; // ── Intent classifier — multilingual yes/no keywords (shared shape with reminder-graph) ── +/** True if `text` mentions a concrete day/date/time (e.g. "Tuesday", "3/10", "2pm"). + * Shared by classifyNode (a bare "yes" vs. "yes, Tuesday at 1pm" both count as + * intent "yes") and by the caller in patient-conversation-router.ts, which uses + * it to decide whether to attempt an immediate booking parse or just ask for a + * day/time first. */ +export function hasDateOrTimeSignal(text: string): boolean { + return ( + /\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|today|tomorrow|next week)\b/i.test(text) || + /\b\d{1,2}[\/\-]\d{1,2}\b/.test(text) || + /\b\d{1,2}(:\d{2})?\s*(am|pm|a\.m\.|p\.m\.)\b/i.test(text) + ); +} + function classifyNode(state: GraphStateType) { const text = state.message.toLowerCase().trim(); @@ -32,10 +45,7 @@ function classifyNode(state: GraphStateType) { // A specific day/date/time, or an appointment-request phrase, is itself a // strong signal the patient wants to book — real replies rarely say a bare // "yes", they jump straight to "Tuesday at 1pm" or "I want an appointment". - const mentionsDateOrTime = - /\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|tomorrow|next week)\b/i.test(text) || - /\b\d{1,2}[\/\-]\d{1,2}\b/.test(text) || - /\b\d{1,2}(:\d{2})?\s*(am|pm|a\.m\.|p\.m\.)\b/i.test(text); + const mentionsDateOrTime = hasDateOrTimeSignal(text); const mentionsAppointment = /\b(appointment|schedule|book|come in|check.?up|cleaning)\b/i.test(text); if (noPatterns.test(text) && !mentionsDateOrTime) return { intent: "no" }; @@ -52,13 +62,13 @@ function routeByIntent(state: GraphStateType): string { // ── Fallbacks ────────────────────────────────────────────────────────────── const WANTS_BOOKING_FALLBACKS: Record = { - English: "Great! What day works best for you?", - Spanish: "¡Genial! ¿Qué día le viene mejor?", - Portuguese: "Ótimo! Qual dia é melhor para você?", - Mandarin: "太好了！您哪天方便？", - Cantonese: "太好喇！您邊日方便？", - Arabic: "رائع! ما هو اليوم الأنسب لك؟", - "Haitian Creole": "Bon! Ki jou ki pi bon pou ou?", + English: "Great! What day and time work best for you?", + Spanish: "¡Genial! ¿Qué día y hora le vienen mejor?", + Portuguese: "Ótimo! Qual dia e horário são melhores para você?", + Mandarin: "太好了！您哪天、几点方便？", + Cantonese: "太好喇！您邊日、幾點方便？", + Arabic: "رائع! ما هو اليوم والوقت الأنسب لك؟", + "Haitian Creole": "Bon! Ki jou ak ki lè ki pi bon pou ou?", }; const DECLINES_FALLBACKS: Record = { @@ -98,7 +108,7 @@ async function wantsBookingNode(state: GraphStateType, config: any) { { role: "system", content: - `You are a friendly dental office assistant. The patient agreed to come in for a routine exam and cleaning. Write a short, warm reply (1-2 sentences max) and ask what day works best for them. Do NOT introduce yourself or say your name — the introduction has already been sent in a separate message. You MUST reply in ${lang}. No formatting, no extra text.`, + `You are a friendly dental office assistant. The patient agreed to come in for a routine exam and cleaning. Write a short, warm reply (1-2 sentences max) and ask what day AND time work best for them. Do NOT introduce yourself or say your name — the introduction has already been sent in a separate message. You MUST reply in ${lang}. No formatting, no extra text.`, }, { role: "user", content: `Patient replied: "${state.message}"` }, ]); diff --git a/apps/Backend/src/ai/reschedule-graph.ts b/apps/Backend/src/ai/reschedule-graph.ts index 2c82a494..1e492c33 100644 --- a/apps/Backend/src/ai/reschedule-graph.ts +++ b/apps/Backend/src/ai/reschedule-graph.ts @@ -392,18 +392,35 @@ async function isSlotAvailable( // ── Time parsing (legacy) ───────────────────────────────────────────────────── export async function parseTime(message: string, apiKey: string, provider: AiProvider = "google", model?: string): Promise { - const t = message.toLowerCase(); + // Twilio speech-to-text usually renders "pm"/"am" with periods ("p.m."), + // and often drops the colon in a spoken time too ("1:00pm" -> "100 pm", + // "12:30pm" -> "1230 pm") — normalize both before matching. + const t = message.toLowerCase() + .replace(/\ba\.m\.?/g, "am") + .replace(/\bp\.m\.?/g, "pm"); if (/\bmorning\b|mañana|manhã|上午|صباح|maten/i.test(t)) return "09:00"; if (/\bafternoon\b|tarde|après-midi|下午|مساء|aprèmidi/i.test(t)) return "13:00"; - const ampm = t.match(/\b(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/); + const ampm = t.match(/\b(\d{1,2}):([0-5]\d)\s*(am|pm)\b/); + const glued = !ampm ? t.match(/\b(\d{1,4})\s*(am|pm)\b/) : null; const clock = t.match(/\b([01]?\d|2[0-3]):([0-5]\d)\b/); if (ampm) { let h = parseInt(ampm[1]!); - const m = ampm[2] ? parseInt(ampm[2]) : 0; + const m = parseInt(ampm[2]!); if (ampm[3] === "pm" && h < 12) h += 12; if (ampm[3] === "am" && h === 12) h = 0; return `${String(h).padStart(2,"0")}:${String(m).padStart(2,"0")}`; } + if (glued) { + const digits = glued[1]!; + let h: number, m: number; + if (digits.length <= 2) { h = parseInt(digits); m = 0; } + else { h = parseInt(digits.slice(0, -2)); m = parseInt(digits.slice(-2)); } + if (h >= 1 && h <= 12 && m <= 59) { + if (glued[2] === "pm" && h < 12) h += 12; + if (glued[2] === "am" && h === 12) h = 0; + return `${String(h).padStart(2,"0")}:${String(m).padStart(2,"0")}`; + } + } if (clock) return clock[0]!; try { const llm = getLlm(provider, apiKey, model); diff --git a/apps/Backend/src/routes/twilio-webhooks.ts b/apps/Backend/src/routes/twilio-webhooks.ts index 8932669d..2db50ff3 100644 --- a/apps/Backend/src/routes/twilio-webhooks.ts +++ b/apps/Backend/src/routes/twilio-webhooks.ts @@ -12,7 +12,7 @@ import { getOfficeHoursDisplay, timeLabel, } from "../ai/reschedule-graph"; -import { getLlm, resolveAiProvider } from "../ai/llm-factory"; +import { getLlm, resolveAiProvider, type AiProvider } from "../ai/llm-factory"; import { soundsLikeGoodbye } from "../ai/voice-assistant"; import { getPublicBaseUrl } from "../utils/publicUrl"; import { @@ -116,6 +116,64 @@ const NEW_OR_RESCHEDULE_Q: Record = { "Haitian Creole": "Jis pou konfime — èske ou ta renmen pran yon nouvo randevou, oswa èske ou ta renmen reprogramè randevou aktyèl ou?", }; +// ── Stale-conversation handling (SMS) ───────────────────────────────────────── +// Unlike a phone call, a long gap between texts doesn't necessarily mean the +// patient walked away — replies are often just slow. So instead of hard +// resetting on any idle gap, we only second-guess the stored stage once the +// gap crosses STALE_SMS_THRESHOLD_MS, and even then we ask the LLM to look at +// the recent history before deciding whether the new text is still answering +// the last question or reads like a fresh, unrelated conversation. + +const STALE_SMS_THRESHOLD_MS = 3 * 60 * 60 * 1000; // 3 hours + +const STILL_HERE_MSGS: Record = { + English: "It's been a while since our last message. How can I help you today?", + Spanish: "Ha pasado un tiempo desde nuestro último mensaje. ¿Cómo puedo ayudarle hoy?", + Portuguese: "Já faz um tempo desde nossa última mensagem. Como posso ajudá-lo hoje?", + Mandarin: "距离我们上次的消息已经有一段时间了。今天我能帮您什么？", + Cantonese: "距離我們上次嘅消息已經有一段時間喇。今日我可以點幫您？", + Arabic: "لقد مر بعض الوقت منذ آخر رسالة لنا. كيف يمكنني مساعدتك اليوم؟", + "Haitian Creole": "Sa gen yon ti tan depi dènye mesaj nou. Kijan mwen ka ede ou jodi a?", +}; + +/** + * After a long idle gap, ask the LLM whether the patient's new text still + * answers what was last asked (recent history + stage in context) or reads + * like an unrelated, fresh message. Defaults to "continue" on any failure — + * an LLM call error shouldn't be the reason a legitimate reply gets bounced. + */ +async function isLikelyContinuation( + recentHistory: { role: "user" | "assistant"; text: string }[], + newMessage: string, + stage: ConversationStage, + apiKey: string, + provider: AiProvider, + model?: string, +): Promise { + if (recentHistory.length === 0) return true; + try { + const llm = getLlm(provider, apiKey, model); + const transcript = recentHistory + .map(m => `${m.role === "assistant" ? "Office" : "Patient"}: ${m.text}`) + .join("\n"); + const res = await llm.invoke([ + { + role: "system", + content: + `You are helping a dental office text-messaging assistant resume a conversation with a patient after a long gap of silence. ` + + `Here is the recent conversation (oldest first):\n${transcript}\n\n` + + `The assistant is currently waiting on stage "${stage}" of a booking flow. ` + + `The patient just sent a new message after the long gap: "${newMessage}"\n\n` + + `Does this new message still look like it's answering/continuing that same conversation, or does it read like an unrelated, fresh message (e.g. a new question, a greeting, a different topic)? ` + + `Reply with ONLY one word: "continue" or "restart".`, + }, + ]); + return !/restart/i.test(String(res.content)); + } catch { + return true; + } +} + // ── POST /api/twilio/webhook/sms ────────────────────────────────────────────── const CONVO_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes @@ -427,16 +485,33 @@ router.post("/webhook/sms", async (req: Request, res: Response): Promise => let stage = await getStage(patient.userId, patient.id); const chatTemplates = await storage.getAiChatTemplates(patient.userId); - // Reset conversation if idle for more than 5 minutes + // Long idle gap: don't blindly resume (SMS replies are often just slow), + // but don't blindly reset either — check whether the new text still reads + // like a continuation before deciding. if (stage !== "initial" && stage !== "done") { const convRow = await db.patientConversation.findUnique({ where: { patientId: patient.id }, select: { updatedAt: true }, }); - if (convRow?.updatedAt && Date.now() - convRow.updatedAt.getTime() > CONVO_TIMEOUT_MS) { - clearPendingReschedule(patient.userId, patient.id); - await setStage(patient.userId, patient.id, "initial"); - stage = "initial"; + if (convRow?.updatedAt && Date.now() - convRow.updatedAt.getTime() > STALE_SMS_THRESHOLD_MS) { + const recentComms = await storage.getCommunicationsByPatient(patient.id); + const recentHistory = recentComms.slice(-5).map((c: any) => ({ + role: (c.direction === "outbound" ? "assistant" : "user") as "assistant" | "user", + text: c.body || "", + })); + const continuing = await isLikelyContinuation( + recentHistory, Body, stage, activeAi.key, activeAi.provider, activeAi.model + ); + if (!continuing) { + clearPendingReschedule(patient.userId, patient.id); + await setStage(patient.userId, patient.id, "initial"); + stage = "initial"; + const stillHereText = STILL_HERE_MSGS[language] ?? STILL_HERE_MSGS["English"]!; + await saveOutbound(patient.id, stillHereText); + res.set("Content-Type", "text/xml"); + return res.send(twimlMessages(stillHereText)); + } + // Looks like a continuation — fall through and resume the stored stage. } } const officeContact = await storage.getOfficeContact(patient.userId); @@ -690,7 +765,7 @@ function aiVoiceTwiml(sayText: string, gatherActionUrl: string, locale: VoiceLoc // silence timeout — nothing after would ever run. return ` - + ${escapeXml(sayText)} `; @@ -741,10 +816,16 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise= MAX_AI_CALL_TURNS) { aiCallSessions.delete(CallSid); + await setStage(session.userId, session.patientId, "done"); return res.send(aiVoiceHangup("Thank you for calling, have a great day. Goodbye.", locale)); } @@ -802,6 +888,7 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise { const config = readRcloneConfig(); + // Serve only the backups folder — never the app root, which contains + // .env (DB/Twilio/AI credentials) and the full source tree. + const backupsDir = path.join(APP_ROOT, "backups"); + if (!fs.existsSync(backupsDir)) { + fs.mkdirSync(backupsDir, { recursive: true }); + } + const args = [ "serve", "webdav", - APP_ROOT, + backupsDir, "--addr", `:${config.serverPort}`, "--user", RCLONE_USER, "--pass", RCLONE_PASS, diff --git a/apps/Backend/src/storage/twilio-storage.ts b/apps/Backend/src/storage/twilio-storage.ts index e165c203..b0cabcba 100644 --- a/apps/Backend/src/storage/twilio-storage.ts +++ b/apps/Backend/src/storage/twilio-storage.ts @@ -5,7 +5,7 @@ import { prisma as db } from "@repo/db/client"; export const DEFAULT_GREETING = "Hi {firstName}, this is Lisa, the AI assistant at {officeName}. How can I help you today?"; export const DEFAULT_RECALL_GREETING = - "Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning, which most insurance plans cover at no cost. Would you like to schedule one?"; + "Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning. Would you like to make an appointment because your insurance covers it?"; export type TwilioSettingsData = { accountSid: string; diff --git a/apps/Frontend/src/components/settings/ai-chat-settings-card.tsx b/apps/Frontend/src/components/settings/ai-chat-settings-card.tsx index 5f629795..f3a65fee 100644 --- a/apps/Frontend/src/components/settings/ai-chat-settings-card.tsx +++ b/apps/Frontend/src/components/settings/ai-chat-settings-card.tsx @@ -53,7 +53,7 @@ const CALL_DEFAULTS = { newPatientGreeting: "No problem, let's get you set up.", generalFallback: "Sure, how can I help?", recallGreeting: - "Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning, which most insurance plans cover at no cost. Would you like to schedule one?", + "Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning. Would you like to make an appointment because your insurance covers it?", }; const DEFAULT_SMS_TEMPLATES = [