fix: AI recall-call flow, speech-to-text time parsing, and stale conversation state

- Reset conversation stage to a fresh start on every new voice call
  (recall and reminder alike), and on every hangup/timeout/goodbye,
  so an abandoned or completed call never leaves a patient stuck
  mid-flow for their next call or text.
- Add a staleness check to the SMS flow: after a long idle gap,
  check whether the patient's reply still reads as a continuation
  before resuming the stored stage, falling back to a friendly
  "it's been a while" re-greeting when unsure.
- Fix parseTime() to handle Twilio's speech-to-text output, which
  renders "pm"/"am" with periods and often drops the colon in a
  spoken time (e.g. "100 p.m." -> 1:00 PM) — this was silently
  failing and causing the AI to loop asking for a time it had
  already been given.
- Extend the voice Gather timeout so the AI doesn't hang up on a
  patient who just needs a few seconds to respond.
- Reword the recall-call opening line and rework the post-"yes"
  prompt to ask for day and time together.
- Fix rclone WebDAV server to serve only the backups folder instead
  of the entire backend app directory (was exposing .env and source).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ff
2026-08-06 09:35:48 -04:00
co-authored by Claude Sonnet 5
parent 55ce52f276
commit 0cd095330a
7 changed files with 167 additions and 34 deletions
@@ -2,7 +2,7 @@ import twilio from "twilio";
import { prisma as db } from "@repo/db/client";
import { storage } from "../storage";
import { runReminderGraph } from "./reminder-graph";
import { runRecallGraph, runRecallBookingStep } from "./recall-graph";
import { runRecallGraph, runRecallBookingStep, hasDateOrTimeSignal } from "./recall-graph";
import { runNewPatientStep } from "./new-patient-graph";
import {
runRescheduleStep,
@@ -290,10 +290,16 @@ export async function routePatientTurn(params: {
if (intent === "yes") {
// Patient may have already given a date (or date + time) in this same
// reply ("Tuesday 1pm") — try to book right away instead of asking again.
const { reply: bookingReply, nextStage: bookingNextStage } = await runRecallBookingStep(
message, "recall_asked_datetime", language, patient.id, patient.userId, activeAi.key, activeAi.provider, activeAi.model
);
return finish(patient.userId, patient.id, [bookingReply], bookingNextStage);
// A bare "yes"/"ok" has nothing to parse, so skip straight to asking for
// day + time instead of wasting a parse call that used to come back
// empty and send a mismatched "Sorry, I didn't catch a day" reply.
if (hasDateOrTimeSignal(message)) {
const { reply: bookingReply, nextStage: bookingNextStage } = await runRecallBookingStep(
message, "recall_asked_datetime", language, patient.id, patient.userId, activeAi.key, activeAi.provider, activeAi.model
);
return finish(patient.userId, patient.id, [bookingReply], bookingNextStage);
}
return finish(patient.userId, patient.id, [intentReply], "recall_asked_datetime");
}
if (intent === "no") {
return finish(patient.userId, patient.id, [intentReply], "done");
+22 -12
View File
@@ -23,6 +23,19 @@ type GraphStateType = typeof GraphState.State;
// ── Intent classifier — multilingual yes/no keywords (shared shape with reminder-graph) ──
/** True if `text` mentions a concrete day/date/time (e.g. "Tuesday", "3/10", "2pm").
* Shared by classifyNode (a bare "yes" vs. "yes, Tuesday at 1pm" both count as
* intent "yes") and by the caller in patient-conversation-router.ts, which uses
* it to decide whether to attempt an immediate booking parse or just ask for a
* day/time first. */
export function hasDateOrTimeSignal(text: string): boolean {
return (
/\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|today|tomorrow|next week)\b/i.test(text) ||
/\b\d{1,2}[\/\-]\d{1,2}\b/.test(text) ||
/\b\d{1,2}(:\d{2})?\s*(am|pm|a\.m\.|p\.m\.)\b/i.test(text)
);
}
function classifyNode(state: GraphStateType) {
const text = state.message.toLowerCase().trim();
@@ -32,10 +45,7 @@ function classifyNode(state: GraphStateType) {
// A specific day/date/time, or an appointment-request phrase, is itself a
// strong signal the patient wants to book — real replies rarely say a bare
// "yes", they jump straight to "Tuesday at 1pm" or "I want an appointment".
const mentionsDateOrTime =
/\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|tomorrow|next week)\b/i.test(text) ||
/\b\d{1,2}[\/\-]\d{1,2}\b/.test(text) ||
/\b\d{1,2}(:\d{2})?\s*(am|pm|a\.m\.|p\.m\.)\b/i.test(text);
const mentionsDateOrTime = hasDateOrTimeSignal(text);
const mentionsAppointment = /\b(appointment|schedule|book|come in|check.?up|cleaning)\b/i.test(text);
if (noPatterns.test(text) && !mentionsDateOrTime) return { intent: "no" };
@@ -52,13 +62,13 @@ function routeByIntent(state: GraphStateType): string {
// ── Fallbacks ──────────────────────────────────────────────────────────────
const WANTS_BOOKING_FALLBACKS: Record<string, string> = {
English: "Great! What day works best for you?",
Spanish: "¡Genial! ¿Qué día le viene mejor?",
Portuguese: "Ótimo! Qual dia é melhor para você?",
Mandarin: "太好了!您哪天方便?",
Cantonese: "太好喇!您邊日方便?",
Arabic: "رائع! ما هو اليوم الأنسب لك؟",
"Haitian Creole": "Bon! Ki jou ki pi bon pou ou?",
English: "Great! What day and time work best for you?",
Spanish: "¡Genial! ¿Qué día y hora le vienen mejor?",
Portuguese: "Ótimo! Qual dia e horário são melhores para você?",
Mandarin: "太好了!您哪天、几点方便?",
Cantonese: "太好喇!您邊日、幾點方便?",
Arabic: "رائع! ما هو اليوم والوقت الأنسب لك؟",
"Haitian Creole": "Bon! Ki jou ak ki lè ki pi bon pou ou?",
};
const DECLINES_FALLBACKS: Record<string, string> = {
@@ -98,7 +108,7 @@ async function wantsBookingNode(state: GraphStateType, config: any) {
{
role: "system",
content:
`You are a friendly dental office assistant. The patient agreed to come in for a routine exam and cleaning. Write a short, warm reply (1-2 sentences max) and ask what day works best for them. Do NOT introduce yourself or say your name — the introduction has already been sent in a separate message. You MUST reply in ${lang}. No formatting, no extra text.`,
`You are a friendly dental office assistant. The patient agreed to come in for a routine exam and cleaning. Write a short, warm reply (1-2 sentences max) and ask what day AND time work best for them. Do NOT introduce yourself or say your name — the introduction has already been sent in a separate message. You MUST reply in ${lang}. No formatting, no extra text.`,
},
{ role: "user", content: `Patient replied: "${state.message}"` },
]);
+20 -3
View File
@@ -392,18 +392,35 @@ async function isSlotAvailable(
// ── Time parsing (legacy) ─────────────────────────────────────────────────────
export async function parseTime(message: string, apiKey: string, provider: AiProvider = "google", model?: string): Promise<string | null> {
const t = message.toLowerCase();
// Twilio speech-to-text usually renders "pm"/"am" with periods ("p.m."),
// and often drops the colon in a spoken time too ("1:00pm" -> "100 pm",
// "12:30pm" -> "1230 pm") — normalize both before matching.
const t = message.toLowerCase()
.replace(/\ba\.m\.?/g, "am")
.replace(/\bp\.m\.?/g, "pm");
if (/\bmorning\b|mañana|manhã|上午|صباح|maten/i.test(t)) return "09:00";
if (/\bafternoon\b|tarde|après-midi|下午|مساء|aprèmidi/i.test(t)) return "13:00";
const ampm = t.match(/\b(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/);
const ampm = t.match(/\b(\d{1,2}):([0-5]\d)\s*(am|pm)\b/);
const glued = !ampm ? t.match(/\b(\d{1,4})\s*(am|pm)\b/) : null;
const clock = t.match(/\b([01]?\d|2[0-3]):([0-5]\d)\b/);
if (ampm) {
let h = parseInt(ampm[1]!);
const m = ampm[2] ? parseInt(ampm[2]) : 0;
const m = parseInt(ampm[2]!);
if (ampm[3] === "pm" && h < 12) h += 12;
if (ampm[3] === "am" && h === 12) h = 0;
return `${String(h).padStart(2,"0")}:${String(m).padStart(2,"0")}`;
}
if (glued) {
const digits = glued[1]!;
let h: number, m: number;
if (digits.length <= 2) { h = parseInt(digits); m = 0; }
else { h = parseInt(digits.slice(0, -2)); m = parseInt(digits.slice(-2)); }
if (h >= 1 && h <= 12 && m <= 59) {
if (glued[2] === "pm" && h < 12) h += 12;
if (glued[2] === "am" && h === 12) h = 0;
return `${String(h).padStart(2,"0")}:${String(m).padStart(2,"0")}`;
}
}
if (clock) return clock[0]!;
try {
const llm = getLlm(provider, apiKey, model);
+104 -11
View File
@@ -12,7 +12,7 @@ import {
getOfficeHoursDisplay,
timeLabel,
} from "../ai/reschedule-graph";
import { getLlm, resolveAiProvider } from "../ai/llm-factory";
import { getLlm, resolveAiProvider, type AiProvider } from "../ai/llm-factory";
import { soundsLikeGoodbye } from "../ai/voice-assistant";
import { getPublicBaseUrl } from "../utils/publicUrl";
import {
@@ -116,6 +116,64 @@ const NEW_OR_RESCHEDULE_Q: Record<string, string> = {
"Haitian Creole": "Jis pou konfime — èske ou ta renmen pran yon nouvo randevou, oswa èske ou ta renmen reprogramè randevou aktyèl ou?",
};
// ── Stale-conversation handling (SMS) ─────────────────────────────────────────
// Unlike a phone call, a long gap between texts doesn't necessarily mean the
// patient walked away — replies are often just slow. So instead of hard
// resetting on any idle gap, we only second-guess the stored stage once the
// gap crosses STALE_SMS_THRESHOLD_MS, and even then we ask the LLM to look at
// the recent history before deciding whether the new text is still answering
// the last question or reads like a fresh, unrelated conversation.
const STALE_SMS_THRESHOLD_MS = 3 * 60 * 60 * 1000; // 3 hours
const STILL_HERE_MSGS: Record<string, string> = {
English: "It's been a while since our last message. How can I help you today?",
Spanish: "Ha pasado un tiempo desde nuestro último mensaje. ¿Cómo puedo ayudarle hoy?",
Portuguese: "Já faz um tempo desde nossa última mensagem. Como posso ajudá-lo hoje?",
Mandarin: "距离我们上次的消息已经有一段时间了。今天我能帮您什么?",
Cantonese: "距離我們上次嘅消息已經有一段時間喇。今日我可以點幫您?",
Arabic: "لقد مر بعض الوقت منذ آخر رسالة لنا. كيف يمكنني مساعدتك اليوم؟",
"Haitian Creole": "Sa gen yon ti tan depi dènye mesaj nou. Kijan mwen ka ede ou jodi a?",
};
/**
* After a long idle gap, ask the LLM whether the patient's new text still
* answers what was last asked (recent history + stage in context) or reads
* like an unrelated, fresh message. Defaults to "continue" on any failure —
* an LLM call error shouldn't be the reason a legitimate reply gets bounced.
*/
async function isLikelyContinuation(
recentHistory: { role: "user" | "assistant"; text: string }[],
newMessage: string,
stage: ConversationStage,
apiKey: string,
provider: AiProvider,
model?: string,
): Promise<boolean> {
if (recentHistory.length === 0) return true;
try {
const llm = getLlm(provider, apiKey, model);
const transcript = recentHistory
.map(m => `${m.role === "assistant" ? "Office" : "Patient"}: ${m.text}`)
.join("\n");
const res = await llm.invoke([
{
role: "system",
content:
`You are helping a dental office text-messaging assistant resume a conversation with a patient after a long gap of silence. ` +
`Here is the recent conversation (oldest first):\n${transcript}\n\n` +
`The assistant is currently waiting on stage "${stage}" of a booking flow. ` +
`The patient just sent a new message after the long gap: "${newMessage}"\n\n` +
`Does this new message still look like it's answering/continuing that same conversation, or does it read like an unrelated, fresh message (e.g. a new question, a greeting, a different topic)? ` +
`Reply with ONLY one word: "continue" or "restart".`,
},
]);
return !/restart/i.test(String(res.content));
} catch {
return true;
}
}
// ── POST /api/twilio/webhook/sms ──────────────────────────────────────────────
const CONVO_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
@@ -427,16 +485,33 @@ router.post("/webhook/sms", async (req: Request, res: Response): Promise<any> =>
let stage = await getStage(patient.userId, patient.id);
const chatTemplates = await storage.getAiChatTemplates(patient.userId);
// Reset conversation if idle for more than 5 minutes
// Long idle gap: don't blindly resume (SMS replies are often just slow),
// but don't blindly reset either — check whether the new text still reads
// like a continuation before deciding.
if (stage !== "initial" && stage !== "done") {
const convRow = await db.patientConversation.findUnique({
where: { patientId: patient.id },
select: { updatedAt: true },
});
if (convRow?.updatedAt && Date.now() - convRow.updatedAt.getTime() > CONVO_TIMEOUT_MS) {
clearPendingReschedule(patient.userId, patient.id);
await setStage(patient.userId, patient.id, "initial");
stage = "initial";
if (convRow?.updatedAt && Date.now() - convRow.updatedAt.getTime() > STALE_SMS_THRESHOLD_MS) {
const recentComms = await storage.getCommunicationsByPatient(patient.id);
const recentHistory = recentComms.slice(-5).map((c: any) => ({
role: (c.direction === "outbound" ? "assistant" : "user") as "assistant" | "user",
text: c.body || "",
}));
const continuing = await isLikelyContinuation(
recentHistory, Body, stage, activeAi.key, activeAi.provider, activeAi.model
);
if (!continuing) {
clearPendingReschedule(patient.userId, patient.id);
await setStage(patient.userId, patient.id, "initial");
stage = "initial";
const stillHereText = STILL_HERE_MSGS[language] ?? STILL_HERE_MSGS["English"]!;
await saveOutbound(patient.id, stillHereText);
res.set("Content-Type", "text/xml");
return res.send(twimlMessages(stillHereText));
}
// Looks like a continuation — fall through and resume the stored stage.
}
}
const officeContact = await storage.getOfficeContact(patient.userId);
@@ -690,7 +765,7 @@ function aiVoiceTwiml(sayText: string, gatherActionUrl: string, locale: VoiceLoc
// silence timeout — nothing after </Gather> would ever run.
return `<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Gather input="speech" language="${locale.gatherLang}" speechTimeout="auto" action="${gatherActionUrl}" method="POST">
<Gather input="speech" language="${locale.gatherLang}" speechTimeout="auto" timeout="15" action="${gatherActionUrl}" method="POST">
<Say voice="${locale.sayVoice}">${escapeXml(sayText)}</Say>
</Gather>
</Response>`;
@@ -741,10 +816,16 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<an
// A "recall" call (staff-triggered from Patient Connection, offering an
// existing patient a routine exam & cleaning) seeds a different stage
// than the default reminder-style check-in.
const currentStage = await getStage(patient.userId, patient.id);
if (currentStage === "initial" || currentStage === "done") {
if (isRecallCall) await startRecallConversation(patient.userId, patient.id);
else await resetConversation(patient.userId, patient.id);
// Either kind of call always speaks its own pitch fresh (see greeting
// below), so it must always reset the stage — even if a prior call of
// either kind left the patient stuck mid-flow (e.g. "recall_asked_datetime"
// or "asked_reschedule_datetime"). Otherwise the reply below gets
// interpreted against that stale leftover stage instead of the fresh
// pitch that was just spoken.
if (isRecallCall) {
await startRecallConversation(patient.userId, patient.id);
} else {
await resetConversation(patient.userId, patient.id);
}
const officeContact = await storage.getOfficeContact(patient.userId);
@@ -779,8 +860,12 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<an
const locale = voiceLocaleFor(session.language);
// ── Caller stayed silent past the speech timeout ──────────────────────
// A call is a single continuous conversation, unlike SMS — if it ends
// here, mid-flow or not, there's nothing left to resume, so reset the
// stage now instead of leaving the patient stuck for their next call/text.
if (!SpeechResult?.trim()) {
aiCallSessions.delete(CallSid);
await setStage(session.userId, session.patientId, "done");
return res.send(aiVoiceHangup("I didn't catch a response. Our staff will follow up with you if needed. Goodbye.", locale));
}
@@ -795,6 +880,7 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<an
// ── Caller said something that sounds like goodbye, or we've hit the cap ──
if (soundsLikeGoodbye(SpeechResult) || session.turns >= MAX_AI_CALL_TURNS) {
aiCallSessions.delete(CallSid);
await setStage(session.userId, session.patientId, "done");
return res.send(aiVoiceHangup("Thank you for calling, have a great day. Goodbye.", locale));
}
@@ -802,6 +888,7 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<an
const activeAi = resolveAiProvider(aiSettings ?? {});
if (!activeAi) {
aiCallSessions.delete(CallSid);
await setStage(session.userId, session.patientId, "done");
return res.send(aiVoiceHangup("Our AI assistant is not available right now. Our staff will follow up with you. Goodbye.", locale));
}
@@ -826,6 +913,12 @@ router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<an
if (routed.nextStage === "done" || soundsLikeGoodbye(reply)) {
aiCallSessions.delete(CallSid);
// routePatientTurn() already persisted nextStage — but if we're hanging
// up because the reply sounded like a goodbye while nextStage was still
// mid-flow, force it to "done" so the next call/text doesn't resume there.
if (routed.nextStage !== "done") {
await setStage(session.userId, session.patientId, "done");
}
return res.send(aiVoiceHangup(reply, locale));
}
+8 -1
View File
@@ -46,9 +46,16 @@ export async function startWebDavServer(): Promise<void> {
const config = readRcloneConfig();
// Serve only the backups folder — never the app root, which contains
// .env (DB/Twilio/AI credentials) and the full source tree.
const backupsDir = path.join(APP_ROOT, "backups");
if (!fs.existsSync(backupsDir)) {
fs.mkdirSync(backupsDir, { recursive: true });
}
const args = [
"serve", "webdav",
APP_ROOT,
backupsDir,
"--addr", `:${config.serverPort}`,
"--user", RCLONE_USER,
"--pass", RCLONE_PASS,
+1 -1
View File
@@ -5,7 +5,7 @@ import { prisma as db } from "@repo/db/client";
export const DEFAULT_GREETING =
"Hi {firstName}, this is Lisa, the AI assistant at {officeName}. How can I help you today?";
export const DEFAULT_RECALL_GREETING =
"Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning, which most insurance plans cover at no cost. Would you like to schedule one?";
"Hi {firstName}, this is Lisa from {officeName}. We wanted to check in — you're due for a routine exam and cleaning. Would you like to make an appointment because your insurance covers it?";
export type TwilioSettingsData = {
accountSid: string;