feat: add AI-driven phone calls to patient connection

Adds an "AI Call" option alongside Call/SMS/Chat that places an outbound
call where Lisa converses live with the patient via Twilio speech
gather + TTS, using the same conversational AI as chat. Includes a
separate, user-editable Call Template (Settings > AI Chat/Call Settings)
so the call greeting can be customized independently of SMS templates.

Also fixes the outbound-call webhook URL to always use the fixed public
Twilio hostname (CLOUDFLARE_HOST) instead of deriving it from the
triggering request, since staff-browser requests arrive over the
LAN-only hostname, which Twilio can never reach.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ff
2026-07-31 20:14:09 -04:00
parent ed5a8af835
commit 089e94ba88
9 changed files with 364 additions and 6 deletions

View File

@@ -0,0 +1,65 @@
import { getLlm, type AiProvider } from "./llm-factory";
export interface VoiceTurn {
role: "assistant" | "user";
text: string;
}
export interface VoiceAssistantContext {
firstName: string;
officeName: string;
officeAddress: string;
officePhone: string;
appointmentDatetime: string;
}
const CLOSING_PATTERNS =
/\b(bye|goodbye|good bye|that'?s all|nothing else|no thanks|no thank you|i'?m good|that'?s it|hang up)\b/i;
export function soundsLikeGoodbye(text: string): boolean {
return CLOSING_PATTERNS.test(text.trim());
}
/**
* Generate Lisa's next spoken reply given the call transcript so far.
* Falls back to a plain closing line if the LLM call fails, so a broken
* key never leaves the caller stuck mid-call.
*/
export async function runVoiceAssistantTurn(
history: VoiceTurn[],
ctx: VoiceAssistantContext,
apiKey: string,
provider: AiProvider = "google",
model?: string
): Promise<string> {
const fallback =
"I'm sorry, I'm having trouble right now — our staff will follow up with you shortly. Thank you for calling, goodbye.";
try {
const llm = getLlm(provider, apiKey, model);
const system = [
`You are Lisa, a friendly AI phone assistant for ${ctx.officeName || "a dental office"}.`,
`You are speaking live by phone with ${ctx.firstName || "the patient"}.`,
ctx.appointmentDatetime
? `Their next appointment is on ${ctx.appointmentDatetime}.`
: `They have no upcoming appointment on file.`,
ctx.officeAddress ? `Office address: ${ctx.officeAddress}.` : "",
ctx.officePhone ? `Office phone: ${ctx.officePhone}.` : "",
"You can confirm their appointment and answer general questions about the practice.",
"If they want to reschedule or need something you can't resolve yourself, tell them a staff member will call or text them to confirm — never claim you rebooked or changed anything yourself.",
"This reply will be read aloud by text-to-speech: keep it conversational and SHORT, 1-2 sentences, no formatting, no lists.",
"If they indicate they're done (goodbye, that's all, etc.), give a brief warm goodbye.",
].filter(Boolean).join(" ");
const messages = [
{ role: "system", content: system },
...history.map((t) => ({ role: t.role === "assistant" ? "assistant" : "user", content: t.text })),
];
const res = await llm.invoke(messages as any);
const text = String(res.content).trim();
return text || fallback;
} catch {
return fallback;
}
}

View File

@@ -214,8 +214,8 @@ router.put("/chat-templates", async (req: Request, res: Response): Promise<any>
try {
const userId = req.user?.id;
if (!userId) return res.status(401).json({ message: "Unauthorized" });
const { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms } = req.body;
await storage.saveAiChatTemplates(userId, { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms });
const { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms, callTemplate } = req.body;
await storage.saveAiChatTemplates(userId, { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms, callTemplate });
const updated = await storage.getAiChatTemplates(userId);
return res.status(200).json(updated);
} catch (err) {

View File

@@ -14,6 +14,8 @@ import {
timeLabel,
} from "../ai/reschedule-graph";
import { getLlm, resolveAiProvider } from "../ai/llm-factory";
import { runVoiceAssistantTurn, soundsLikeGoodbye } from "../ai/voice-assistant";
import { getPublicBaseUrl } from "../utils/publicUrl";
import { runEligibilityProcessor } from "../queue/processors/eligibilityProcessor";
import {
getHandoff, getAfterHoursHandoff,
@@ -1169,7 +1171,7 @@ router.post("/webhook/voice", async (req: Request, res: Response): Promise<any>
});
}
const recordingCallbackUrl = `${process.env.BASE_URL || "https://communitydentistsoflowell.mydentalofficemanagement.com"}/api/twilio/webhook/voice-recording`;
const recordingCallbackUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/voice-recording`;
res.set("Content-Type", "text/xml");
return res.send(`<?xml version="1.0" encoding="UTF-8"?>
@@ -1228,4 +1230,144 @@ router.post("/webhook/voice-browser", async (req: Request, res: Response): Promi
}
});
// ── AI phone call (outbound "AI Call" from Patient Connection) ───────────────
const MAX_AI_CALL_TURNS = 12;
interface AiCallSession {
patientId: number;
userId: number;
history: { role: "assistant" | "user"; text: string }[];
turns: number;
}
const aiCallSessions = new Map<string, AiCallSession>();
function aiVoiceTwiml(sayText: string, gatherActionUrl: string): string {
// action is always set, so Twilio posts back to it on both speech and
// silence timeout — nothing after </Gather> would ever run.
return `<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Gather input="speech" language="en-US" speechTimeout="auto" action="${gatherActionUrl}" method="POST">
<Say voice="alice">${escapeXml(sayText)}</Say>
</Gather>
</Response>`;
}
function aiVoiceHangup(sayText: string): string {
return `<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Say voice="alice">${escapeXml(sayText)}</Say>
<Hangup/>
</Response>`;
}
// POST /api/twilio/webhook/ai-voice
// Entry point + conversation loop for outbound AI-driven calls placed via
// POST /api/twilio/make-ai-call. Twilio hits this once when the call connects
// (no CallSid session yet) and again after every <Gather> with SpeechResult.
router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<any> => {
res.set("Content-Type", "text/xml");
try {
const { CallSid, SpeechResult } = req.body as { CallSid?: string; SpeechResult?: string };
const patientIdParam = req.query.patientId as string | undefined;
if (!CallSid) return res.send(aiVoiceHangup("Sorry, something went wrong. Goodbye."));
let session = aiCallSessions.get(CallSid);
// ── First hit for this call: set up session + greet ──────────────────
if (!session) {
const patientId = parseInt(patientIdParam || "", 10);
const patient = !isNaN(patientId)
? await db.patient.findUnique({ where: { id: patientId }, select: { id: true, userId: true, firstName: true } })
: null;
if (!patient) return res.send(aiVoiceHangup("Sorry, we could not find your patient record. Goodbye."));
session = { patientId: patient.id, userId: patient.userId, history: [], turns: 0 };
aiCallSessions.set(CallSid, session);
const officeContact = await storage.getOfficeContact(patient.userId);
const officeName = (officeContact as any)?.officeName?.trim() || "";
const officeAddress = [
(officeContact as any)?.streetAddress?.trim(),
(officeContact as any)?.city?.trim(),
(officeContact as any)?.state?.trim(),
(officeContact as any)?.zipCode?.trim(),
].filter(Boolean).join(", ");
const officePhone = (officeContact as any)?.phoneNumber?.trim() || "";
const appointmentDatetime = await getAppointmentDatetime(patient.id);
const chatTemplates = await storage.getAiChatTemplates(patient.userId);
const rawCallTemplate = chatTemplates.callTemplate?.trim() ||
"Hi {firstName}, this is Lisa, the AI assistant at {officeName}. How can I help you today?";
const greeting = rawCallTemplate
.replace(/\{firstName\}/g, patient.firstName || "there")
.replace(/\{officeName\}/g, officeName || "our office");
session.history.push({ role: "assistant", text: greeting });
await storage.createCommunication({
patientId: patient.id, userId: patient.userId, channel: "voice",
direction: "outbound", status: "completed", body: greeting, twilioSid: CallSid,
});
// Stash office context on the session for later turns
(session as any).ctx = { officeName, officeAddress, officePhone, appointmentDatetime };
const actionUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/ai-voice?patientId=${patient.id}`;
return res.send(aiVoiceTwiml(greeting, actionUrl));
}
// ── Caller stayed silent past the speech timeout ──────────────────────
if (!SpeechResult?.trim()) {
aiCallSessions.delete(CallSid);
return res.send(aiVoiceHangup("I didn't catch a response. Our staff will follow up with you if needed. Goodbye."));
}
session.history.push({ role: "user", text: SpeechResult.trim() });
await storage.createCommunication({
patientId: session.patientId, userId: session.userId, channel: "voice",
direction: "inbound", status: "completed", body: SpeechResult.trim(), twilioSid: CallSid,
});
session.turns++;
// ── Caller said something that sounds like goodbye, or we've hit the cap ──
if (soundsLikeGoodbye(SpeechResult) || session.turns >= MAX_AI_CALL_TURNS) {
aiCallSessions.delete(CallSid);
return res.send(aiVoiceHangup("Thank you for calling, have a great day. Goodbye."));
}
const aiSettings = await storage.getAiSettings(session.userId);
const activeAi = resolveAiProvider(aiSettings ?? {});
if (!activeAi) {
aiCallSessions.delete(CallSid);
return res.send(aiVoiceHangup("Our AI assistant is not available right now. Our staff will follow up with you. Goodbye."));
}
const patientRow = await db.patient.findUnique({ where: { id: session.patientId }, select: { firstName: true } });
const ctx = (session as any).ctx || {};
const reply = await runVoiceAssistantTurn(
session.history,
{ firstName: patientRow?.firstName || "", ...ctx },
activeAi.key, activeAi.provider, activeAi.model
);
session.history.push({ role: "assistant", text: reply });
await storage.createCommunication({
patientId: session.patientId, userId: session.userId, channel: "voice",
direction: "outbound", status: "completed", body: reply, twilioSid: CallSid,
});
if (soundsLikeGoodbye(reply)) {
aiCallSessions.delete(CallSid);
return res.send(aiVoiceHangup(reply));
}
const actionUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/ai-voice?patientId=${session.patientId}`;
return res.send(aiVoiceTwiml(reply, actionUrl));
} catch (err) {
return res.send(aiVoiceHangup("Sorry, something went wrong on our end. Goodbye."));
}
});
export default router;

View File

@@ -3,6 +3,8 @@ import twilio from "twilio";
import { storage } from "../storage";
import { prisma as db } from "@repo/db/client";
import { getHandoff, setHandoff, resetConversation, startNewPatientConversation, startRescheduleConversation, getAfterHoursHandoff, setAfterHoursHandoff } from "../ai/aiHandoffStore";
import { resolveAiProvider } from "../ai/llm-factory";
import { getTwilioPublicBaseUrl } from "../utils/publicUrl";
const router = express.Router();
@@ -117,6 +119,52 @@ router.post("/send-sms", async (req: Request, res: Response): Promise<any> => {
}
});
// POST /api/twilio/make-ai-call
// Places an outbound call where Lisa (the AI assistant) conducts the whole
// conversation live over the phone — see /webhook/ai-voice for the loop.
router.post("/make-ai-call", async (req: Request, res: Response): Promise<any> => {
try {
const userId = req.user?.id;
if (!userId) return res.status(401).json({ message: "Unauthorized" });
const { to, patientId } = req.body;
if (!to || !patientId) return res.status(400).json({ message: "to and patientId are required" });
const settings = await storage.getTwilioSettings(userId);
if (!settings) {
return res.status(400).json({ message: "Twilio is not configured. Please add your Twilio credentials in Settings." });
}
const aiSettings = await storage.getAiSettings(userId);
if (!resolveAiProvider(aiSettings ?? {})) {
return res.status(400).json({ message: "AI is not configured. Please add an AI provider key in Settings before placing an AI call." });
}
const patient = await db.patient.findFirst({ where: { id: Number(patientId), userId } });
if (!patient) return res.status(404).json({ message: "Patient not found" });
let publicBaseUrl: string;
try {
publicBaseUrl = getTwilioPublicBaseUrl();
} catch (err: any) {
return res.status(500).json({ message: err.message });
}
const webhookUrl = `${publicBaseUrl}/api/twilio/webhook/ai-voice?patientId=${patient.id}`;
const client = getTwilioClient(settings.accountSid, settings.authToken);
const call = await client.calls.create({
url: webhookUrl,
method: "POST",
from: settings.phoneNumber,
to,
});
return res.status(200).json({ sid: call.sid, status: call.status });
} catch (err: any) {
return res.status(500).json({ error: err.message || "Failed to place AI call" });
}
});
// POST /api/twilio/send-reminders-batch
router.post("/send-reminders-batch", async (req: Request, res: Response): Promise<any> => {
try {

View File

@@ -77,10 +77,11 @@ export const twilioStorage = {
generalFallback: all["_ai_chat_general_fallback"] ?? "",
rescheduleGreeting: all["_ai_chat_reschedule_greeting"] ?? "",
reminderSms: all["_ai_chat_reminder_sms"] ?? "",
callTemplate: all["_ai_chat_call_template"] ?? "",
};
},
async saveAiChatTemplates(userId: number, templates: { reminderGreeting?: string; newPatientGreeting?: string; generalFallback?: string; rescheduleGreeting?: string; reminderSms?: string }) {
async saveAiChatTemplates(userId: number, templates: { reminderGreeting?: string; newPatientGreeting?: string; generalFallback?: string; rescheduleGreeting?: string; reminderSms?: string; callTemplate?: string }) {
const settings = await db.twilioSettings.findUnique({ where: { userId } });
const existing = (settings?.templates as Record<string, string>) || {};
const updated: Record<string, string> = { ...existing };
@@ -89,6 +90,7 @@ export const twilioStorage = {
if (templates.generalFallback !== undefined) updated["_ai_chat_general_fallback"] = templates.generalFallback;
if (templates.rescheduleGreeting !== undefined) updated["_ai_chat_reschedule_greeting"] = templates.rescheduleGreeting;
if (templates.reminderSms !== undefined) updated["_ai_chat_reminder_sms"] = templates.reminderSms;
if (templates.callTemplate !== undefined) updated["_ai_chat_call_template"] = templates.callTemplate;
return db.twilioSettings.upsert({
where: { userId },
update: { templates: updated },

View File

@@ -0,0 +1,33 @@
import type { Request } from "express";
/**
* Public HTTPS origin this request arrived on (e.g. https://broadwaydental.mydentalofficemanagement.com).
* Derived from the Host header rather than an env var so it's automatically correct
* per-office — each office's nginx forwards its own public hostname in Host,
* and Cloudflare/Let's Encrypt terminate TLS in front of it either way.
*
* Only valid for requests that actually arrived via the public tunnel (i.e. inside
* a Twilio webhook handler). For anything triggered from the staff browser — which
* reaches the backend over the LAN-only local-* hostname — use getTwilioPublicBaseUrl()
* instead, since Twilio can never reach that LAN hostname to fetch a callback URL.
*/
export function getPublicBaseUrl(req: Request): string {
return `https://${req.get("host")}`;
}
/**
* Fixed public origin Twilio must use to call back into this office's server
* (e.g. for outbound-call TwiML webhooks). Comes from CLOUDFLARE_HOST, the
* per-office env var already reserved for this — not derived from the
* triggering request, since that request may have arrived over the LAN-only
* hostname, which Twilio cannot reach.
*/
export function getTwilioPublicBaseUrl(): string {
const host = process.env.CLOUDFLARE_HOST?.trim();
if (!host) {
throw new Error(
"CLOUDFLARE_HOST is not set — required to build a Twilio-reachable callback URL. Set it in .env to this office's public tunnel hostname."
);
}
return `https://${host}`;
}