feat: add AI-driven phone calls to patient connection
Adds an "AI Call" option alongside Call/SMS/Chat that places an outbound call where Lisa converses live with the patient via Twilio speech gather + TTS, using the same conversational AI as chat. Includes a separate, user-editable Call Template (Settings > AI Chat/Call Settings) so the call greeting can be customized independently of SMS templates. Also fixes the outbound-call webhook URL to always use the fixed public Twilio hostname (CLOUDFLARE_HOST) instead of deriving it from the triggering request, since staff-browser requests arrive over the LAN-only hostname, which Twilio can never reach. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
65
apps/Backend/src/ai/voice-assistant.ts
Normal file
65
apps/Backend/src/ai/voice-assistant.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { getLlm, type AiProvider } from "./llm-factory";
|
||||
|
||||
export interface VoiceTurn {
|
||||
role: "assistant" | "user";
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface VoiceAssistantContext {
|
||||
firstName: string;
|
||||
officeName: string;
|
||||
officeAddress: string;
|
||||
officePhone: string;
|
||||
appointmentDatetime: string;
|
||||
}
|
||||
|
||||
const CLOSING_PATTERNS =
|
||||
/\b(bye|goodbye|good bye|that'?s all|nothing else|no thanks|no thank you|i'?m good|that'?s it|hang up)\b/i;
|
||||
|
||||
export function soundsLikeGoodbye(text: string): boolean {
|
||||
return CLOSING_PATTERNS.test(text.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Lisa's next spoken reply given the call transcript so far.
|
||||
* Falls back to a plain closing line if the LLM call fails, so a broken
|
||||
* key never leaves the caller stuck mid-call.
|
||||
*/
|
||||
export async function runVoiceAssistantTurn(
|
||||
history: VoiceTurn[],
|
||||
ctx: VoiceAssistantContext,
|
||||
apiKey: string,
|
||||
provider: AiProvider = "google",
|
||||
model?: string
|
||||
): Promise<string> {
|
||||
const fallback =
|
||||
"I'm sorry, I'm having trouble right now — our staff will follow up with you shortly. Thank you for calling, goodbye.";
|
||||
|
||||
try {
|
||||
const llm = getLlm(provider, apiKey, model);
|
||||
const system = [
|
||||
`You are Lisa, a friendly AI phone assistant for ${ctx.officeName || "a dental office"}.`,
|
||||
`You are speaking live by phone with ${ctx.firstName || "the patient"}.`,
|
||||
ctx.appointmentDatetime
|
||||
? `Their next appointment is on ${ctx.appointmentDatetime}.`
|
||||
: `They have no upcoming appointment on file.`,
|
||||
ctx.officeAddress ? `Office address: ${ctx.officeAddress}.` : "",
|
||||
ctx.officePhone ? `Office phone: ${ctx.officePhone}.` : "",
|
||||
"You can confirm their appointment and answer general questions about the practice.",
|
||||
"If they want to reschedule or need something you can't resolve yourself, tell them a staff member will call or text them to confirm — never claim you rebooked or changed anything yourself.",
|
||||
"This reply will be read aloud by text-to-speech: keep it conversational and SHORT, 1-2 sentences, no formatting, no lists.",
|
||||
"If they indicate they're done (goodbye, that's all, etc.), give a brief warm goodbye.",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: system },
|
||||
...history.map((t) => ({ role: t.role === "assistant" ? "assistant" : "user", content: t.text })),
|
||||
];
|
||||
|
||||
const res = await llm.invoke(messages as any);
|
||||
const text = String(res.content).trim();
|
||||
return text || fallback;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
@@ -214,8 +214,8 @@ router.put("/chat-templates", async (req: Request, res: Response): Promise<any>
|
||||
try {
|
||||
const userId = req.user?.id;
|
||||
if (!userId) return res.status(401).json({ message: "Unauthorized" });
|
||||
const { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms } = req.body;
|
||||
await storage.saveAiChatTemplates(userId, { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms });
|
||||
const { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms, callTemplate } = req.body;
|
||||
await storage.saveAiChatTemplates(userId, { reminderGreeting, newPatientGreeting, generalFallback, rescheduleGreeting, reminderSms, callTemplate });
|
||||
const updated = await storage.getAiChatTemplates(userId);
|
||||
return res.status(200).json(updated);
|
||||
} catch (err) {
|
||||
|
||||
@@ -14,6 +14,8 @@ import {
|
||||
timeLabel,
|
||||
} from "../ai/reschedule-graph";
|
||||
import { getLlm, resolveAiProvider } from "../ai/llm-factory";
|
||||
import { runVoiceAssistantTurn, soundsLikeGoodbye } from "../ai/voice-assistant";
|
||||
import { getPublicBaseUrl } from "../utils/publicUrl";
|
||||
import { runEligibilityProcessor } from "../queue/processors/eligibilityProcessor";
|
||||
import {
|
||||
getHandoff, getAfterHoursHandoff,
|
||||
@@ -1169,7 +1171,7 @@ router.post("/webhook/voice", async (req: Request, res: Response): Promise<any>
|
||||
});
|
||||
}
|
||||
|
||||
const recordingCallbackUrl = `${process.env.BASE_URL || "https://communitydentistsoflowell.mydentalofficemanagement.com"}/api/twilio/webhook/voice-recording`;
|
||||
const recordingCallbackUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/voice-recording`;
|
||||
|
||||
res.set("Content-Type", "text/xml");
|
||||
return res.send(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
@@ -1228,4 +1230,144 @@ router.post("/webhook/voice-browser", async (req: Request, res: Response): Promi
|
||||
}
|
||||
});
|
||||
|
||||
// ── AI phone call (outbound "AI Call" from Patient Connection) ───────────────
|
||||
|
||||
const MAX_AI_CALL_TURNS = 12;
|
||||
|
||||
interface AiCallSession {
|
||||
patientId: number;
|
||||
userId: number;
|
||||
history: { role: "assistant" | "user"; text: string }[];
|
||||
turns: number;
|
||||
}
|
||||
|
||||
const aiCallSessions = new Map<string, AiCallSession>();
|
||||
|
||||
function aiVoiceTwiml(sayText: string, gatherActionUrl: string): string {
|
||||
// action is always set, so Twilio posts back to it on both speech and
|
||||
// silence timeout — nothing after </Gather> would ever run.
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Response>
|
||||
<Gather input="speech" language="en-US" speechTimeout="auto" action="${gatherActionUrl}" method="POST">
|
||||
<Say voice="alice">${escapeXml(sayText)}</Say>
|
||||
</Gather>
|
||||
</Response>`;
|
||||
}
|
||||
|
||||
function aiVoiceHangup(sayText: string): string {
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Response>
|
||||
<Say voice="alice">${escapeXml(sayText)}</Say>
|
||||
<Hangup/>
|
||||
</Response>`;
|
||||
}
|
||||
|
||||
// POST /api/twilio/webhook/ai-voice
|
||||
// Entry point + conversation loop for outbound AI-driven calls placed via
|
||||
// POST /api/twilio/make-ai-call. Twilio hits this once when the call connects
|
||||
// (no CallSid session yet) and again after every <Gather> with SpeechResult.
|
||||
router.post("/webhook/ai-voice", async (req: Request, res: Response): Promise<any> => {
|
||||
res.set("Content-Type", "text/xml");
|
||||
try {
|
||||
const { CallSid, SpeechResult } = req.body as { CallSid?: string; SpeechResult?: string };
|
||||
const patientIdParam = req.query.patientId as string | undefined;
|
||||
if (!CallSid) return res.send(aiVoiceHangup("Sorry, something went wrong. Goodbye."));
|
||||
|
||||
let session = aiCallSessions.get(CallSid);
|
||||
|
||||
// ── First hit for this call: set up session + greet ──────────────────
|
||||
if (!session) {
|
||||
const patientId = parseInt(patientIdParam || "", 10);
|
||||
const patient = !isNaN(patientId)
|
||||
? await db.patient.findUnique({ where: { id: patientId }, select: { id: true, userId: true, firstName: true } })
|
||||
: null;
|
||||
if (!patient) return res.send(aiVoiceHangup("Sorry, we could not find your patient record. Goodbye."));
|
||||
|
||||
session = { patientId: patient.id, userId: patient.userId, history: [], turns: 0 };
|
||||
aiCallSessions.set(CallSid, session);
|
||||
|
||||
const officeContact = await storage.getOfficeContact(patient.userId);
|
||||
const officeName = (officeContact as any)?.officeName?.trim() || "";
|
||||
const officeAddress = [
|
||||
(officeContact as any)?.streetAddress?.trim(),
|
||||
(officeContact as any)?.city?.trim(),
|
||||
(officeContact as any)?.state?.trim(),
|
||||
(officeContact as any)?.zipCode?.trim(),
|
||||
].filter(Boolean).join(", ");
|
||||
const officePhone = (officeContact as any)?.phoneNumber?.trim() || "";
|
||||
const appointmentDatetime = await getAppointmentDatetime(patient.id);
|
||||
|
||||
const chatTemplates = await storage.getAiChatTemplates(patient.userId);
|
||||
const rawCallTemplate = chatTemplates.callTemplate?.trim() ||
|
||||
"Hi {firstName}, this is Lisa, the AI assistant at {officeName}. How can I help you today?";
|
||||
const greeting = rawCallTemplate
|
||||
.replace(/\{firstName\}/g, patient.firstName || "there")
|
||||
.replace(/\{officeName\}/g, officeName || "our office");
|
||||
|
||||
session.history.push({ role: "assistant", text: greeting });
|
||||
await storage.createCommunication({
|
||||
patientId: patient.id, userId: patient.userId, channel: "voice",
|
||||
direction: "outbound", status: "completed", body: greeting, twilioSid: CallSid,
|
||||
});
|
||||
|
||||
// Stash office context on the session for later turns
|
||||
(session as any).ctx = { officeName, officeAddress, officePhone, appointmentDatetime };
|
||||
|
||||
const actionUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/ai-voice?patientId=${patient.id}`;
|
||||
return res.send(aiVoiceTwiml(greeting, actionUrl));
|
||||
}
|
||||
|
||||
// ── Caller stayed silent past the speech timeout ──────────────────────
|
||||
if (!SpeechResult?.trim()) {
|
||||
aiCallSessions.delete(CallSid);
|
||||
return res.send(aiVoiceHangup("I didn't catch a response. Our staff will follow up with you if needed. Goodbye."));
|
||||
}
|
||||
|
||||
session.history.push({ role: "user", text: SpeechResult.trim() });
|
||||
await storage.createCommunication({
|
||||
patientId: session.patientId, userId: session.userId, channel: "voice",
|
||||
direction: "inbound", status: "completed", body: SpeechResult.trim(), twilioSid: CallSid,
|
||||
});
|
||||
|
||||
session.turns++;
|
||||
|
||||
// ── Caller said something that sounds like goodbye, or we've hit the cap ──
|
||||
if (soundsLikeGoodbye(SpeechResult) || session.turns >= MAX_AI_CALL_TURNS) {
|
||||
aiCallSessions.delete(CallSid);
|
||||
return res.send(aiVoiceHangup("Thank you for calling, have a great day. Goodbye."));
|
||||
}
|
||||
|
||||
const aiSettings = await storage.getAiSettings(session.userId);
|
||||
const activeAi = resolveAiProvider(aiSettings ?? {});
|
||||
if (!activeAi) {
|
||||
aiCallSessions.delete(CallSid);
|
||||
return res.send(aiVoiceHangup("Our AI assistant is not available right now. Our staff will follow up with you. Goodbye."));
|
||||
}
|
||||
|
||||
const patientRow = await db.patient.findUnique({ where: { id: session.patientId }, select: { firstName: true } });
|
||||
const ctx = (session as any).ctx || {};
|
||||
const reply = await runVoiceAssistantTurn(
|
||||
session.history,
|
||||
{ firstName: patientRow?.firstName || "", ...ctx },
|
||||
activeAi.key, activeAi.provider, activeAi.model
|
||||
);
|
||||
|
||||
session.history.push({ role: "assistant", text: reply });
|
||||
await storage.createCommunication({
|
||||
patientId: session.patientId, userId: session.userId, channel: "voice",
|
||||
direction: "outbound", status: "completed", body: reply, twilioSid: CallSid,
|
||||
});
|
||||
|
||||
if (soundsLikeGoodbye(reply)) {
|
||||
aiCallSessions.delete(CallSid);
|
||||
return res.send(aiVoiceHangup(reply));
|
||||
}
|
||||
|
||||
const actionUrl = `${getPublicBaseUrl(req)}/api/twilio/webhook/ai-voice?patientId=${session.patientId}`;
|
||||
return res.send(aiVoiceTwiml(reply, actionUrl));
|
||||
} catch (err) {
|
||||
return res.send(aiVoiceHangup("Sorry, something went wrong on our end. Goodbye."));
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -3,6 +3,8 @@ import twilio from "twilio";
|
||||
import { storage } from "../storage";
|
||||
import { prisma as db } from "@repo/db/client";
|
||||
import { getHandoff, setHandoff, resetConversation, startNewPatientConversation, startRescheduleConversation, getAfterHoursHandoff, setAfterHoursHandoff } from "../ai/aiHandoffStore";
|
||||
import { resolveAiProvider } from "../ai/llm-factory";
|
||||
import { getTwilioPublicBaseUrl } from "../utils/publicUrl";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -117,6 +119,52 @@ router.post("/send-sms", async (req: Request, res: Response): Promise<any> => {
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/twilio/make-ai-call
|
||||
// Places an outbound call where Lisa (the AI assistant) conducts the whole
|
||||
// conversation live over the phone — see /webhook/ai-voice for the loop.
|
||||
router.post("/make-ai-call", async (req: Request, res: Response): Promise<any> => {
|
||||
try {
|
||||
const userId = req.user?.id;
|
||||
if (!userId) return res.status(401).json({ message: "Unauthorized" });
|
||||
|
||||
const { to, patientId } = req.body;
|
||||
if (!to || !patientId) return res.status(400).json({ message: "to and patientId are required" });
|
||||
|
||||
const settings = await storage.getTwilioSettings(userId);
|
||||
if (!settings) {
|
||||
return res.status(400).json({ message: "Twilio is not configured. Please add your Twilio credentials in Settings." });
|
||||
}
|
||||
|
||||
const aiSettings = await storage.getAiSettings(userId);
|
||||
if (!resolveAiProvider(aiSettings ?? {})) {
|
||||
return res.status(400).json({ message: "AI is not configured. Please add an AI provider key in Settings before placing an AI call." });
|
||||
}
|
||||
|
||||
const patient = await db.patient.findFirst({ where: { id: Number(patientId), userId } });
|
||||
if (!patient) return res.status(404).json({ message: "Patient not found" });
|
||||
|
||||
let publicBaseUrl: string;
|
||||
try {
|
||||
publicBaseUrl = getTwilioPublicBaseUrl();
|
||||
} catch (err: any) {
|
||||
return res.status(500).json({ message: err.message });
|
||||
}
|
||||
const webhookUrl = `${publicBaseUrl}/api/twilio/webhook/ai-voice?patientId=${patient.id}`;
|
||||
|
||||
const client = getTwilioClient(settings.accountSid, settings.authToken);
|
||||
const call = await client.calls.create({
|
||||
url: webhookUrl,
|
||||
method: "POST",
|
||||
from: settings.phoneNumber,
|
||||
to,
|
||||
});
|
||||
|
||||
return res.status(200).json({ sid: call.sid, status: call.status });
|
||||
} catch (err: any) {
|
||||
return res.status(500).json({ error: err.message || "Failed to place AI call" });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/twilio/send-reminders-batch
|
||||
router.post("/send-reminders-batch", async (req: Request, res: Response): Promise<any> => {
|
||||
try {
|
||||
|
||||
@@ -77,10 +77,11 @@ export const twilioStorage = {
|
||||
generalFallback: all["_ai_chat_general_fallback"] ?? "",
|
||||
rescheduleGreeting: all["_ai_chat_reschedule_greeting"] ?? "",
|
||||
reminderSms: all["_ai_chat_reminder_sms"] ?? "",
|
||||
callTemplate: all["_ai_chat_call_template"] ?? "",
|
||||
};
|
||||
},
|
||||
|
||||
async saveAiChatTemplates(userId: number, templates: { reminderGreeting?: string; newPatientGreeting?: string; generalFallback?: string; rescheduleGreeting?: string; reminderSms?: string }) {
|
||||
async saveAiChatTemplates(userId: number, templates: { reminderGreeting?: string; newPatientGreeting?: string; generalFallback?: string; rescheduleGreeting?: string; reminderSms?: string; callTemplate?: string }) {
|
||||
const settings = await db.twilioSettings.findUnique({ where: { userId } });
|
||||
const existing = (settings?.templates as Record<string, string>) || {};
|
||||
const updated: Record<string, string> = { ...existing };
|
||||
@@ -89,6 +90,7 @@ export const twilioStorage = {
|
||||
if (templates.generalFallback !== undefined) updated["_ai_chat_general_fallback"] = templates.generalFallback;
|
||||
if (templates.rescheduleGreeting !== undefined) updated["_ai_chat_reschedule_greeting"] = templates.rescheduleGreeting;
|
||||
if (templates.reminderSms !== undefined) updated["_ai_chat_reminder_sms"] = templates.reminderSms;
|
||||
if (templates.callTemplate !== undefined) updated["_ai_chat_call_template"] = templates.callTemplate;
|
||||
return db.twilioSettings.upsert({
|
||||
where: { userId },
|
||||
update: { templates: updated },
|
||||
|
||||
33
apps/Backend/src/utils/publicUrl.ts
Normal file
33
apps/Backend/src/utils/publicUrl.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import type { Request } from "express";
|
||||
|
||||
/**
|
||||
* Public HTTPS origin this request arrived on (e.g. https://broadwaydental.mydentalofficemanagement.com).
|
||||
* Derived from the Host header rather than an env var so it's automatically correct
|
||||
* per-office — each office's nginx forwards its own public hostname in Host,
|
||||
* and Cloudflare/Let's Encrypt terminate TLS in front of it either way.
|
||||
*
|
||||
* Only valid for requests that actually arrived via the public tunnel (i.e. inside
|
||||
* a Twilio webhook handler). For anything triggered from the staff browser — which
|
||||
* reaches the backend over the LAN-only local-* hostname — use getTwilioPublicBaseUrl()
|
||||
* instead, since Twilio can never reach that LAN hostname to fetch a callback URL.
|
||||
*/
|
||||
export function getPublicBaseUrl(req: Request): string {
|
||||
return `https://${req.get("host")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed public origin Twilio must use to call back into this office's server
|
||||
* (e.g. for outbound-call TwiML webhooks). Comes from CLOUDFLARE_HOST, the
|
||||
* per-office env var already reserved for this — not derived from the
|
||||
* triggering request, since that request may have arrived over the LAN-only
|
||||
* hostname, which Twilio cannot reach.
|
||||
*/
|
||||
export function getTwilioPublicBaseUrl(): string {
|
||||
const host = process.env.CLOUDFLARE_HOST?.trim();
|
||||
if (!host) {
|
||||
throw new Error(
|
||||
"CLOUDFLARE_HOST is not set — required to build a Twilio-reachable callback URL. Set it in .env to this office's public tunnel hostname."
|
||||
);
|
||||
}
|
||||
return `https://${host}`;
|
||||
}
|
||||
Reference in New Issue
Block a user