Voice AI Architecture Guide: Cascaded vs Speech-to-Speech in 2026

Example: Cascaded Voice Agent with Deepgram + GPT-4 + ElevenLabs

import Deepgram from "@deepgram/sdk";
import OpenAI from "openai";
import { ElevenLabsClient } from "elevenlabs";

// Initialize clients
const deepgram = new Deepgram(process.env.DEEPGRAM_API_KEY);
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
const elevenlabs = new ElevenLabsClient({ apiKey: process.env.ELEVENLABS_API_KEY });

async function processVoiceInput(audioBuffer) {
  // Step 1: STT - Transcribe audio (100-500ms)
  const transcription = await deepgram.transcription.preRecorded({
    buffer: audioBuffer,
    mimetype: "audio/wav"
  }, { model: "nova-3", language: "en" });

  const userText = transcription.results.channels[0].alternatives[0].transcript;
  console.log("User said:", userText);

  // Step 2: LLM - Generate response (200-2000ms)
  const completion = await openai.chat.completions.create({
    model: "gpt-4",
    messages: [
      { role: "system", content: "You are a helpful voice assistant." },
      { role: "user", content: userText }
    ],
    max_tokens: 150
  });

  const responseText = completion.choices[0].message.content;
  console.log("Assistant responds:", responseText);

  // Step 3: TTS - Synthesize speech (200-800ms)
  const audio = await elevenlabs.generate({
    voice: "Rachel",
    text: responseText,
    model_id: "eleven_multilingual_v2"
  });

  return audio; // Total: 500-3300ms depending on streaming optimizations
}

Example: GPT-4o Realtime API (WebSocket)

import WebSocket from "ws";

const ws = new WebSocket("wss://api.openai.com/v1/realtime", {
  headers: {
    "Authorization": `Bearer ${process.env.OPENAI_API_KEY}`,
    "OpenAI-Beta": "realtime=v1"
  }
});

ws.on("open", () => {
  // Configure session for voice conversation
  ws.send(JSON.stringify({
    type: "session.update",
    session: {
      modalities: ["text", "audio"],
      instructions: "You are a helpful voice assistant.",
      voice: "alloy",
      input_audio_format: "pcm16",
      output_audio_format: "pcm16",
      turn_detection: { type: "server_vad" } // Built-in VAD
    }
  }));
});

ws.on("message", (data) => {
  const event = JSON.parse(data);

  if (event.type === "response.audio.delta") {
    // Stream audio chunks directly to speaker (sub-1s latency)
    playAudioChunk(event.delta);
  }

  if (event.type === "response.audio.done") {
    console.log("Response complete");
  }
});

// Send audio input
function sendUserAudio(audioBuffer) {
  ws.send(JSON.stringify({
    type: "input_audio_buffer.append",
    audio: audioBuffer.toString("base64")
  }));
}