import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs';
import { openWebSocket } from './websocket-client.mjs';
async function waitFor(session, type) {
for (;;) {
const event = await session.receive();
if (event.type === type) return event;
if (event.type === 'rate_limits.updated') continue;
if (type === 'input_audio_buffer.committed' && ['conversation.item.added', 'conversation.item.done'].includes(event.type)) continue;
throw new Error('Unexpected session event.');
}
}
let session;
try {
const mode = process.env.API_REALTIME_MODE ?? 'text';
const voice = process.env.API_VOICE ?? 'marin';
const outputPath = process.env.API_AUDIO_OUTPUT ?? 'realtime-output.wav';
const pcmPath = process.env.API_AUDIO_PCM_PATH;
if (!['text', 'audio'].includes(mode) || !voice.trim()) throw new Error('Invalid Realtime mode or voice.');
if (mode === 'audio' && existsSync(outputPath)) throw new Error('Output already exists.');
let inputAudio;
if (pcmPath) {
const file = statSync(pcmPath);
// Raw PCM16LE, mono, 24 kHz: this example accepts 0.1..30 seconds.
if (!file.isFile() || file.size < 4800 || file.size > 1440000 || file.size % 2) throw new Error('Invalid PCM input size.');
inputAudio = readFileSync(pcmPath);
if (['RIFF', 'OggS', 'ID3'].some((tag) => inputAudio.subarray(0, tag.length).toString() === tag)) {
throw new Error('Provide raw PCM, not a media container.');
}
}
session = await openWebSocket('realtime', { model: process.env.API_MODEL ?? '' });
const created = await waitFor(session, 'session.created');
if (created.session?.type !== 'realtime' || !created.session.id) throw new Error('Invalid GA session.');
const audio = { input: { format: { type: 'audio/pcm', rate: 24000 }, turn_detection: null } };
if (mode === 'audio') audio.output = { format: { type: 'audio/pcm', rate: 24000 }, voice };
await session.send({ type: 'session.update', session: {
type: 'realtime', output_modalities: [mode], audio, tools: [],
instructions: 'Answer briefly. If given audio, respond to what the speaker says.',
} });
const updated = (await waitFor(session, 'session.updated')).session;
if (updated?.id !== created.session.id || updated.type !== 'realtime'
|| JSON.stringify(updated.output_modalities) !== JSON.stringify([mode])
|| updated.audio?.input?.turn_detection !== null
|| updated.audio.input.format?.type !== 'audio/pcm' || updated.audio.input.format.rate !== 24000) {
throw new Error('Session configuration was not applied.');
}
if (mode === 'audio' && (updated.audio?.output?.format?.type !== 'audio/pcm'
|| updated.audio.output.format.rate !== 24000 || updated.audio.output.voice !== voice)) {
throw new Error('Unexpected output audio configuration.');
}
if (inputAudio) {
for (let offset = 0; offset < inputAudio.length; offset += 24000) {
await session.send({ type: 'input_audio_buffer.append', audio: inputAudio.subarray(offset, offset + 24000).toString('base64') });
}
await session.send({ type: 'input_audio_buffer.commit' });
const committed = await waitFor(session, 'input_audio_buffer.committed');
if (typeof committed.item_id !== 'string' || !committed.item_id) throw new Error('Audio was not committed.');
} else {
await session.send({ type: 'conversation.item.create', item: {
type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Say hello in one short sentence.' }],
} });
}
await session.send({ type: 'response.create', response: { output_modalities: [mode] } });
let responseId;
let itemId;
let contentDone = false;
let text = '';
let bytes = 0;
const audioChunks = [];
for (;;) {
const event = await session.receive();
if (event.type === 'response.created') {
if (responseId || typeof event.response?.id !== 'string' || !event.response.id) throw new Error('Invalid response identity.');
responseId = event.response.id;
continue;
}
if (event.response_id && event.response_id !== responseId) throw new Error('Unexpected response identity.');
if (['response.audio.delta', 'response.text.delta', 'response.audio.done', 'response.text.done'].includes(event.type)) {
throw new Error('Legacy Realtime events are not supported by this GA example.');
}
if ([`response.output_${mode}.delta`, `response.output_${mode}.done`].includes(event.type)) {
if (!responseId || event.response_id !== responseId || event.output_index !== 0 || event.content_index !== 0
|| typeof event.item_id !== 'string' || !event.item_id || contentDone) throw new Error('Invalid content event.');
if (itemId && itemId !== event.item_id) throw new Error('Mixed output items.');
itemId = event.item_id;
if (event.type.endsWith('.done')) {
if (mode === 'text' && event.text !== text) throw new Error('Text completion mismatch.');
contentDone = true;
} else {
if (typeof event.delta !== 'string' || !event.delta.length) throw new Error('Invalid content delta.');
if (mode === 'text') {
text += event.delta;
bytes += Buffer.byteLength(event.delta);
} else {
const chunk = Buffer.from(event.delta, 'base64');
if (!chunk.length || chunk.toString('base64') !== event.delta) throw new Error('Invalid Base64 audio.');
audioChunks.push(chunk);
bytes += chunk.length;
}
if (bytes > 10 * 1024 * 1024) throw new Error('Output limit exceeded.');
}
} else if (event.type === 'response.done') {
const result = event.response;
if (!responseId || result?.id !== responseId || result.status !== 'completed' || result.error || !contentDone
|| !Array.isArray(result.output) || result.output.length !== 1) throw new Error('Response did not complete.');
const item = result.output[0];
if (item.id !== itemId || item.type !== 'message' || item.role !== 'assistant' || item.status !== 'completed'
|| !Array.isArray(item.content) || item.content.length !== 1 || item.content[0].type !== `output_${mode}`) {
throw new Error('Unexpected output, refusal, or tool call.');
}
if (mode === 'text') {
if (!text.trim() || item.content[0].text !== text) throw new Error('Final text mismatch.');
console.log(text);
} else {
if (!bytes || bytes % 2) throw new Error('Incomplete PCM sample data.');
const header = Buffer.alloc(44);
header.write('RIFF', 0); header.writeUInt32LE(36 + bytes, 4); header.write('WAVEfmt ', 8);
header.writeUInt32LE(16, 16); header.writeUInt16LE(1, 20); header.writeUInt16LE(1, 22);
header.writeUInt32LE(24000, 24); header.writeUInt32LE(48000, 28);
header.writeUInt16LE(2, 32); header.writeUInt16LE(16, 34);
header.write('data', 36); header.writeUInt32LE(bytes, 40);
writeFileSync(outputPath, Buffer.concat([header, ...audioChunks]), { flag: 'wx' });
console.log(`Saved ${bytes} PCM bytes to ${outputPath}`);
}
break;
} else if (['response.failed', 'response.incomplete', 'response.cancelled'].includes(event.type)) {
throw new Error('Response ended without success.');
}
}
} catch {
console.error('Realtime failed. Check configuration, session, media format, events, and deadline; no automatic retry.');
process.exitCode = 1;
} finally {
session?.close();
}