Realtime Models

TrueFoundry offers a secure and efficient gateway to seamlessly integrate various Large Language Models (LLMs) into your applications, including realtime models hosted on various providers.

Add Realtime Models to Gateway

The process is pretty much the same. Follow these steps -

Navigate to Integrations.
Click on "Add New Integration".
Select the provider from the list.
Enter the required API keys and access credentials.

Once completed, your models will be ready for use in both the playground and your code.

Get Started using Rest API

import asyncio
import json
import websockets

BASE_URL = "wss://llm-gateway.truefoundry.com/api/llm/api/inference/openai/v1/realtime?model=openai-main/realtime-preview"

headers = {
    "Authorization": f"Bearer Enter your API Key here",
}

async def connect():
    async with websockets.connect(BASE_URL, additional_headers=headers) as ws:
        print("WebSocket connection established!")

        # Can follow the openai realtime api docs
        # https://platform.openai.com/docs/api-reference/realtime

        # Update session
        session_update = {
            "type": "session.update",
            "session": {
                "modalities": ["text"],
                "model": "openai-main/realtime-preview",
            },
        }
        await ws.send(json.dumps(session_update))
        print("Session updated!")

        # Send user message
        message = {
            "type": "conversation.item.create",
            "item": {
                "type": "message",
                "role": "user",
                "content": [{"type": "input_text", "text": "Say hello!"}],
            },
        }
        await ws.send(json.dumps(message))
        print("Conversation item created!")

        # Request response
        response_request = {"type": "response.create"}
        await ws.send(json.dumps(response_request))
        print("Response requested!")

        # Read responses
        async for msg in ws:
            event = json.loads(msg)

            if event.get("type") == "response.text.delta":
                print(event["delta"], end="", flush=True)

            elif event.get("type") == "response.text.done":
                print("
Response completed.")

            elif event.get("type") == "response.done":
                print("Session complete, closing connection.")
                break

asyncio.run(connect())

Get Started using NodeJS


// Requires: yarn add ws @types/ws
const { OpenAIRealtimeWS } = require('openai/beta/realtime/ws');
const OpenAI = require('openai');

const openai = new OpenAI({
  apiKey: 'Enter your API Key here',
  baseURL: 'wss://llm-gateway.truefoundry.com/api/llm/api/inference/openai/v1',
});

const rt = new OpenAIRealtimeWS(
  { model: 'openai-main/realtime-preview', options: {} },
  openai
);

// Access the underlying WebSocket instance
rt.socket.on('open', () => {
  console.log('Connection opened!');

  // Can follow the openai realtime api docs
  // https://platform.openai.com/docs/api-reference/realtime
  rt.send({
    type: 'session.update',
    session: {
      modalities: ['text'],
      model: 'openai-main/realtime-preview',
    },
  });

  console.log('Session updated!');

  rt.send({
    type: 'conversation.item.create',
    item: {
      type: 'message',
      role: 'user',
      content: [{ type: 'input_text', text: 'Say a couple paragraphs!' }],
    },
  });

  console.log('Conversation item created!');

  rt.send({ type: 'response.create' });

  console.log('Response created!');
});

rt.on('error', (err) => {
  console.error('WebSocket Error:', err);
});

rt.on('session.created', (event) => {
  console.log('session created!', event.session);
  console.log();
  });

  

rt.on('response.text.delta', (event) => process.stdout.write(event.delta));
rt.on('response.text.done', () => console.log());
rt.on('response.done', () => rt.close());

rt.socket.on('close', () => console.log('
Connection closed!'));

Get Started using OpenAI Library


import asyncio
from openai import AsyncOpenAI

async def main():
    client = AsyncOpenAI(
        api_key="Enter your API Key here",
        base_url="wss://llm-gateway.truefoundry.com/api/llm/api/inference/openai/v1"
    )

    try:
        async with client.beta.realtime.connect(model="openai-main/realtime-preview") as connection:
            # Can follow the openai realtime api docs
            # https://platform.openai.com/docs/api-reference/realtime

            print("WebSocket connection established!")

            # Update session
            await connection.session.update(session={'modalities': ['text']})
            print("Session updated!")

            # Send user message
            await connection.conversation.item.create(
                item={
                    "type": "message",
                    "role": "user",
                    "content": [{"type": "input_text", "text": "Say hello!"}],
                }
            )
            print("Conversation item created!")

            # Request response
            await connection.response.create()
            print("Response requested!")

            # Read responses
            async for event in connection:
                if event.type == 'response.text.delta':
                    print(event.delta, flush=True, end="")

                elif event.type == 'response.text.done':
                    print("
Response completed.")

                elif event.type == "response.done":
                    print("Session complete, closing connection.")
                    break

    except Exception as e:
        print(f"Error: {e}")

asyncio.run(main())

Get Started using Curl


# we're using websocat for this example, but you can use any websocket client
websocat "wss://llm-gateway.truefoundry.com/api/llm/api/inference/openai/v1/realtime?model=openai-main/realtime-preview" \
  -H 'Authorization: Bearer Enter your API Key here'
# once connected, you can send your messages as you would with OpenAI's Realtime API