Using Responses API (OpenAI only)

This guide provides instructions for using the OpenAI Responses API through the Truefoundry LLM Gateway. The API enables:

Stateful conversations by utilizing previous responses as input for context-aware interactions
Enhanced model capabilities with integrated tools such as file search, web search, and more
Seamless integration with external systems through support for function calling

Prerequisites

TrueFoundry API Key
Provider account configured in TrueFoundry (e.g., OpenAI)

Authentication

All API requests require authentication using your TrueFoundry API key.

from openai import OpenAI

API_KEY = "your-truefoundry-api-key"
BASE_URL = "https://{controlPlaneUrl}/api/llm/api/inference/openai"

# Configure OpenAI client with TrueFoundry settings
client = OpenAI(
    api_key=API_KEY,
    base_url=BASE_URL
)

extra_headers = {
    "x-tfy-provider-name": "tfy-provider-name"  # Replace with your provider integration name
}

Core API Operations

1. Create a Model Response

Creates a model response. Provide text or image inputs to generate text or JSON outputs

response = client.responses.create(
    model="tfy-model-name",  # e.g., "openai-main/gpt-4"
    input=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Hello, how are you?"}
    ],
    extra_headers=extra_headers
)

print(response.id) # e.g - resp_6810b2fefb7081929caf20e63b16933a09840671ba0b62de

2. Get a model response

Retrieves a model response with the given ID.

response = client.responses.retrieve(
    response_id=response.id,
    extra_headers=extra_headers
)

print(response)

3. Delete a Model Response

Deletes a model response with the given ID.

response = client.responses.del(
    response_id=response.id,
    extra_headers=extra_headers
)

print(response)

4. List Input Items

Retrieve the input messages used to generate a response.

response = client.responses.input_items.list(
    response_id=response.id,
    extra_headers=extra_headers
)

for item in input_items.data:
    print(f"Role: {item.role}, Content: {item.content}")

Advanced Usage

Streaming Responses

For real-time generation of responses:

stream = client.responses.create(
    model="tfy-model-name",
    input=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Write a short poem about AI."}
    ],
    stream=True,
    extra_headers=extra_headers
)

for chunk in stream:
    if chunk.output and chunk.output[0].content:
        print(chunk.output[0].content[0].text, end="", flush=True)

Using Function Calling

Integrate with external systems by defining tools:

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get current weather in a location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "City and state, e.g., San Francisco, CA"
                    }
                },
                "required": ["location"]
            }
        }
    }
]

response = client.responses.create(
    model="tfy-model-name",
    input=[
        {"role": "user", "content": "What's the weather like in New York?"}
    ],
    tools=tools,
    tool_choice="auto",
    extra_headers=extra_headers
)

# Process tool calls from the response
if response.output[0].content[0].type == "tool_calls":
    tool_calls = response.output[0].content[0].tool_calls
    for call in tool_calls:
        function_name = call.function.name
        function_args = json.loads(call.function.arguments)
        print(f"Function: {function_name}, Arguments: {function_args}")

Complete Workflow Example

The following example demonstrates a complete conversation workflow:

import json
from openai import OpenAI

API_KEY = "your-truefoundry-api-key"
BASE_URL = "https://{controlPlaneUrl}/api/llm/api/inference/openai"

client = OpenAI(
    api_key=API_KEY,
    base_url=BASE_URL
)

extra_headers = {
    "x-tfy-provider-name": "tfy-provider-name"
}

# Initial conversation
response = client.responses.create(
    model="tfy-model-name",
    input=[
        {"role": "system", "content": "You are a helpful assistant specializing in Python programming."},
        {"role": "user", "content": "How do I read a JSON file in Python?"}
    ],
    extra_headers=extra_headers
)

response_id = response.id
print(f"Assistant: {response.output[0].content[0].text}\n")

# Continue the conversation
follow_up_response = client.responses.create(
    model="tfy-model-name",
    input=[
        {"role": "user", "content": "What if the JSON file has nested objects?"}
    ],
    previous_response_id=response_id,  # Links this response to the previous conversation
    extra_headers=extra_headers
)

# Get list of inputs
input_items_response = client.responses.input_items.list(
    response_id=follow_up_response.id,
    extra_headers=extra_headers
)

print(input_items_response)

Response Structure Reference

The API response includes these key fields:

{
  "id": "resp_6810a8778f5c8192826af07837cba3320ef4ee23cd28cc30",
  "object": "response",
  "created_at": 1745922167,
  "status": "completed", // Possible values: completed, incomplete, error
  "error": null, // Contains error information if status is "error"
  "model": "gpt-4-0613",
  "output": [
    {
      "id": "msg_6810a878647081928412a73a17bfb29a0ef4ee23cd28cc30",
      "type": "message",
      "status": "completed",
      "content": [
        {
          "type": "output_text", // Could also be "tool_calls" for function calling
          "annotations": [],
          "text": "Sample Text"
        }
      ],
      "role": "assistant"
    }
  ],
  "parallel_tool_calls": true,
  "previous_response_id": null, // ID of a previous response in a conversation
  "temperature": 0.7,
  "tool_choice": "auto",
  "tools": [],
  "top_p": 1,
  "usage": {
    "input_tokens": 24,
    "output_tokens": 40,
    "total_tokens": 64
  },
  "user": null,
  "metadata": {},
  "provider": "openai"
}