Build Your First Voice Assistant - Uplift AI API Docs

Beta Feature

The Realtime Assistants API is currently in beta. This tutorial uses the latest features available.

What We’ll Build

In this tutorial, we’ll create a voice assistant that can:

The complete example code is available on GitHub: react-assistant-demo

Prerequisites

Before starting, you’ll need:

Step 1: Create Your Assistant

First, let’s create an assistant using the API or the UpliftAI platform.

  1. Go to platform.upliftai.org
  2. Navigate to Assistants → Create New
  3. Configure your assistant:
    • Name: “My First Assistant”
    • Instructions: “You are a helpful and friendly assistant”
    • Enable “Public Access” for this tutorial
  4. Copy the Assistant ID
curl -X POST https://api.upliftai.org/v1/realtime-assistants \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{\n    "name": "My First Assistant",\n    "public": true,\n    "config": {\n      "agent": {\n        "instructions": "You are a helpful and friendly assistant. Be conversational and engaging.",\n        "initialGreeting": true,\n        "greetingInstructions": "Say hello and ask how you can help today."\n      },\n      "stt": {\n        "default": {\n          "provider": "groq",\n          "model": "whisper-large-v3"\n        }\n      },\n      "tts": {\n        "default": {\n          "provider": "upliftai",\n          "voiceId": "v_meklc281",\n          "outputFormat": "MP3_22050_32"\n        }\n      },\n      "llm": {\n        "default": {\n          "provider": "groq",\n          "model": "openai/gpt-oss-120b"\n        }\n      }\n    }\n  }'

Save the returned realtimeAssistantId

Step 2: Set Up Your React Project

Create a new React application and install the required dependencies:

# Create a new React app
npx create-react-app voice-assistant-demo
cd voice-assistant-demo

# Install UpliftAI SDK and dependencies
npm install @upliftai/assistants-react @livekit/components-react livekit-client

Step 3: Create the Connection Logic

Create a component to handle assistant connection:

App.js

import { useState } from 'react';
import { UpliftAIRoom } from '@upliftai/assistants-react';
import AssistantView from './AssistantView';
import './App.css';

function App() {
  const [sessionData, setSessionData] = useState(null);
  const [assistantId, setAssistantId] = useState('');
  const [loading, setLoading] = useState(false);
  const [error, setError] = useState(null);

const connectToAssistant = async () => {
    if (!assistantId.trim()) {
      setError('Please enter an Assistant ID');
      return;
    }

setLoading(true);
    setError(null);

try {
      const response = await fetch(
        `https://api.upliftai.org/v1/realtime-assistants/${assistantId}/createPublicSession`,
        {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json',
          },
          body: JSON.stringify({
            participantName: 'Demo User',
          }),
        }
      );

if (!response.ok) {
        throw new Error(`Failed to create session: ${response.status}`);
      }

const data = await response.json();
      setSessionData(data);
    } catch (err) {
      setError(err.message);
    } finally {
      setLoading(false);
    }
  };

if (!sessionData) {
    return (
      <div className="connect-screen">
        <h1>Voice Assistant Demo</h1>
        <input
          type="text"
          placeholder="Enter Assistant ID"
          value={assistantId}
          onChange={(e) => setAssistantId(e.target.value)}
        />
        <button onClick={connectToAssistant} disabled={loading}>
          {loading ? 'Connecting...' : 'Connect'}
        </button>
        {error && <p className="error">{error}</p>}
      </div>
    );
  }

return (
    <UpliftAIRoom
      token={sessionData.token}
      serverUrl={sessionData.wsUrl}
      connect={true}
      audio={true}
      video={false}
    >
      <AssistantView />
    </UpliftAIRoom>
  );
}

export default App;

Step 4: Build the Assistant Interface

Create the main assistant view with voice visualization:

AssistantView.js

import {
  useUpliftAIRoom,
  useVoiceAssistant,
  BarVisualizer,
  TrackToggle,
  DisconnectButton,
  AudioTrack,
  useTracks
} from '@upliftai/assistants-react';
import { Track } from 'livekit-client';

function AssistantView() {
  const { isConnected, agentParticipant } = useUpliftAIRoom();
  const { state } = useVoiceAssistant();

const tracks = useTracks([Track.Source.Microphone], {
    onlySubscribed: true,
  });
  const agentTrack = tracks.find((t) => !t.participant.isLocal);

return (
    <div className="assistant-container">
      <div className="status-bar">
        <span className={`status ${isConnected ? 'connected' : 'disconnected'}`}>{isConnected ? '🟢 Connected' : '🔴 Disconnected'}</span>
        {agentParticipant && <span>Agent: {agentParticipant.identity}</span>}
      </div>

<div className="visualizer">
        {agentTrack && (
          <>  
            <AudioTrack trackRef={agentTrack} />
            <BarVisualizer
              state={state}
              trackRef={agentTrack}
              barCount={20}
              className="bar-visualizer"
            />
          <>  
        )}

<div className="agent-state">
          {state === 'speaking' && '🗣️ Speaking...'}
          {state === 'thinking' && '🤔 Thinking...'}
          {state === 'listening' && '👂 Listening...'}
        </div>
      </div>

<div className="controls">
        <TrackToggle source={Track.Source.Microphone}>🎤 Microphone</TrackToggle>
        <DisconnectButton>End Call</DisconnectButton>
      </div>
    </div>
  );
}

export default AssistantView;

Step 5: Add Custom Tools

Extend your assistant with custom functionality:

AssistantWithTools.js

import { useState, useCallback } from 'react';
import {
  UpliftAIRoom,
  useUpliftAIRoom,
  ToolConfig
} from '@upliftai/assistants-react';

const customTools: ToolConfig[] = [
  {
    name: 'get_weather',
    description: 'Get current weather for a location',
    parameters: {
      type: 'object',
      properties: {
        location: {
          type: 'string',
          description: 'City and state, e.g., San Francisco, CA',
        },
      },
      required: ['location'],
    },
    timeout: 10,
    handler: async (data) => {
      const payload = JSON.parse(data.payload);
      const { location } = payload.arguments.raw_arguments;

const weather = {
        location,
        temperature: Math.floor(Math.random() * 30 + 50),
        condition: ['sunny', 'cloudy', 'rainy'][Math.floor(Math.random() * 3)],
      };

return JSON.stringify({
        result: weather,
        presentationInstructions:
          `The weather in ${location} is ${weather.temperature}°F and ${weather.condition}`,
      });
    },
  },
  {
    name: 'tell_joke',
    description: 'Tell a random joke',
    parameters: {
      type: 'object',
      properties: {},
      required: [],
    },
    timeout: 5,
    handler: async () => {
      const jokes = [
        "Why don't scientists trust atoms? Because they make up everything!",
        "What do you call a bear with no teeth? A gummy bear!",
        "Why did the math book look so sad? It had too many problems!",
      ];

const joke = jokes[Math.floor(Math.random() * jokes.length)];

return JSON.stringify({
        joke,
        presentationInstructions: joke,
      });
    },
  },
];

function EnhancedAssistant({ sessionData }) {
  return (
    <UpliftAIRoom
      token={sessionData.token}
      serverUrl={sessionData.wsUrl}
      connect={true}
      audio={true}
      video={false}
      tools={customTools}
    >
      <AssistantViewWithTools />
    </UpliftAIRoom>
  );
}

Step 6: Dynamic Instruction Updates

function InstructionManager() {
  const { updateInstruction } = useUpliftAIRoom();
  const [instructions, setInstructions] = useState('');

const handleUpdate = async () => {
    try {
      await updateInstruction(instructions);
      alert('Instructions updated!');
    } catch (error) {
      console.error('Failed to update:', error);
    }
  };

return (
    <div className="instruction-manager">
      <h3>Customize Behavior</h3>
      <textarea
        value={instructions}
        onChange={(e) => setInstructions(e.target.value)}
        placeholder="e.g., 'Speak like a pirate' or 'Be extra helpful with math'"
        rows={3}
      />
      <button onClick={handleUpdate}>Update Instructions</button>
    </div>
  );
}

Step 7: Add Styling

App.css

.connect-screen {
  display: flex;
  flex-direction: column;
  align-items: center;
  justify-content: center;
  min-height: 100vh;
  gap: 20px;
}

.connect-screen input {
  padding: 10px;
  font-size: 16px;
  border: 1px solid #ddd;
  border-radius: 4px;
  width: 300px;
}

.connect-screen button {
  padding: 10px 20px;
  font-size: 16px;
  background: #16A34A;
  color: white;
  border: none;
  border-radius: 4px;
  cursor: pointer;
}

.connect-screen button:disabled {
  opacity: 0.5;
  cursor: not-allowed;
}

.assistant-container {
  max-width: 800px;
  margin: 0 auto;
  padding: 20px;
}

.status-bar {
  display: flex;
  justify-content: space-between;
  padding: 10px;
  background: #f3f4f6;
  border-radius: 8px;
  margin-bottom: 20px;
}

.status.connected {
  color: #16A34A;
}

.status.disconnected {
  color: #dc2626;
}

.visualizer {
  background: #1f2937;
  border-radius: 8px;
  padding: 40px;
  margin: 20px 0;
  text-align: center;
}

.bar-visualizer {
  height: 100px;
  margin: 20px 0;
}

.agent-state {
  color: white;
  font-size: 18px;
  margin-top: 20px;
}

.controls {
  display: flex;
  gap: 10px;
  justify-content: center;
}

.controls button {
  padding: 10px 20px;
  font-size: 16px;
  border-radius: 4px;
  cursor: pointer;
}

.error {
  color: #dc2626;
  margin-top: 10px;
}

Step 8: Test Your Assistant

  1. Start your development server:
   npm start
  1. Open http://localhost:3000 in your browser
  2. Enter your Assistant ID and click Connect
  3. Start talking! Try:
    • “Hello, how are you?”
    • “What’s the weather in New York?”
    • “Tell me a joke”
    • “Can you help me with math?”

Production Considerations

Security

// Backend session creation for private assistants
app.post('/api/create-session', authenticate, async (req, res) => {
  const response = await fetch(
    `https://api.upliftai.org/v1/realtime-assistants/${ASSISTANT_ID}/createSession`,
    {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.UPLIFTAI_API_KEY}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        participantName: req.user.name,
      }),
    }
  );

const sessionData = await response.json();
  res.json(sessionData);
});

Error Handling

const { error, connectionState } = useConnectionState();

if (error) {
  return <ErrorView error={error} onRetry={reconnect} />;
}

Tool Implementation

{
  name: 'get_custom_data',
  description: 'Query user data',
  parameters: { /* ... */ },
  handler: async (data) => {
    try {
      const result = await query_your_api(data);
      return JSON.stringify({ result });
    } catch (error) {
      console.error('Tool error:', error);
      return JSON.stringify({
        error: 'Unable to complete request',
        presentationInstructions: 'I encountered an error. Please try again.'
      });
    }
  }
}

Next Steps

Explore Advanced Features \n\nLearn about building complex tools and integrations

View Complete Example \n\nCheck out the full example with all features

SDK Reference \n\nDeep dive into the React SDK documentation

Deploy Your Assistant \n\nLearn how to deploy to production

Troubleshooting

Assistant not connecting

No audio from assistant

Tools not executing

High latency or delays

Get support from us