AI Calling Agent Platform

A full-stack solution for AI-powered calling agents with Twilio and OpenAI integration

Project Structure

ai-calling-agent/
backend/
config/
index.js
controllers/
agent.controller.js
call.controller.js
models/
agent.model.js
routes/
agent.routes.js
call.routes.js
services/
ai.service.js
twilio.service.js
tts.service.js
stt.service.js
app.js
server.js
Dockerfile
frontend/
public/
src/
components/
AgentCard.jsx
AgentForm.jsx
CallPanel.jsx
pages/
AgentsPage.jsx
CallPage.jsx
services/
agent.service.js
call.service.js
socket.service.js
App.jsx
main.jsx
Dockerfile
vite.config.js
docker-compose.yml
README_USER_MANUAL.md
.env.example

Backend Implementation

app.js (Express Setup)

import express from 'express';
import cors from 'cors';
import http from 'http';
import { Server } from 'socket.io';
import agentRoutes from './routes/agent.routes.js';
import callRoutes from './routes/call.routes.js';
import { initSocket } from './services/socket.service.js';

const app = express();
const server = http.createServer(app);

// Middleware
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: true }));

// Initialize Socket.IO
const io = new Server(server, {
  cors: {
    origin: process.env.FRONTEND_URL || 'http://localhost:3000',
    methods: ['GET', 'POST']
  }
});

initSocket(io);

// Routes
app.use('/api/agents', agentRoutes);
app.use('/api/calls', callRoutes);

// Health check
app.get('/health', (req, res) => {
  res.status(200).json({ status: 'OK' });
});

export { app, server };

agent.controller.js

import { Agent } from '../models/agent.model.js';
import { generateResponse } from '../services/ai.service.js';

export const createAgent = async (req, res) => {
  try {
    const { name, prompt, voice } = req.body;
    const agent = new Agent({ name, prompt, voice });
    await agent.save();
    res.status(201).json(agent);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

export const getAgents = async (req, res) => {
  try {
    const agents = await Agent.find();
    res.status(200).json(agents);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

export const getAgent = async (req, res) => {
  try {
    const agent = await Agent.findById(req.params.id);
    if (!agent) {
      return res.status(404).json({ error: 'Agent not found' });
    }
    res.status(200).json(agent);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

export const updateAgent = async (req, res) => {
  try {
    const { name, prompt, voice } = req.body;
    const agent = await Agent.findByIdAndUpdate(
      req.params.id,
      { name, prompt, voice },
      { new: true }
    );
    if (!agent) {
      return res.status(404).json({ error: 'Agent not found' });
    }
    res.status(200).json(agent);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

export const deleteAgent = async (req, res) => {
  try {
    const agent = await Agent.findByIdAndDelete(req.params.id);
    if (!agent) {
      return res.status(404).json({ error: 'Agent not found' });
    }
    res.status(200).json({ message: 'Agent deleted successfully' });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

export const testAgent = async (req, res) => {
  try {
    const agent = await Agent.findById(req.params.id);
    if (!agent) {
      return res.status(404).json({ error: 'Agent not found' });
    }
    
    const response = await generateResponse(agent.prompt, req.body.message);
    res.status(200).json({ response });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
};

twilio.service.js

import twilio from 'twilio';
import { textToSpeech } from './tts.service.js';
import { Agent } from '../models/agent.model.js';

const accountSid = process.env.TWILIO_ACCOUNT_SID;
const authToken = process.env.TWILIO_AUTH_TOKEN;
const twilioPhoneNumber = process.env.TWILIO_PHONE_NUMBER;

const client = twilio(accountSid, authToken);

export const initiateCall = async (agentId, phoneNumber) => {
  try {
    const agent = await Agent.findById(agentId);
    if (!agent) {
      throw new Error('Agent not found');
    }

    const call = await client.calls.create({
      url: `${process.env.BASE_URL}/api/calls/initiate/${agentId}`,
      to: phoneNumber,
      from: twilioPhoneNumber,
    });

    return call;
  } catch (error) {
    throw error;
  }
};

export const handleIncomingCall = async (agentId, callSid) => {
  try {
    const agent = await Agent.findById(agentId);
    if (!agent) {
      throw new Error('Agent not found');
    }

    // Generate initial greeting
    const greeting = await generateResponse(agent.prompt, 'Greet the caller');
    const greetingAudio = await textToSpeech(greeting, agent.voice);

    // Update the call with the greeting
    await client.calls(callSid).update({
      twiml: `<Response>
        <Play>${greetingAudio}</Play>
        <Record action="/api/calls/handle-recording/${agentId}" />
      </Response>`
    });

    return { success: true };
  } catch (error) {
    throw error;
  }
};

export const handleRecording = async (agentId, recordingUrl) => {
  try {
    const agent = await Agent.findById(agentId);
    if (!agent) {
      throw new Error('Agent not found');
    }

    // Convert recording to text
    const userMessage = await speechToText(recordingUrl);
    
    // Get AI response
    const aiResponse = await generateResponse(agent.prompt, userMessage);
    
    // Convert response to speech
    const responseAudio = await textToSpeech(aiResponse, agent.voice);

    return {
      twiml: `<Response>
        <Play>${responseAudio}</Play>
        <Record action="/api/calls/handle-recording/${agentId}" />
      </Response>`
    };
  } catch (error) {
    throw error;
  }
};

Frontend Implementation

AgentCard.jsx

import React from 'react';
import { useNavigate } from 'react-router-dom';
import { deleteAgent } from '../services/agent.service';

const AgentCard = ({ agent, onDelete }) => {
  const navigate = useNavigate();

  const handleDelete = async () => {
    try {
      await deleteAgent(agent._id);
      onDelete(agent._id);
    } catch (error) {
      console.error('Error deleting agent:', error);
    }
  };

  return (
    <div className="bg-white rounded-lg shadow-md p-6 hover:shadow-lg transition-shadow">
      <h3 className="text-xl font-semibold text-gray-800 mb-2">{agent.name}</h3>
      <p className="text-gray-600 mb-4 line-clamp-3">{agent.prompt}</p>
      <div className="flex items-center mb-4">
        <span className="bg-blue-100 text-blue-800 text-xs px-2 py-1 rounded">
          {agent.voice}
        </span>
      </div>
      <div className="flex space-x-2">
        <button
          onClick={() => navigate(`/agents/${agent._id}`)}
          className="px-4 py-2 bg-blue-600 text-white rounded hover:bg-blue-700 transition-colors"
        >
          Edit
        </button>
        <button
          onClick={() => navigate(`/call/${agent._id}`)}
          className="px-4 py-2 bg-green-600 text-white rounded hover:bg-green-700 transition-colors"
        >
          Call
        </button>
        <button
          onClick={handleDelete}
          className="px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700 transition-colors"
        >
          Delete
        </button>
      </div>
    </div>
  );
};

export default AgentCard;

CallPanel.jsx

import React, { useState, useEffect } from 'react';
import { initiateCall } from '../services/call.service';
import { useSocket } from '../services/socket.service';

const CallPanel = ({ agentId }) => {
  const [phoneNumber, setPhoneNumber] = useState('');
  const [callStatus, setCallStatus] = useState('idle');
  const [transcript, setTranscript] = useState([]);
  const socket = useSocket();

  useEffect(() => {
    if (!socket) return;

    socket.on('call_status', (status) => {
      setCallStatus(status);
    });

    socket.on('transcript_update', (data) => {
      setTranscript(prev => [...prev, data]);
    });

    return () => {
      socket.off('call_status');
      socket.off('transcript_update');
    };
  }, [socket]);

  const handleCall = async () => {
    try {
      setCallStatus('dialing');
      await initiateCall(agentId, phoneNumber);
    } catch (error) {
      console.error('Error initiating call:', error);
      setCallStatus('error');
    }
  };

  const callStatusColors = {
    idle: 'bg-gray-200',
    dialing: 'bg-yellow-200',
    in_progress: 'bg-blue-200',
    completed: 'bg-green-200',
    error: 'bg-red-200'
  };

  return (
    <div className="bg-white rounded-lg shadow-md p-6">
      <h2 className="text-2xl font-semibold text-gray-800 mb-4">Initiate Call</h2>
      
      <div className="mb-6">
        <label className="block text-gray-700 mb-2">Phone Number</label>
        <input
          type="tel"
          value={phoneNumber}
          onChange={(e) => setPhoneNumber(e.target.value)}
          className="w-full px-4 py-2 border rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
          placeholder="+1234567890"
        />
      </div>

      <div className="flex items-center mb-6">
        <span className={`px-4 py-2 rounded-lg ${callStatusColors[callStatus]}`}>
          Status: {callStatus.replace('_', ' ')}
        </span>
      </div>

      <button
        onClick={handleCall}
        disabled={!phoneNumber || callStatus !== 'idle'}
        className="px-6 py-3 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors disabled:bg-gray-400"
      >
        Start Call
      </button>

      {transcript.length > 0 && (
        <div className="mt-6 border-t pt-4">
          <h3 className="text-lg font-medium text-gray-800 mb-2">Conversation Transcript</h3>
          <div className="space-y-2">
            {transcript.map((item, index) => (
              <div key={index} className={`p-3 rounded-lg ${item.speaker === 'user' ? 'bg-gray-100' : 'bg-blue-50'}`}>
                <div className="font-medium">{item.speaker === 'user' ? 'Caller' : 'Agent'}</div>
                <div>{item.text}</div>
              </div>
            ))}
          </div>
        </div>
      )}
    </div>
  );
};

export default CallPanel;

Docker Configuration

docker-compose.yml

version: '3.8'

services:
  backend:
    build: ./backend
    ports:
      - "4000:4000"
    environment:
      - NODE_ENV=development
      - PORT=4000
      - FRONTEND_URL=http://localhost:3000
      - TWILIO_ACCOUNT_SID=${TWILIO_ACCOUNT_SID}
      - TWILIO_AUTH_TOKEN=${TWILIO_AUTH_TOKEN}
      - TWILIO_PHONE_NUMBER=${TWILIO_PHONE_NUMBER}
      - OPENAI_API_KEY=${OPENAI_API_KEY}
      - BASE_URL=http://localhost:4000
    volumes:
      - ./backend:/app
      - /app/node_modules
    depends_on:
      - postgres
    restart: unless-stopped

  frontend:
    build: ./frontend
    ports:
      - "3000:3000"
    environment:
      - VITE_API_URL=http://localhost:4000/api
    volumes:
      - ./frontend:/app
      - /app/node_modules
    depends_on:
      - backend
    restart: unless-stopped

  postgres:
    image: postgres:14
    environment:
      - POSTGRES_USER=${DB_USER}
      - POSTGRES_PASSWORD=${DB_PASSWORD}
      - POSTGRES_DB=${DB_NAME}
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
    restart: unless-stopped

volumes:
  postgres_data:

Backend Dockerfile

FROM node:18-alpine

WORKDIR /app

COPY package*.json ./

RUN npm install

COPY . .

EXPOSE 4000

CMD ["npm", "run", "dev"]

Frontend Dockerfile

FROM node:18-alpine

WORKDIR /app

COPY package*.json ./

RUN npm install

COPY . .

EXPOSE 3000

CMD ["npm", "run", "dev"]

Environment & User Manual

.env.example

# Backend
PORT=4000
FRONTEND_URL=http://localhost:3000
BASE_URL=http://localhost:4000

# Database
DB_USER=postgres
DB_PASSWORD=secret
DB_NAME=ai_agents

# Twilio
TWILIO_ACCOUNT_SID=your_account_sid
TWILIO_AUTH_TOKEN=your_auth_token
TWILIO_PHONE_NUMBER=+1234567890

# OpenAI
OPENAI_API_KEY=your_openai_key

README_USER_MANUAL.md

# AI Calling Agent Platform - User Manual

## Overview
This platform allows you to create AI-powered calling agents that can conduct phone conversations using OpenAI for responses and Twilio for telephony.

## Prerequisites
- Docker and Docker Compose installed
- Twilio account with a phone number
- OpenAI API key

## Setup
1. Clone the repository
2. Create a `.env` file based on `.env.example`
3. Fill in all required environment variables
4. Run `docker-compose up --build`

## Running the Application
1. After containers are running:
   - Frontend: http://localhost:3000
   - Backend: http://localhost:4000
2. Use the web interface to:
   - Create and manage agents
   - Initiate test calls
   - Monitor call status and transcripts

## Key Features
- **Agent Management**: Create, read, update, and delete AI agents
- **Call Initiation**: Start outbound calls to any phone number
- **Real-time Monitoring**: View call status and conversation transcripts in real-time
- **Customizable Responses**: Each agent has its own prompt template for responses

## API Endpoints
- `GET /api/agents` - List all agents
- `POST /api/agents` - Create new agent
- `GET /api/agents/:id` - Get agent details
- `PUT /api/agents/:id` - Update agent
- `DELETE /api/agents/:id` - Delete agent
- `POST /api/calls/initiate` - Start a new call

## Troubleshooting
- Check container logs with `docker-compose logs`
- Ensure all environment variables are set correctly
- Verify Twilio phone number is properly configured

Made with DeepSite LogoDeepSite - 🧬 Remix