From 726901cb319a332013dcca089f435577668e06ba Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 9 Jun 2025 19:11:59 -0700 Subject: [PATCH] feat(voice): speech to speech mode for deployed chat (#467) * finished barebones, optimized speech to speech chat deploy * better visualization, interruption still not good * fixed some turn detection, still not ideal. have to press mute + unmute to process successive queries * improvements * removed MediaSource in favor of blob, simplified echo cancellation and overall logic * simplified * cleanups * ack PR comments --- apps/sim/app/api/proxy/tts/stream/route.ts | 118 +++ apps/sim/app/chat/[subdomain]/chat-client.tsx | 176 +++- .../[subdomain]/components/chat-client.tsx | 967 ------------------ .../[subdomain]/components/input/input.tsx | 261 +++-- .../components/input/voice-input.tsx | 114 +++ .../components/message/message.tsx | 15 +- .../voice-interface/components/particles.tsx | 503 +++++++++ .../voice-interface/voice-interface.tsx | 510 +++++++++ .../[subdomain]/hooks/use-audio-streaming.ts | 159 +++ .../[subdomain]/hooks/use-chat-streaming.ts | 203 +++- apps/sim/lib/env.ts | 1 + apps/sim/lib/utils.ts | 5 + apps/sim/package.json | 3 + bun.lock | 25 +- 14 files changed, 1950 insertions(+), 1110 deletions(-) create mode 100644 apps/sim/app/api/proxy/tts/stream/route.ts delete mode 100644 apps/sim/app/chat/[subdomain]/components/chat-client.tsx create mode 100644 apps/sim/app/chat/[subdomain]/components/input/voice-input.tsx create mode 100644 apps/sim/app/chat/[subdomain]/components/voice-interface/components/particles.tsx create mode 100644 apps/sim/app/chat/[subdomain]/components/voice-interface/voice-interface.tsx create mode 100644 apps/sim/app/chat/[subdomain]/hooks/use-audio-streaming.ts diff --git a/apps/sim/app/api/proxy/tts/stream/route.ts b/apps/sim/app/api/proxy/tts/stream/route.ts new file mode 100644 index 000000000..dde8f7f4b --- /dev/null +++ b/apps/sim/app/api/proxy/tts/stream/route.ts @@ -0,0 +1,118 @@ +import type { NextRequest } from 'next/server' +import { env } from '@/lib/env' +import { createLogger } from '@/lib/logs/console-logger' + +const logger = createLogger('ProxyTTSStreamAPI') + +export async function POST(request: NextRequest) { + try { + const body = await request.json() + const { text, voiceId, modelId = 'eleven_turbo_v2_5' } = body + + if (!text || !voiceId) { + return new Response('Missing required parameters', { status: 400 }) + } + + const apiKey = env.ELEVENLABS_API_KEY + if (!apiKey) { + logger.error('ELEVENLABS_API_KEY not configured on server') + return new Response('ElevenLabs service not configured', { status: 503 }) + } + + const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream` + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + Accept: 'audio/mpeg', + 'Content-Type': 'application/json', + 'xi-api-key': apiKey, + }, + body: JSON.stringify({ + text, + model_id: modelId, + // Maximum performance settings + optimize_streaming_latency: 4, + output_format: 'mp3_22050_32', // Fastest format + voice_settings: { + stability: 0.5, + similarity_boost: 0.8, + style: 0.0, + use_speaker_boost: false, + }, + enable_ssml_parsing: false, + apply_text_normalization: 'off', + // Use auto mode for fastest possible streaming + // Note: This may sacrifice some quality for speed + use_pvc_as_ivc: false, // Use fastest voice processing + }), + }) + + if (!response.ok) { + logger.error(`Failed to generate Stream TTS: ${response.status} ${response.statusText}`) + return new Response(`Failed to generate TTS: ${response.status} ${response.statusText}`, { + status: response.status, + }) + } + + if (!response.body) { + logger.error('No response body received from ElevenLabs') + return new Response('No audio stream received', { status: 422 }) + } + + // Create optimized streaming response + const { readable, writable } = new TransformStream({ + transform(chunk, controller) { + // Pass through chunks immediately without buffering + controller.enqueue(chunk) + }, + flush(controller) { + // Ensure all data is flushed immediately + controller.terminate() + }, + }) + + const writer = writable.getWriter() + const reader = response.body.getReader() + + ;(async () => { + try { + while (true) { + const { done, value } = await reader.read() + if (done) { + await writer.close() + break + } + // Write immediately without waiting + writer.write(value).catch(logger.error) + } + } catch (error) { + logger.error('Error during Stream streaming:', error) + await writer.abort(error) + } + })() + + return new Response(readable, { + headers: { + 'Content-Type': 'audio/mpeg', + 'Transfer-Encoding': 'chunked', + 'Cache-Control': 'no-cache, no-store, must-revalidate', + Pragma: 'no-cache', + Expires: '0', + 'X-Content-Type-Options': 'nosniff', + 'Access-Control-Allow-Origin': '*', + Connection: 'keep-alive', + // Stream headers for better streaming + 'X-Accel-Buffering': 'no', // Disable nginx buffering + 'X-Stream-Type': 'real-time', + }, + }) + } catch (error) { + logger.error('Error in Stream TTS:', error) + + return new Response( + `Internal Server Error: ${error instanceof Error ? error.message : 'Unknown error'}`, + { status: 500 } + ) + } +} diff --git a/apps/sim/app/chat/[subdomain]/chat-client.tsx b/apps/sim/app/chat/[subdomain]/chat-client.tsx index f9f27ba55..b28182c4d 100644 --- a/apps/sim/app/chat/[subdomain]/chat-client.tsx +++ b/apps/sim/app/chat/[subdomain]/chat-client.tsx @@ -2,6 +2,8 @@ import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' import { v4 as uuidv4 } from 'uuid' +import { createLogger } from '@/lib/logs/console-logger' +import { noop } from '@/lib/utils' import { getFormattedGitHubStars } from '@/app/(landing)/actions/github' import EmailAuth from './components/auth/email/email-auth' import PasswordAuth from './components/auth/password/password-auth' @@ -11,8 +13,12 @@ import { ChatInput } from './components/input/input' import { ChatLoadingState } from './components/loading-state/loading-state' import type { ChatMessage } from './components/message/message' import { ChatMessageContainer } from './components/message-container/message-container' +import { VoiceInterface } from './components/voice-interface/voice-interface' +import { useAudioStreaming } from './hooks/use-audio-streaming' import { useChatStreaming } from './hooks/use-chat-streaming' +const logger = createLogger('ChatClient') + interface ChatConfig { id: string title: string @@ -26,6 +32,39 @@ interface ChatConfig { authType?: 'public' | 'password' | 'email' } +interface AudioStreamingOptions { + voiceId: string + onError: (error: Error) => void +} + +const DEFAULT_VOICE_SETTINGS = { + voiceId: 'EXAVITQu4vr4xnSDxMaL', // Default ElevenLabs voice (Bella) +} + +/** + * Creates an audio stream handler for text-to-speech conversion + * @param streamTextToAudio - Function to stream text to audio + * @param voiceId - The voice ID to use for TTS + * @returns Audio stream handler function or undefined + */ +function createAudioStreamHandler( + streamTextToAudio: (text: string, options: AudioStreamingOptions) => Promise, + voiceId: string +) { + return async (text: string) => { + try { + await streamTextToAudio(text, { + voiceId, + onError: (error: Error) => { + logger.error('Audio streaming error:', error) + }, + }) + } catch (error) { + logger.error('TTS error:', error) + } + } +} + function throttle any>(func: T, delay: number): T { let timeoutId: NodeJS.Timeout | null = null let lastExecTime = 0 @@ -60,19 +99,17 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { const [starCount, setStarCount] = useState('3.4k') const [conversationId, setConversationId] = useState('') - // Simple state for showing scroll button const [showScrollButton, setShowScrollButton] = useState(false) - - // Track if user has manually scrolled during response const [userHasScrolled, setUserHasScrolled] = useState(false) const isUserScrollingRef = useRef(false) - // Authentication state const [authRequired, setAuthRequired] = useState<'password' | 'email' | null>(null) - // Use the custom streaming hook + const [isVoiceFirstMode, setIsVoiceFirstMode] = useState(false) const { isStreamingResponse, abortControllerRef, stopStreaming, handleStreamedResponse } = useChatStreaming() + const audioContextRef = useRef(null) + const { isPlayingAudio, streamTextToAudio, stopAudio } = useAudioStreaming(audioContextRef) const scrollToBottom = useCallback(() => { if (messagesEndRef.current) { @@ -193,7 +230,7 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { ]) } } catch (error) { - console.error('Error fetching chat config:', error) + logger.error('Error fetching chat config:', error) setError('This chat is currently unavailable. Please try again later.') } } @@ -208,7 +245,7 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { setStarCount(formattedStars) }) .catch((err) => { - console.error('Failed to fetch GitHub stars:', err) + logger.error('Failed to fetch GitHub stars:', err) }) }, [subdomain]) @@ -224,7 +261,7 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { } // Handle sending a message - const handleSendMessage = async (messageParam?: string) => { + const handleSendMessage = async (messageParam?: string, isVoiceInput = false) => { const messageToSend = messageParam ?? inputValue if (!messageToSend.trim() || isLoading) return @@ -278,18 +315,32 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { const contentType = response.headers.get('Content-Type') || '' if (contentType.includes('text/plain')) { - // Handle streaming response - pass the current userHasScrolled value + const shouldPlayAudio = isVoiceInput || isVoiceFirstMode + + const audioStreamHandler = shouldPlayAudio + ? createAudioStreamHandler(streamTextToAudio, DEFAULT_VOICE_SETTINGS.voiceId) + : undefined + + // Handle streaming response with audio support await handleStreamedResponse( response, setMessages, setIsLoading, scrollToBottom, - userHasScrolled + userHasScrolled, + { + voiceSettings: { + isVoiceEnabled: true, + voiceId: DEFAULT_VOICE_SETTINGS.voiceId, + autoPlayResponses: isVoiceInput || isVoiceFirstMode, + }, + audioStreamHandler, + } ) } else { // Fallback to JSON response handling const responseData = await response.json() - console.log('Message response:', responseData) + logger.info('Message response:', responseData) // Handle different response formats from API if ( @@ -321,6 +372,23 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { // Add all messages at once setMessages((prev) => [...prev, ...assistantMessages]) + + // Play audio for the full response if voice mode is enabled + if (isVoiceInput || isVoiceFirstMode) { + const fullContent = assistantMessages.map((m: ChatMessage) => m.content).join(' ') + if (fullContent.trim()) { + try { + await streamTextToAudio(fullContent, { + voiceId: DEFAULT_VOICE_SETTINGS.voiceId, + onError: (error) => { + logger.error('Audio playback error:', error) + }, + }) + } catch (error) { + logger.error('TTS error:', error) + } + } + } } else { // Handle single output as before let messageContent = responseData.output @@ -349,10 +417,29 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { } setMessages((prev) => [...prev, assistantMessage]) + + // Play audio for the response if voice mode is enabled + if ((isVoiceInput || isVoiceFirstMode) && assistantMessage.content) { + const contentString = + typeof assistantMessage.content === 'string' + ? assistantMessage.content + : JSON.stringify(assistantMessage.content) + + try { + await streamTextToAudio(contentString, { + voiceId: DEFAULT_VOICE_SETTINGS.voiceId, + onError: (error) => { + logger.error('Audio playback error:', error) + }, + }) + } catch (error) { + logger.error('TTS error:', error) + } + } } } } catch (error) { - console.error('Error sending message:', error) + logger.error('Error sending message:', error) const errorMessage: ChatMessage = { id: crypto.randomUUID(), @@ -367,6 +454,45 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { } } + // Stop audio when component unmounts or when streaming is stopped + useEffect(() => { + return () => { + stopAudio() + if (audioContextRef.current && audioContextRef.current.state !== 'closed') { + audioContextRef.current.close() + } + } + }, [stopAudio]) + + // Voice interruption - stop audio when user starts speaking + const handleVoiceInterruption = useCallback(() => { + stopAudio() + + // Stop any ongoing streaming response + if (isStreamingResponse) { + stopStreaming(setMessages) + } + }, [isStreamingResponse, stopStreaming, setMessages, stopAudio]) + + // Handle voice mode activation + const handleVoiceStart = useCallback(() => { + setIsVoiceFirstMode(true) + }, []) + + // Handle exiting voice mode + const handleExitVoiceMode = useCallback(() => { + setIsVoiceFirstMode(false) + stopAudio() // Stop any playing audio when exiting + }, [stopAudio]) + + // Handle voice transcript from voice-first interface + const handleVoiceTranscript = useCallback( + (transcript: string) => { + handleSendMessage(transcript, true) + }, + [handleSendMessage] + ) + // If error, show error message using the extracted component if (error) { return @@ -405,6 +531,27 @@ export default function ChatClient({ subdomain }: { subdomain: string }) { return } + // Voice-first mode interface + if (isVoiceFirstMode) { + return ( + ({ + content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), + type: msg.type, + }))} + /> + ) + } + + // Standard text-based chat interface return (
{/* Header component */} @@ -426,11 +573,12 @@ export default function ChatClient({ subdomain }: { subdomain: string }) {
{ - void handleSendMessage(value) + onSubmit={(value, isVoiceInput) => { + void handleSendMessage(value, isVoiceInput) }} isStreaming={isStreamingResponse} onStopStreaming={() => stopStreaming(setMessages)} + onVoiceStart={handleVoiceStart} />
diff --git a/apps/sim/app/chat/[subdomain]/components/chat-client.tsx b/apps/sim/app/chat/[subdomain]/components/chat-client.tsx deleted file mode 100644 index e2b31feff..000000000 --- a/apps/sim/app/chat/[subdomain]/components/chat-client.tsx +++ /dev/null @@ -1,967 +0,0 @@ -'use client' - -import { type KeyboardEvent, useEffect, useMemo, useRef, useState } from 'react' -import { ArrowUp, Loader2, Lock, Mail } from 'lucide-react' -import { v4 as uuidv4 } from 'uuid' -import { Button } from '@/components/ui/button' -import { Input } from '@/components/ui/input' -import { OTPInputForm } from '@/components/ui/input-otp-form' -import { getFormattedGitHubStars } from '@/app/(landing)/actions/github' -import HeaderLinks from './components/header-links/header-links' -import MarkdownRenderer from './components/markdown-renderer/markdown-renderer' - -// Define message type -interface ChatMessage { - id: string - content: string - type: 'user' | 'assistant' - timestamp: Date -} - -// Define chat config type -interface ChatConfig { - id: string - title: string - description: string - customizations: { - primaryColor?: string - logoUrl?: string - welcomeMessage?: string - headerText?: string - } - authType?: 'public' | 'password' | 'email' -} - -// ChatGPT-style message component -function ClientChatMessage({ message }: { message: ChatMessage }) { - // Check if content is a JSON object - const isJsonObject = useMemo(() => { - return typeof message.content === 'object' && message.content !== null - }, [message.content]) - - // For user messages (on the right) - if (message.type === 'user') { - return ( -
-
-
-
-
- {isJsonObject ? ( - 
{JSON.stringify(message.content, null, 2)}
- ) : ( - {message.content} - )} -
-
-
-
-
- ) - } - - // For assistant messages (on the left) - return ( -
-
-
-
-
- {isJsonObject ? ( - 
{JSON.stringify(message.content, null, 2)}
- ) : ( - - )} -
-
-
-
-
- ) -} - -export default function ChatClient({ subdomain }: { subdomain: string }) { - const [messages, setMessages] = useState([]) - const [inputValue, setInputValue] = useState('') - const [isLoading, setIsLoading] = useState(false) - const [chatConfig, setChatConfig] = useState(null) - const [error, setError] = useState(null) - const messagesEndRef = useRef(null) - const messagesContainerRef = useRef(null) - const inputRef = useRef(null) - const [starCount, setStarCount] = useState('3.4k') - const [conversationId, setConversationId] = useState('') - - // Authentication state - const [authRequired, setAuthRequired] = useState<'password' | 'email' | null>(null) - const [password, setPassword] = useState('') - const [email, setEmail] = useState('') - const [authError, setAuthError] = useState(null) - const [isAuthenticating, setIsAuthenticating] = useState(false) - - // OTP verification state - const [showOtpVerification, setShowOtpVerification] = useState(false) - const [otpValue, setOtpValue] = useState('') - const [isSendingOtp, setIsSendingOtp] = useState(false) - const [isVerifyingOtp, setIsVerifyingOtp] = useState(false) - - // Fetch chat config function - const fetchChatConfig = async () => { - try { - // Use relative URL instead of absolute URL with env.NEXT_PUBLIC_APP_URL - const response = await fetch(`/api/chat/${subdomain}`, { - credentials: 'same-origin', - headers: { - 'X-Requested-With': 'XMLHttpRequest', - }, - }) - - if (!response.ok) { - // Check if auth is required - if (response.status === 401) { - const errorData = await response.json() - - if (errorData.error === 'auth_required_password') { - setAuthRequired('password') - return - } - if (errorData.error === 'auth_required_email') { - setAuthRequired('email') - return - } - } - - throw new Error(`Failed to load chat configuration: ${response.status}`) - } - - const data = await response.json() - - // The API returns the data directly without a wrapper - setChatConfig(data) - - // Add welcome message if configured - if (data?.customizations?.welcomeMessage) { - setMessages([ - { - id: 'welcome', - content: data.customizations.welcomeMessage, - type: 'assistant', - timestamp: new Date(), - }, - ]) - } - } catch (error) { - console.error('Error fetching chat config:', error) - setError('This chat is currently unavailable. Please try again later.') - } - } - - // Fetch chat config on mount and generate new conversation ID - useEffect(() => { - fetchChatConfig() - // Generate a new conversation ID whenever the page/chat is refreshed - setConversationId(uuidv4()) - - // Fetch GitHub stars - getFormattedGitHubStars() - .then((formattedStars) => { - setStarCount(formattedStars) - }) - .catch((err) => { - console.error('Failed to fetch GitHub stars:', err) - }) - }, [subdomain]) - - // Handle keyboard input for message sending - const handleKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Enter' && !e.shiftKey) { - e.preventDefault() - handleSendMessage() - } - } - - // Handle keyboard input for auth forms - const _handleAuthKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Enter') { - e.preventDefault() - handleAuthenticate() - } - } - - // Handle authentication - const handleAuthenticate = async () => { - if (authRequired === 'password') { - // Password auth remains the same - setAuthError(null) - setIsAuthenticating(true) - - try { - const payload = { password } - - const response = await fetch(`/api/chat/${subdomain}`, { - method: 'POST', - credentials: 'same-origin', - headers: { - 'Content-Type': 'application/json', - 'X-Requested-With': 'XMLHttpRequest', - }, - body: JSON.stringify(payload), - }) - - if (!response.ok) { - const errorData = await response.json() - setAuthError(errorData.error || 'Authentication failed') - return - } - - await response.json() - - // Authentication successful, fetch config again - await fetchChatConfig() - - // Reset auth state - setAuthRequired(null) - setPassword('') - } catch (error) { - console.error('Authentication error:', error) - setAuthError('An error occurred during authentication') - } finally { - setIsAuthenticating(false) - } - } else if (authRequired === 'email') { - // For email auth, we now send an OTP first - if (!showOtpVerification) { - // Step 1: User has entered email, send OTP - setAuthError(null) - setIsSendingOtp(true) - - try { - const response = await fetch(`/api/chat/${subdomain}/otp`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Requested-With': 'XMLHttpRequest', - }, - body: JSON.stringify({ email }), - }) - - if (!response.ok) { - const errorData = await response.json() - setAuthError(errorData.error || 'Failed to send verification code') - return - } - - // OTP sent successfully, show OTP input - setShowOtpVerification(true) - } catch (error) { - console.error('Error sending OTP:', error) - setAuthError('An error occurred while sending the verification code') - } finally { - setIsSendingOtp(false) - } - } else { - // Step 2: User has entered OTP, verify it - setAuthError(null) - setIsVerifyingOtp(true) - - try { - const response = await fetch(`/api/chat/${subdomain}/otp`, { - method: 'PUT', - headers: { - 'Content-Type': 'application/json', - 'X-Requested-With': 'XMLHttpRequest', - }, - body: JSON.stringify({ email, otp: otpValue }), - }) - - if (!response.ok) { - const errorData = await response.json() - setAuthError(errorData.error || 'Invalid verification code') - return - } - - await response.json() - - // OTP verified successfully, fetch config again - await fetchChatConfig() - - // Reset auth state - setAuthRequired(null) - setEmail('') - setOtpValue('') - setShowOtpVerification(false) - } catch (error) { - console.error('Error verifying OTP:', error) - setAuthError('An error occurred during verification') - } finally { - setIsVerifyingOtp(false) - } - } - } - } - - // Add this function to handle resending OTP - const handleResendOtp = async () => { - setAuthError(null) - setIsSendingOtp(true) - - try { - const response = await fetch(`/api/chat/${subdomain}/otp`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Requested-With': 'XMLHttpRequest', - }, - body: JSON.stringify({ email }), - }) - - if (!response.ok) { - const errorData = await response.json() - setAuthError(errorData.error || 'Failed to resend verification code') - return - } - - // Show a message that OTP was sent - setAuthError('Verification code sent. Please check your email.') - } catch (error) { - console.error('Error resending OTP:', error) - setAuthError('An error occurred while resending the verification code') - } finally { - setIsSendingOtp(false) - } - } - - // Add a function to handle email input key down - const handleEmailKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Enter') { - e.preventDefault() - handleAuthenticate() - } - } - - // Add a function to handle OTP input key down - const _handleOtpKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Enter') { - e.preventDefault() - handleAuthenticate() - } - } - - // Scroll to bottom of messages - useEffect(() => { - if (messagesEndRef.current) { - messagesEndRef.current.scrollIntoView({ behavior: 'smooth' }) - } - }, [messages]) - - // Handle sending a message - const handleSendMessage = async () => { - if (!inputValue.trim() || isLoading) return - - const userMessage: ChatMessage = { - id: crypto.randomUUID(), - content: inputValue, - type: 'user', - timestamp: new Date(), - } - - setMessages((prev) => [...prev, userMessage]) - setInputValue('') - setIsLoading(true) - - // Ensure focus remains on input field - if (inputRef.current) { - inputRef.current.focus() - } - - try { - // Send structured payload to maintain chat context - const payload = { - message: userMessage.content, - conversationId, - } - - // Use relative URL with credentials - const response = await fetch(`/api/chat/${subdomain}`, { - method: 'POST', - credentials: 'same-origin', - headers: { - 'Content-Type': 'application/json', - 'X-Requested-With': 'XMLHttpRequest', - }, - body: JSON.stringify(payload), - }) - - if (!response.ok) { - throw new Error('Failed to get response') - } - - // Detect streaming response via content-type (text/plain) or absence of JSON content-type - const contentType = response.headers.get('Content-Type') || '' - - if (contentType.includes('text/plain')) { - // Handle streaming response - const messageId = crypto.randomUUID() - - // Add placeholder message - setMessages((prev) => [ - ...prev, - { - id: messageId, - content: '', - type: 'assistant', - timestamp: new Date(), - }, - ]) - - // Stop showing loading indicator once streaming begins - setIsLoading(false) - - // Ensure the response body exists and is a ReadableStream - const reader = response.body?.getReader() - if (reader) { - const decoder = new TextDecoder() - let done = false - while (!done) { - const { value, done: readerDone } = await reader.read() - if (value) { - const chunk = decoder.decode(value, { stream: true }) - if (chunk) { - setMessages((prev) => - prev.map((msg) => - msg.id === messageId ? { ...msg, content: msg.content + chunk } : msg - ) - ) - } - } - done = readerDone - } - } - } else { - // Fallback to JSON response handling - const responseData = await response.json() - console.log('Message response:', responseData) - - // Handle different response formats from API - if ( - responseData.multipleOutputs && - responseData.contents && - Array.isArray(responseData.contents) - ) { - // For multiple outputs, create separate assistant messages for each - const assistantMessages = responseData.contents.map((content: any) => { - // Format the content appropriately - let formattedContent = content - - // Convert objects to strings for display - if (typeof formattedContent === 'object' && formattedContent !== null) { - try { - formattedContent = JSON.stringify(formattedContent) - } catch (_e) { - formattedContent = 'Received structured data response' - } - } - - return { - id: crypto.randomUUID(), - content: formattedContent || 'No content found', - type: 'assistant' as const, - timestamp: new Date(), - } - }) - - // Add all messages at once - setMessages((prev) => [...prev, ...assistantMessages]) - } else { - // Handle single output as before - let messageContent = responseData.output - - if (!messageContent && responseData.content) { - if (typeof responseData.content === 'object') { - if (responseData.content.text) { - messageContent = responseData.content.text - } else { - try { - messageContent = JSON.stringify(responseData.content) - } catch (_e) { - messageContent = 'Received structured data response' - } - } - } else { - messageContent = responseData.content - } - } - - const assistantMessage: ChatMessage = { - id: crypto.randomUUID(), - content: messageContent || "Sorry, I couldn't process your request.", - type: 'assistant', - timestamp: new Date(), - } - - setMessages((prev) => [...prev, assistantMessage]) - } - } - } catch (error) { - console.error('Error sending message:', error) - - const errorMessage: ChatMessage = { - id: crypto.randomUUID(), - content: 'Sorry, there was an error processing your message. Please try again.', - type: 'assistant', - timestamp: new Date(), - } - - setMessages((prev) => [...prev, errorMessage]) - } finally { - setIsLoading(false) - // Ensure focus remains on input field even after the response - if (inputRef.current) { - inputRef.current.focus() - } - } - } - - // If error, show error message - if (error) { - return ( -
-
- -

Error

-

{error}

-
-
- ) - } - - // If authentication is required, show auth form - if (authRequired) { - // Get title and description from the URL params or use defaults - const title = new URLSearchParams(window.location.search).get('title') || 'chat' - const _primaryColor = new URLSearchParams(window.location.search).get('color') || '#802FFF' - - return ( -
-
- -
-

{title}

-

- {authRequired === 'password' - ? 'This chat is password-protected. Please enter the password to continue.' - : 'This chat requires email verification. Please enter your email to continue.'} -

-
- - {authError && ( -
- {authError} -
- )} - -
- {authRequired === 'password' ? ( -
-
-
-
- -
-
- -

Password Required

-

- Enter the password to access this chat -

- -
{ - e.preventDefault() - handleAuthenticate() - }} - > -
-
- - setPassword(e.target.value)} - placeholder='Enter password' - disabled={isAuthenticating} - className='w-full' - /> -
- - {authError && ( -
{authError}
- )} - - -
-
-
-
- ) : ( -
-
-
-
- -
-
- -

Email Verification

- - {!showOtpVerification ? ( - // Step 1: Email Input - <> -

- Enter your email address to access this chat -

- -
-
- - setEmail(e.target.value)} - onKeyDown={handleEmailKeyDown} - disabled={isSendingOtp || isAuthenticating} - className='w-full' - /> -
- - {authError && ( -
{authError}
- )} - - -
- - ) : ( - // Step 2: OTP Verification with OTPInputForm - <> -

- Enter the verification code sent to -

-

{email}

- - { - setOtpValue(value) - handleAuthenticate() - }} - isLoading={isVerifyingOtp} - error={authError} - /> - -
- - • - -
- - )} -
-
- )} -
-
-
- ) - } - - // Loading state while fetching config - if (!chatConfig) { - return ( -
-
-
-
-
-
- ) - } - - return ( -
- - - {/* Header with title and links */} -
-
- {chatConfig?.customizations?.logoUrl && ( - {`${chatConfig?.title - )} -

- {chatConfig?.customizations?.headerText || chatConfig?.title || 'Chat'} -

-
-
- - {!chatConfig?.customizations?.logoUrl && ( - - - - - - - - - - - )} -
-
- - {/* Messages container */} -
-
- {messages.length === 0 ? ( -
-
-

How can I help you today?

-

- {chatConfig.description || 'Ask me anything.'} -

-
-
- ) : ( - messages.map((message) => ) - )} - - {/* Loading indicator (shows only when executing) */} - {isLoading && ( -
-
-
-
-
-
-
-
-
-
-
- )} - -
-
-
- - {/* Input area (fixed at bottom) */} -
-
-
- setInputValue(e.target.value)} - onKeyDown={handleKeyDown} - placeholder='Message...' - className='min-h-[50px] flex-1 rounded-2xl border-0 bg-transparent py-7 pr-16 pl-6 text-base focus-visible:ring-0 focus-visible:ring-offset-0' - /> - -
-
-
-
- ) -} diff --git a/apps/sim/app/chat/[subdomain]/components/input/input.tsx b/apps/sim/app/chat/[subdomain]/components/input/input.tsx index c8b0d4413..126d2de99 100644 --- a/apps/sim/app/chat/[subdomain]/components/input/input.tsx +++ b/apps/sim/app/chat/[subdomain]/components/input/input.tsx @@ -4,8 +4,10 @@ import type React from 'react' import { useEffect, useRef, useState } from 'react' import { motion } from 'framer-motion' import { Send, Square } from 'lucide-react' +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' +import { VoiceInput } from './voice-input' -const PLACEHOLDER = 'Enter a message' +const PLACEHOLDER = 'Enter a message or click the mic to speak' const MAX_TEXTAREA_HEIGHT = 160 // Max height in pixels (e.g., for about 4-5 lines) const containerVariants = { @@ -20,15 +22,21 @@ const containerVariants = { } as const export const ChatInput: React.FC<{ - onSubmit?: (value: string) => void + onSubmit?: (value: string, isVoiceInput?: boolean) => void isStreaming?: boolean onStopStreaming?: () => void -}> = ({ onSubmit, isStreaming = false, onStopStreaming }) => { + onVoiceStart?: () => void + voiceOnly?: boolean +}> = ({ onSubmit, isStreaming = false, onStopStreaming, onVoiceStart, voiceOnly = false }) => { const wrapperRef = useRef(null) const textareaRef = useRef(null) // Ref for the textarea const [isActive, setIsActive] = useState(false) const [inputValue, setInputValue] = useState('') + // Check if speech-to-text is available in the browser + const isSttAvailable = + typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition) + // Function to adjust textarea height const adjustTextareaHeight = () => { if (textareaRef.current) { @@ -84,7 +92,7 @@ export const ChatInput: React.FC<{ const handleSubmit = () => { if (!inputValue.trim()) return - onSubmit?.(inputValue.trim()) + onSubmit?.(inputValue.trim(), false) // false = not voice input setInputValue('') if (textareaRef.current) { textareaRef.current.style.height = 'auto' // Reset height after submit @@ -97,114 +105,143 @@ export const ChatInput: React.FC<{ setInputValue(e.target.value) } - return ( -
- -
- {/* Text Input & Placeholder */} -
-