import React, { useState, useEffect, useRef } from "react"; import { useFireproof } from "use-fireproof"; import { callAI } from "call-ai"; export default function App() { const { useDocument, useLiveQuery, database } = useFireproof("video-call-webrtc-robust"); const [localStream, setLocalStream] = useState(null); const [remoteStream, setRemoteStream] = useState(null); const [recording, setRecording] = useState(false); const [transcribing, setTranscribing] = useState(false); const [mediaRecorder, setMediaRecorder] = useState(null); const [audioChunks, setAudioChunks] = useState([]); const [peerConnection, setPeerConnection] = useState(null); const [callStatus, setCallStatus] = useState('idle'); // idle, creating, connecting, connected, reconnecting, ended const [connectionError, setConnectionError] = useState(null); const [copiedToClipboard, setCopiedToClipboard] = useState(false); const [callIdInput, setCallIdInput] = useState(""); const [isInitiator, setIsInitiator] = useState(false); const [reconnectAttempts, setReconnectAttempts] = useState(0); const [networkStats, setNetworkStats] = useState({ quality: 'unknown' }); const localVideoRef = useRef(null); const remoteVideoRef = useRef(null); const pendingCandidatesRef = useRef([]); const pollingIntervalRef = useRef(null); const statsIntervalRef = useRef(null); const { doc: currentCall, merge: mergeCall, submit: submitCall, reset: resetCall } = useDocument({ type: "call", name: "", startTime: null, endTime: null, transcription: [], offer: null, answer: null, iceCandidates: [], lastUpdated: Date.now(), reconnectCounter: 0 }); const { docs: callHistory } = useLiveQuery("type", { key: "call", descending: true, limit: 10 }); // Configuration for ICE servers including public STUN and TURN fallbacks const iceServers = [ { urls: 'stun:stun.l.google.com:19302' }, { urls: 'stun:stun1.l.google.com:19302' }, { urls: 'stun:stun2.l.google.com:19302' }, { urls: 'stun:stun3.l.google.com:19302' }, { urls: 'stun:stun4.l.google.com:19302' }, { urls: 'turn:openrelay.metered.ca:80', username: 'openrelayproject', credential: 'openrelayproject' }, { urls: 'turn:openrelay.metered.ca:443', username: 'openrelayproject', credential: 'openrelayproject' } ]; // Check for document updates to handle connection setup and reconnection useEffect(() => { const now = Date.now(); // Set up initial connection if (currentCall.offer && !peerConnection && callStatus === 'idle') { console.log("Found offer in document, ready to join"); } // Handle answer received if (currentCall.answer && callStatus === 'connecting' && peerConnection && isInitiator) { console.log("Answer received, processing..."); (async () => { try { await handleReceivedAnswer(); } catch (error) { console.error("Error handling answer:", error); } })(); } // Handle reconnection attempts from peer if (peerConnection && currentCall.reconnectCounter && currentCall.lastUpdated > now - 10000 && // Within the last 10 seconds callStatus !== 'idle' && callStatus !== 'ended') { console.log("Peer is requesting reconnection:", currentCall.reconnectCounter); // Only the non-initiator should recreate connection on even counts // Initiator handles odd counts const shouldHandle = isInitiator ? (currentCall.reconnectCounter % 2 !== 0) : (currentCall.reconnectCounter % 2 === 0); if (shouldHandle && callStatus !== 'reconnecting') { console.log("Handling reconnection request"); setCallStatus('reconnecting'); recreateConnection(); } } }, [currentCall.offer, currentCall.answer, currentCall.reconnectCounter, currentCall.lastUpdated]); // Set up polling for connection health useEffect(() => { if (callStatus === 'connected' || callStatus === 'connecting') { // Check connection health every 5 seconds if (!pollingIntervalRef.current) { pollingIntervalRef.current = setInterval(() => { if (peerConnection) { // Update last active timestamp to show we're still here mergeCall({ lastUpdated: Date.now() }); submitCall(); // Check connection state if (peerConnection.iceConnectionState === 'disconnected' || peerConnection.iceConnectionState === 'failed' || peerConnection.connectionState === 'failed') { console.log("Connection issues detected in polling, attempting to reconnect"); setConnectionError("Connection issues detected, attempting to reconnect..."); initiateReconnection(); } } }, 5000); } } else { // Clear polling when not in active call if (pollingIntervalRef.current) { clearInterval(pollingIntervalRef.current); pollingIntervalRef.current = null; } } return () => { if (pollingIntervalRef.current) { clearInterval(pollingIntervalRef.current); pollingIntervalRef.current = null; } }; }, [callStatus, peerConnection]); // Set up connection statistics monitoring useEffect(() => { if (peerConnection && callStatus === 'connected') { if (!statsIntervalRef.current) { statsIntervalRef.current = setInterval(async () => { try { const stats = await peerConnection.getStats(); let inboundRtp = null; let outboundRtp = null; let remoteInbound = null; stats.forEach(report => { if (report.type === 'inbound-rtp' && report.kind === 'video') { inboundRtp = report; } else if (report.type === 'outbound-rtp' && report.kind === 'video') { outboundRtp = report; } else if (report.type === 'remote-inbound-rtp' && report.kind === 'video') { remoteInbound = report; } }); // Calculate network quality let quality = 'unknown'; let roundTripTime = 0; let packetLoss = 0; if (remoteInbound) { roundTripTime = remoteInbound.roundTripTime || 0; packetLoss = remoteInbound.fractionLost || 0; } if (roundTripTime < 0.1 && packetLoss < 0.01) { quality = 'excellent'; } else if (roundTripTime < 0.3 && packetLoss < 0.05) { quality = 'good'; } else if (roundTripTime < 0.5 && packetLoss < 0.1) { quality = 'fair'; } else if (roundTripTime || packetLoss) { quality = 'poor'; } setNetworkStats({ quality, roundTripTime: roundTripTime ? `${Math.round(roundTripTime * 1000)}ms` : 'unknown', packetLoss: packetLoss ? `${Math.round(packetLoss * 100)}%` : 'unknown' }); } catch (error) { console.error("Error getting connection stats:", error); } }, 2000); } } else { if (statsIntervalRef.current) { clearInterval(statsIntervalRef.current); statsIntervalRef.current = null; } } return () => { if (statsIntervalRef.current) { clearInterval(statsIntervalRef.current); statsIntervalRef.current = null; } }; }, [peerConnection, callStatus]); // Handle ICE candidates useEffect(() => { if (peerConnection && currentCall.iceCandidates && currentCall.iceCandidates.length > 0) { (async () => { try { // Apply any pending candidates if remote description is set if (peerConnection.remoteDescription) { console.log("Processing ICE candidates:", currentCall.iceCandidates.length); const candidates = [...pendingCandidatesRef.current, ...currentCall.iceCandidates]; pendingCandidatesRef.current = []; for (const candidate of candidates) { if (candidate) { try { await peerConnection.addIceCandidate(new RTCIceCandidate(candidate)); } catch (err) { console.warn("Failed to add ICE candidate, may be duplicate:", err); } } } } else { // Store candidates for later when remote description is set console.log("Storing ICE candidates for later:", currentCall.iceCandidates.length); pendingCandidatesRef.current = [...pendingCandidatesRef.current, ...currentCall.iceCandidates]; } } catch (error) { console.error("Error processing ICE candidates:", error); } })(); } }, [currentCall.iceCandidates, peerConnection]); // Initialize or reinitialize local media stream const startLocalStream = async () => { try { // Close existing stream if any if (localStream) { localStream.getTracks().forEach(track => track.stop()); } // Get new stream const constraints = { video: { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } }, audio: true }; const stream = await navigator.mediaDevices.getUserMedia(constraints); setLocalStream(stream); if (localVideoRef.current) { localVideoRef.current.srcObject = stream; } // Setup media recorder for audio transcription const recorder = new MediaRecorder(stream); setMediaRecorder(recorder); recorder.ondataavailable = (e) => { if (e.data.size > 0) { setAudioChunks((chunks) => [...chunks, e.data]); } }; recorder.onstop = async () => { if (audioChunks.length > 0) { const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); await transcribeAudio(audioBlob); } }; return stream; } catch (error) { console.error("Error accessing media devices:", error); setConnectionError("Failed to access camera and microphone. Please check permissions."); throw error; } }; // Create a new WebRTC peer connection const createPeerConnection = () => { try { // Create with ICE servers configuration const pc = new RTCPeerConnection({ iceServers }); // Set up event handlers setupPeerConnectionHandlers(pc); return pc; } catch (error) { console.error("Error creating peer connection:", error); setConnectionError(`Failed to create connection: ${error.message}`); throw error; } }; // Create a new call as the initiator const createCall = async (callName) => { try { setCallStatus('creating'); setConnectionError(null); setIsInitiator(true); pendingCandidatesRef.current = []; // Get media stream const stream = await startLocalStream(); // Create a new peer connection const pc = createPeerConnection(); // Add local tracks to the connection stream.getTracks().forEach(track => { pc.addTrack(track, stream); }); // Create and set local description (offer) const offer = await pc.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: true }); await pc.setLocalDescription(offer); console.log("Local description set (offer)"); // Save the peer connection setPeerConnection(pc); // Wait for ICE gathering await new Promise(resolve => { const checkState = () => { if (pc.iceGatheringState === 'complete') { console.log("ICE gathering complete"); resolve(); } else { setTimeout(checkState, 500); } }; setTimeout(checkState, 500); }); // Save the call to Fireproof mergeCall({ name: callName, startTime: Date.now(), lastUpdated: Date.now(), offer: pc.localDescription, iceCandidates: [], reconnectCounter: 0 }); submitCall(); setCallStatus('connecting'); console.log("Call created, waiting for peer"); } catch (error) { console.error("Error creating call:", error); setConnectionError(`Failed to create call: ${error.message}`); setCallStatus('idle'); } }; // Join an existing call as the receiver const joinCall = async () => { try { setCallStatus('connecting'); setConnectionError(null); setIsInitiator(false); pendingCandidatesRef.current = []; if (!currentCall.offer) { throw new Error("Cannot join call - no offer available"); } // Get media stream const stream = await startLocalStream(); // Create a new peer connection const pc = createPeerConnection(); // Add local tracks stream.getTracks().forEach(track => { pc.addTrack(track, stream); }); // Set remote description from the offer await pc.setRemoteDescription(new RTCSessionDescription(currentCall.offer)); console.log("Remote description set from offer"); // Create answer const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); console.log("Local description set (answer)"); // Save peer connection setPeerConnection(pc); // Wait for ICE gathering (with timeout) await Promise.race([ new Promise(resolve => { const checkState = () => { if (pc.iceGatheringState === 'complete') { console.log("ICE gathering complete"); resolve(); } else { setTimeout(checkState, 500); } }; setTimeout(checkState, 500); }), new Promise(resolve => setTimeout(resolve, 5000)) // 5 second timeout ]); // Send answer mergeCall({ answer: pc.localDescription, lastUpdated: Date.now() }); submitCall(); console.log("Answer sent"); // Process any pending ICE candidates if (pendingCandidatesRef.current.length > 0) { console.log("Processing pending ICE candidates", pendingCandidatesRef.current.length); for (const candidate of pendingCandidatesRef.current) { try { await pc.addIceCandidate(new RTCIceCandidate(candidate)); } catch (err) { console.warn("Failed to add pending candidate:", err); } } pendingCandidatesRef.current = []; } } catch (error) { console.error("Error joining call:", error); setConnectionError(`Failed to join call: ${error.message}`); setCallStatus('idle'); } }; // Handle a received answer when we're the caller const handleReceivedAnswer = async () => { try { if (peerConnection && currentCall.answer && isInitiator) { console.log("Setting remote description from answer"); await peerConnection.setRemoteDescription(new RTCSessionDescription(currentCall.answer)); // Process any pending ICE candidates now that remote description is set if (pendingCandidatesRef.current.length > 0) { console.log("Processing pending ICE candidates after receiving answer:", pendingCandidatesRef.current.length); for (const candidate of pendingCandidatesRef.current) { try { await peerConnection.addIceCandidate(new RTCIceCandidate(candidate)); } catch (err) { console.warn("Failed to add pending candidate:", err); } } pendingCandidatesRef.current = []; } } } catch (error) { console.error("Error handling received answer:", error); setConnectionError(`Connection error: ${error.message}`); // If there's an error handling the answer, attempt to reconnect if (reconnectAttempts < 3) { initiateReconnection(); } } }; // Set up event handlers for the peer connection const setupPeerConnectionHandlers = (pc) => { // Handle ICE candidates generation pc.onicecandidate = (event) => { if (event.candidate) { console.log("New ICE candidate generated"); // Save ICE candidate to Fireproof const updatedCandidates = [...(currentCall.iceCandidates || []), event.candidate]; mergeCall({ iceCandidates: updatedCandidates, lastUpdated: Date.now() }); submitCall(); } }; // Handle ICE gathering state changes pc.onicegatheringstatechange = () => { console.log("ICE gathering state:", pc.iceGatheringState); }; // Handle ICE connection state changes pc.oniceconnectionstatechange = () => { console.log("ICE connection state:", pc.iceConnectionState); if (pc.iceConnectionState === 'checking') { console.log("Checking ICE connection..."); } else if (pc.iceConnectionState === 'connected' || pc.iceConnectionState === 'completed') { console.log("ICE Connected successfully"); setCallStatus('connected'); setConnectionError(null); setReconnectAttempts(0); } else if (pc.iceConnectionState === 'disconnected') { console.log("ICE Connection disconnected - may recover automatically"); setConnectionError("Connection interrupted. Attempting to recover..."); // Give it a moment to recover on its own setTimeout(() => { if (pc.iceConnectionState === 'disconnected') { initiateReconnection(); } }, 5000); } else if (pc.iceConnectionState === 'failed') { console.log("ICE Connection failed"); setConnectionError("Connection failed. Attempting to reconnect..."); initiateReconnection(); } else if (pc.iceConnectionState === 'closed') { console.log("ICE Connection closed"); } }; // Handle peer connection state changes pc.onconnectionstatechange = () => { console.log("Connection state changed:", pc.connectionState); if (pc.connectionState === 'connected') { setCallStatus('connected'); setConnectionError(null); setReconnectAttempts(0); } else if (pc.connectionState === 'disconnected') { setConnectionError("Connection disconnected. Attempting to recover..."); } else if (pc.connectionState === 'failed') { setConnectionError("Connection failed. Attempting to reconnect..."); initiateReconnection(); } }; // Handle negotiation needed events pc.onnegotiationneeded = async () => { console.log("Negotiation needed event"); // Only the initiator should handle renegotiation automatically if (isInitiator && pc.signalingState === 'stable' && callStatus === 'connected') { try { const offer = await pc.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: true }); await pc.setLocalDescription(offer); // Update the offer in the database mergeCall({ offer: pc.localDescription, lastUpdated: Date.now() }); submitCall(); console.log("Renegotiation offer created and sent"); } catch (error) { console.error("Failed to create renegotiation offer:", error); } } }; // Handle incoming tracks pc.ontrack = (event) => { console.log("Remote track received", event.streams[0]); setRemoteStream(event.streams[0]); if (remoteVideoRef.current) { remoteVideoRef.current.srcObject = event.streams[0]; } }; }; // Initiate reconnection in case of connection failure const initiateReconnection = () => { if (callStatus === 'reconnecting' || callStatus === 'ended') { return; // Avoid multiple simultaneous reconnection attempts } if (reconnectAttempts >= 3) { setConnectionError("Could not reconnect after multiple attempts. Please try joining the call again."); return; } setCallStatus('reconnecting'); setReconnectAttempts(prev => prev + 1); console.log(`Initiating reconnection attempt ${reconnectAttempts + 1}`); // Update the reconnect counter in the database to signal to the other peer mergeCall({ reconnectCounter: (currentCall.reconnectCounter || 0) + 1, lastUpdated: Date.now() }); submitCall(); // Recreate the connection after a brief delay to allow for state updates setTimeout(() => { recreateConnection(); }, 1000); }; // Recreate the WebRTC connection const recreateConnection = async () => { try { console.log("Recreating peer connection"); setConnectionError("Reconnecting, please wait..."); // Close existing connection if (peerConnection) { peerConnection.close(); } // Create a new peer connection const pc = createPeerConnection(); // Add local tracks from existing stream or get a new one let stream = localStream; if (!stream || stream.getTracks().length === 0) { stream = await startLocalStream(); } stream.getTracks().forEach(track => { pc.addTrack(track, stream); }); // Handle reconnection differently based on initiator role if (isInitiator) { // Create new offer const offer = await pc.createOffer({ offerToReceiveAudio: true, offerToReceiveVideo: true }); await pc.setLocalDescription(offer); // Update the offer and clear the answer mergeCall({ offer: pc.localDescription, answer: null, // Clear existing answer iceCandidates: [], // Clear existing ICE candidates lastUpdated: Date.now() }); submitCall(); // Save peer connection setPeerConnection(pc); console.log("Recreated and sent new offer"); } else { // As responder, if we have an offer, set it as remote description if (currentCall.offer) { await pc.setRemoteDescription(new RTCSessionDescription(currentCall.offer)); // Create new answer const answer = await pc.createAnswer(); await pc.setLocalDescription(answer); // Update the answer mergeCall({ answer: pc.localDescription, iceCandidates: [], // Clear existing ICE candidates lastUpdated: Date.now() }); submitCall(); // Save peer connection setPeerConnection(pc); console.log("Recreated and sent new answer"); } } } catch (error) { console.error("Error recreating connection:", error); setConnectionError(`Reconnection error: ${error.message}`); // If we fail after multiple attempts, suggest rejoining if (reconnectAttempts >= 2) { setCallStatus('ended'); setConnectionError("Connection could not be restored. Please try joining the call again."); } } }; // End the call and clean up const endCall = () => { // Stop the media streams if (localStream) { localStream.getTracks().forEach(track => track.stop()); } // Close the peer connection if (peerConnection) { peerConnection.close(); setPeerConnection(null); } // Stop recording if active if (mediaRecorder && recording) { mediaRecorder.stop(); setRecording(false); setTranscribing(false); } // Reset video elements if (localVideoRef.current) { localVideoRef.current.srcObject = null; } if (remoteVideoRef.current) { remoteVideoRef.current.srcObject = null; } // Reset state setLocalStream(null); setRemoteStream(null); setCallStatus('idle'); pendingCandidatesRef.current = []; setIsInitiator(false); setReconnectAttempts(0); setNetworkStats({ quality: 'unknown' }); // Save call end time if we have an ID if (currentCall._id) { mergeCall({ endTime: Date.now() }); submitCall(); } // Clear audio chunks setAudioChunks([]); // Clear intervals if (pollingIntervalRef.current) { clearInterval(pollingIntervalRef.current); pollingIntervalRef.current = null; } if (statsIntervalRef.current) { clearInterval(statsIntervalRef.current); statsIntervalRef.current = null; } }; // Toggle recording for transcription const toggleRecording = () => { if (mediaRecorder) { if (!recording) { setAudioChunks([]); mediaRecorder.start(1000); // Capture in 1-second chunks setRecording(true); setTranscribing(true); } else { mediaRecorder.stop(); setRecording(false); setTranscribing(false); } } }; // Transcribe audio using AI const transcribeAudio = async (audioBlob) => { try { // Convert the audio blob to base64 const reader = new FileReader(); reader.readAsDataURL(audioBlob); reader.onloadend = async () => { const base64Audio = reader.result.split(',')[1]; // Call AI for transcription const response = await callAI(`Please transcribe this audio snippet from a video call: ${base64Audio.substring(0, 100)}... (audio data truncated)`, { schema: { properties: { transcription: { type: "string", description: "Transcribed text from the audio" } } } }); try { const result = JSON.parse(response); if (result.transcription) { // Add the transcription to the call document mergeCall({ transcription: [...currentCall.transcription, { timestamp: Date.now(), text: result.transcription }] }); submitCall(); } } catch (e) { console.error("Failed to parse transcription response:", e); } }; } catch (error) { console.error("Transcription error:", error); } }; // Start a new call const startCall = () => { if (!currentCall.name.trim()) return; createCall(currentCall.name); }; // Copy call ID to clipboard const copyCallId = () => { navigator.clipboard.writeText(currentCall._id).then(() => { setCopiedToClipboard(true); setTimeout(() => setCopiedToClipboard(false), 3000); }); }; // Load a call by ID const loadCallById = async (id) => { try { // First clean up any existing call endCall(); // Load the call document const callDoc = await database.get(id); // Reset and set the current call resetCall(); mergeCall(callDoc); console.log("Call loaded successfully:", id); } catch (err) { console.error("Error loading call:", err); setConnectionError(`Could not load call: ${err.message}`); } }; // View a previous call const viewCall = async (callId) => { await loadCallById(callId); }; // Create a new empty call const newEmptyCall = () => { // Clean up any existing call endCall(); // Reset the call document resetCall(); setCallIdInput(""); }; // Join a call by ID input const joinCallById = async () => { if (!callIdInput.trim()) { setConnectionError("Please enter a valid call ID"); return; } try { await loadCallById(callIdInput); // If call loaded successfully and has an offer, we can join if (currentCall.offer) { await joinCall(); } else { setConnectionError("This call doesn't have a valid offer to join"); } } catch (err) { console.error("Error joining call by ID:", err); setConnectionError(`Failed to join call: ${err.message}`); } }; // Get status indicator color based on network quality const getNetworkQualityColor = () => { switch (networkStats.quality) { case 'excellent': return 'bg-green-500'; case 'good': return 'bg-green-400'; case 'fair': return 'bg-yellow-400'; case 'poor': return 'bg-red-500'; default: return 'bg-gray-400'; } }; return (

WebRTC Video Call App

*Connect face-to-face with reliable peer-to-peer video calls and get real-time AI transcription of your conversations. Create a new call and share the ID with someone to join, or paste a Call ID to join an existing call.*

{callStatus === 'idle' ? (

Create New Call

mergeCall({ name: e.target.value })} placeholder="Enter call name" className="p-3 border-2 border-purple-200 rounded-lg focus:outline-none focus:border-pink-400" />

Join Existing Call

setCallIdInput(e.target.value)} placeholder="Enter call ID to join" className="p-3 border-2 border-purple-200 rounded-lg focus:outline-none focus:border-pink-400" />

{currentCall.offer && !currentCall.answer && ( )}

{connectionError && (

{connectionError}

)}

) : (

{currentCall.name}

Status: {callStatus}

{callStatus === 'connecting' && (

Connecting...

)} {callStatus === 'connected' && (

Connected

{networkStats.quality}

)} {callStatus === 'reconnecting' && (

Reconnecting...

)} {currentCall._id && (

ID: {currentCall._id.substring(0, 8)}...

)} {callStatus === 'connected' && (

RTT: {networkStats.roundTripTime} Loss: {networkStats.packetLoss}

)}

{connectionError && (

{connectionError}

)}

)} {(callStatus !== 'idle') && (

{remoteStream ? (

)} {callStatus !== 'idle' && (

{callStatus === 'connected' && ( )} {callStatus === 'reconnecting' && ( )}

)} {callStatus === 'connected' && transcribing && (

Transcribing audio in real-time...

)} {currentCall.transcription && currentCall.transcription.length > 0 && (

Call Transcription

{currentCall.transcription.map((item, index) => (

{new Date(item.timestamp).toLocaleTimeString()}

{item.text}

))}

)}

Call History

{callHistory.map((call) => (

viewCall(call._id)} className="p-4 bg-white border-2 border-purple-200 rounded-lg hover:border-pink-400 cursor-pointer transition-all" >

{call.name}

{call.startTime && new Date(call.startTime).toLocaleDateString()} {call.startTime && new Date(call.startTime).toLocaleTimeString()}

{call.transcription ? `${call.transcription.length} entries` : 'No transcription'}

{call.endTime ? 'Ended' : 'Active'}

))}

); }