update speech detection

avatar-frontend/app/page.tsx

@@ -611,25 +611,67 @@ export default function Page() {
     recognition.continuous = true;
     recognition.interimResults = true;
     recognition.lang = selectedLanguage === "nl" ? "nl-NL" : "en-US";
+    // Improved speech recognition settings
+    recognition.maxAlternatives = 3; // Get multiple alternatives for better accuracy
+    recognition.serviceURI = ""; // Use default service
 
     let finalTranscript = "";
+    let interimTranscript = "";
     let timeoutId: NodeJS.Timeout | null = null;
+    let silenceTimeoutId: NodeJS.Timeout | null = null;
+    const SILENCE_TIMEOUT = 2000; // 2 seconds of silence before sending
+    const MIN_CONFIDENCE = 0.7; // Minimum confidence threshold
+
+    recognition.onstart = () => {
+      setStatus("Listening...");
+      finalTranscript = "";
+      interimTranscript = "";
+    };
 
-    recognition.onstart = () => setStatus("Listening...");
     recognition.onresult = (event: any) => {
       if (isAvatarSpeaking) return;
-      let interimTranscript = "";
+      
+      // Clear silence timeout when we get results
+      if (silenceTimeoutId) {
+        clearTimeout(silenceTimeoutId);
+        silenceTimeoutId = null;
+      }
+
+      interimTranscript = "";
+      let hasNewFinal = false;
+
       for (let i = event.resultIndex; i < event.results.length; i++) {
-        const transcript = event.results[i][0].transcript;
-        if (event.results[i].isFinal) finalTranscript += transcript + " ";
-        else interimTranscript += transcript;
+        const result = event.results[i];
+        const transcript = result[0].transcript;
+        const confidence = result[0].confidence || 0.5;
+
+        if (result.isFinal) {
+          // Only accept final results with good confidence
+          if (confidence >= MIN_CONFIDENCE || transcript.trim().length > 2) {
+            finalTranscript += transcript + " ";
+            hasNewFinal = true;
+          }
+        } else {
+          // Show interim results
+          interimTranscript += transcript;
+        }
       }
-      if (interimTranscript || finalTranscript) setStatus(`${interimTranscript || finalTranscript.slice(-50)}`);
-      if (finalTranscript.trim()) {
+
+      // Update status with current transcript
+      const displayText = interimTranscript || finalTranscript.trim();
+      if (displayText) {
+        setStatus(displayText.slice(-60)); // Show last 60 chars
+      }
+
+      // Handle final transcript
+      if (hasNewFinal && finalTranscript.trim()) {
+        // Clear any existing timeout
         if (timeoutId) clearTimeout(timeoutId);
+        
+        // Set a shorter timeout for final results (they're more reliable)
         timeoutId = setTimeout(() => {
           const textToSend = finalTranscript.trim();
-          if (textToSend && wsRef.current && !isAvatarSpeaking) {
+          if (textToSend && wsRef.current && !isAvatarSpeaking && textToSend.length >= 2) {
             setMessages((prev) => [
               ...prev,
               { id: Date.now().toString(), role: "user", content: textToSend, timestamp: new Date() },
@@ -640,28 +682,86 @@ export default function Page() {
               voice: selectedVoice 
             }));
             finalTranscript = "";
+            interimTranscript = "";
             setStatus("Processing...");
           }
-        }, 1500);
+        }, 800); // Shorter timeout for final results
+      } else if (interimTranscript.trim() && !hasNewFinal) {
+        // If we have interim results but no final, wait for silence
+        if (silenceTimeoutId) clearTimeout(silenceTimeoutId);
+        silenceTimeoutId = setTimeout(() => {
+          const textToSend = interimTranscript.trim() || finalTranscript.trim();
+          if (textToSend && wsRef.current && !isAvatarSpeaking && textToSend.length >= 3) {
+            setMessages((prev) => [
+              ...prev,
+              { id: Date.now().toString(), role: "user", content: textToSend, timestamp: new Date() },
+            ]);
+            wsRef.current.send(JSON.stringify({ 
+              type: "speech_end", 
+              text: textToSend,
+              voice: selectedVoice 
+            }));
+            finalTranscript = "";
+            interimTranscript = "";
+            setStatus("Processing...");
+          }
+        }, SILENCE_TIMEOUT);
       }
     };
+
     recognition.onerror = (event: any) => {
-      if (event.error === "no-speech") setStatus("No speech");
-      else if (event.error === "not-allowed") setStatus("Mic denied");
-      else if (event.error === "audio-capture") setStatus("No mic");
+      console.log("[Speech Recognition] Error:", event.error);
+      if (event.error === "no-speech") {
+        // Don't show error for no-speech, just keep listening
+        return;
+      } else if (event.error === "not-allowed") {
+        setStatus("Microphone permission denied");
+        shouldAutoRestartRef.current = false;
+      } else if (event.error === "audio-capture") {
+        setStatus("No microphone found");
+        shouldAutoRestartRef.current = false;
+      } else if (event.error === "network") {
+        setStatus("Network error - retrying...");
+        // Will auto-restart
+      } else if (event.error === "aborted") {
+        // User or system aborted, don't restart
+        shouldAutoRestartRef.current = false;
+      } else {
+        setStatus(`Speech error: ${event.error}`);
+      }
     };
+
     recognition.onend = () => {
-      if (shouldAutoRestartRef.current && recognitionRef.current === recognition) {
+      // Clear timeouts
+      if (timeoutId) clearTimeout(timeoutId);
+      if (silenceTimeoutId) clearTimeout(silenceTimeoutId);
+      
+      // Auto-restart if we should
+      if (shouldAutoRestartRef.current && recognitionRef.current === recognition && !isSpeechPaused) {
         setTimeout(() => {
-          try { recognition.start(); setStatus("Listening..."); } catch {}
+          try { 
+            recognition.start(); 
+            setStatus("Listening..."); 
+          } catch (e) {
+            console.log("[Speech Recognition] Restart failed:", e);
+            // Try again after a longer delay
+            setTimeout(() => {
+              try {
+                recognition.start();
+                setStatus("Listening...");
+              } catch {}
+            }, 500);
+          }
         }, 100);
       }
     };
+
     try {
       recognition.start();
       recognitionRef.current = recognition;
-    } catch {
-      setStatus("Speech failed");
+    } catch (e) {
+      console.error("[Speech Recognition] Start failed:", e);
+      setStatus("Speech recognition failed to start");
     }
   }
 
@@ -839,13 +939,12 @@ export default function Page() {
         localStorage.removeItem("mrrrme_avatar_remote_url");
         localStorage.removeItem("mrrrme_avatar_url");
         setStatus("Avatar updated! ✨");
-        // Only show config screen if we haven't seen it before (check localStorage)
+        // Show config screen if we haven't configured before
         const hasConfiguredBefore = localStorage.getItem("mrrrme_has_configured");
         if (!hasConfiguredBefore) {
           setTimeout(() => {
             setStatus("What can I do for you today?");
             setShowConfigScreen(true);
-            localStorage.setItem("mrrrme_has_configured", "true");
           }, 2500);
         } else {
           setTimeout(() => {
@@ -881,13 +980,12 @@ export default function Page() {
       setStatus("Avatar updated.");
     }
   
-    // Only show config screen if we haven't seen it before (check localStorage)
+    // Show config screen if we haven't configured before
     const hasConfiguredBefore = localStorage.getItem("mrrrme_has_configured");
     if (!hasConfiguredBefore) {
       setTimeout(() => {
         setStatus("What can I do for you today?");
         setShowConfigScreen(true);
-        localStorage.setItem("mrrrme_has_configured", "true");
       }, 2500);
     } else {
       setTimeout(() => {
@@ -1121,6 +1219,7 @@ export default function Page() {
 
             <button
               onClick={() => {
+                localStorage.setItem("mrrrme_has_configured", "true");
                 setShowConfigScreen(false);
                 startCapture();
               }}