backend folder

.gitignore

@@ -4,5 +4,3 @@ __pycache__/
 node_modules/
 .next/
 *.log
-mrrrme/backend/ 
-avatar-frontend/backend/ 

mrrrme/backend/README.md

@@ -0,0 +1,127 @@
+# MrrrMe Backend - Refactored Structure
+
+This is a refactored version of `backend_server.py` split into modular components.
+
+## 📁 Directory Structure
+
+```
+backend/
+├── __init__.py              # Main package init
+├── app.py                   # FastAPI app setup
+├── config.py                # Configuration & constants
+├── websocket.py             # WebSocket handler (TO BE CREATED)
+│
+├── auth/                    # Authentication
+│   ├── __init__.py
+│   ├── models.py            # Pydantic models
+│   ├── database.py          # Database init & helpers
+│   └── routes.py            # Auth endpoints
+│
+├── session/                 # Session management
+│   ├── __init__.py
+│   ├── manager.py           # Token validation, user context
+│   └── summary.py           # AI summary generation
+│
+├── models/                  # AI model loading
+│   ├── __init__.py
+│   └── loader.py            # Async model initialization
+│
+├── processing/              # Core processing logic
+│   ├── __init__.py
+│   ├── video.py             # Video frame processing
+│   ├── audio.py             # Audio chunk processing
+│   ├── speech.py            # Speech processing (TO BE CREATED)
+│   └── fusion.py            # Emotion fusion
+│
+├── debug/                   # Debug endpoints
+│   ├── __init__.py
+│   └── routes.py            # Debug routes
+│
+└── utils/                   # Utilities
+    ├── __init__.py
+    ├── helpers.py           # Helper functions
+    └── patches.py           # GPU & system patches
+```
+
+## 🚀 How to Use
+
+### Option 1: Drop-in Replacement
+1. Copy the `backend/` folder to `mrrrme/backend/`
+2. Create/update `mrrrme/backend_new.py`:
+
+```python
+"""New modular backend server"""
+from backend import app
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+```
+
+3. Run: `python mrrrme/backend_new.py`
+
+### Option 2: Gradual Migration
+Keep your old `backend_server.py` and slowly migrate endpoints:
+
+1. Import refactored modules:
+```python
+from backend.auth import router as auth_router
+from backend.session import validate_token
+```
+
+2. Replace sections incrementally
+
+## 🔧 Still Need to Create
+
+The following files need the full logic from `backend_server.py`:
+
+### `websocket.py` (~400 lines)
+- Main WebSocket endpoint logic
+- Message type routing
+- Greeting generation
+- Video/audio/speech handling orchestration
+
+### `processing/speech.py` (~250 lines)
+- Full speech processing pipeline
+- Transcription filtering
+- Emotion detection coordination
+- LLM context preparation
+- Avatar TTS integration
+
+## 📋 Migration Checklist
+
+- [x] Config & environment setup
+- [x] Authentication (signup/login/logout)
+- [x] Database management
+- [x] Session validation
+- [x] AI summary generation
+- [x] Model loading
+- [x] Video frame processing
+- [x] Audio chunk processing
+- [x] Emotion fusion logic
+- [x] Debug endpoints
+- [ ] WebSocket handler (main logic)
+- [ ] Speech processing pipeline
+- [ ] Greeting generation logic
+- [ ] Full integration testing
+
+## 🎯 Benefits
+
+1. **Modularity**: Each component has a single responsibility
+2. **Testability**: Easy to unit test individual modules
+3. **Maintainability**: Find and fix bugs faster
+4. **Scalability**: Add new features without bloating
+5. **Collaboration**: Multiple devs can work on different modules
+
+## 📦 Original File Size
+
+- `backend_server.py`: 47KB (1,300 lines)
+- Refactored: 12 modules (~100-300 lines each)
+
+## ⚠️ Important Notes
+
+- Ensure all imports match your project structure
+- Update `mrrrme` module imports in `models/loader.py`
+- Test thoroughly before replacing production code
+- Keep `backend_server.py` as backup during migration
+

mrrrme/backend/__init__.py

@@ -0,0 +1,10 @@
+"""MrrrMe Backend - Refactored Modular Structure"""
+from .app import app
+from .config import *
+from .utils import apply_all_patches
+
+# Apply system patches on import
+apply_all_patches()
+
+__version__ = "2.0.0"
+__all__ = ['app']

mrrrme/backend/app.py

@@ -0,0 +1,60 @@
+"""MrrrMe Backend - FastAPI Application Setup"""
+import os
+import asyncio
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from .models.loader import load_models, models_ready
+from .utils.helpers import get_avatar_api_url, check_avatar_service
+from .auth.routes import router as auth_router
+from .debug.routes import router as debug_router
+from .websocket import websocket_endpoint
+
+# Create FastAPI app
+app = FastAPI(title="MrrrMe AI Backend", version="1.0.0")
+
+# CORS for browser access
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(auth_router, prefix="/api")
+app.include_router(debug_router, prefix="/api/debug")
+
+# WebSocket endpoint
+app.add_api_websocket_route("/ws", websocket_endpoint)
+
+@app.on_event("startup")
+async def startup_event():
+    """Start loading models in background after server is ready"""
+    print("[Backend] 🚀 Starting up...")
+    
+    # Check avatar service availability
+    avatar_api = get_avatar_api_url()
+    print(f"[Backend] 🎭 Avatar API URL: {avatar_api}")
+    await check_avatar_service(avatar_api)
+    
+    # Load models asynchronously
+    asyncio.create_task(load_models())
+
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "status": "running",
+        "models_ready": models_ready,
+        "message": "MrrrMe AI Backend"
+    }
+
+@app.get("/health")
+async def health():
+    """Health check - responds immediately"""
+    return {
+        "status": "healthy",
+        "models_ready": models_ready
+    }

mrrrme/backend/auth/__init__.py

@@ -0,0 +1,7 @@
+"""MrrrMe Backend - Authentication Package"""
+from .routes import router
+from .models import SignupRequest, LoginRequest, LogoutRequest
+from .database import init_db, hash_password, get_db_connection
+
+__all__ = ['router', 'SignupRequest', 'LoginRequest', 'LogoutRequest', 
+           'init_db', 'hash_password', 'get_db_connection']

mrrrme/backend/auth/database.py

@@ -0,0 +1,66 @@
+"""MrrrMe Backend - Database Management"""
+import sqlite3
+import hashlib
+from ..config import DB_PATH
+
+def init_db():
+    """Initialize SQLite database with required tables"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # Users table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS users (
+            user_id TEXT PRIMARY KEY,
+            username TEXT UNIQUE NOT NULL,
+            password_hash TEXT NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """)
+    
+    # Sessions table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS sessions (
+            session_id TEXT PRIMARY KEY,
+            user_id TEXT NOT NULL,
+            token TEXT UNIQUE NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            is_active BOOLEAN DEFAULT 1
+        )
+    """)
+    
+    # Messages table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS messages (
+            message_id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL,
+            role TEXT NOT NULL,
+            content TEXT NOT NULL,
+            emotion TEXT,
+            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """)
+    
+    # User summaries table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS user_summaries (
+            user_id TEXT PRIMARY KEY,
+            summary_text TEXT NOT NULL,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """)
+    
+    conn.commit()
+    conn.close()
+    print(f"[Database] ✅ Initialized at {DB_PATH}")
+
+def hash_password(password: str) -> str:
+    """Hash password using SHA-256"""
+    return hashlib.sha256(password.encode()).hexdigest()
+
+def get_db_connection():
+    """Get database connection"""
+    return sqlite3.connect(DB_PATH)
+
+# Initialize database on module import
+init_db()

mrrrme/backend/auth/models.py

@@ -0,0 +1,16 @@
+"""MrrrMe Backend - Authentication Models"""
+from pydantic import BaseModel
+
+class SignupRequest(BaseModel):
+    """User signup request model"""
+    username: str
+    password: str
+
+class LoginRequest(BaseModel):
+    """User login request model"""
+    username: str
+    password: str
+
+class LogoutRequest(BaseModel):
+    """User logout request model"""
+    token: str

mrrrme/backend/auth/routes.py

@@ -0,0 +1,111 @@
+"""MrrrMe Backend - Authentication Routes"""
+import secrets
+import sqlite3
+from fastapi import APIRouter, HTTPException
+from .models import SignupRequest, LoginRequest, LogoutRequest
+from .database import hash_password, get_db_connection
+from ..session.summary import generate_session_summary
+
+router = APIRouter()
+
+@router.post("/signup")
+async def signup(req: SignupRequest):
+    """Create new user account"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    try:
+        user_id = secrets.token_urlsafe(16)
+        cursor.execute(
+            "INSERT INTO users (user_id, username, password_hash) VALUES (?, ?, ?)",
+            (user_id, req.username, hash_password(req.password))
+        )
+        conn.commit()
+        conn.close()
+        return {"success": True, "message": "Account created!"}
+    except sqlite3.IntegrityError:
+        conn.close()
+        raise HTTPException(status_code=400, detail="Username already exists")
+
+@router.post("/login")
+async def login(req: LoginRequest):
+    """Login user and create session"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    # Verify credentials
+    cursor.execute(
+        "SELECT user_id, username FROM users WHERE username = ? AND password_hash = ?",
+        (req.username, hash_password(req.password))
+    )
+    
+    result = cursor.fetchone()
+    
+    if not result:
+        conn.close()
+        raise HTTPException(status_code=401, detail="Invalid credentials")
+    
+    user_id, username = result
+    
+    # Create session
+    session_id = secrets.token_urlsafe(16)
+    token = secrets.token_urlsafe(32)
+    
+    cursor.execute(
+        "INSERT INTO sessions (session_id, user_id, token) VALUES (?, ?, ?)",
+        (session_id, user_id, token)
+    )
+    
+    # Get user summary
+    cursor.execute(
+        "SELECT summary_text FROM user_summaries WHERE user_id = ?",
+        (user_id,)
+    )
+    summary_row = cursor.fetchone()
+    summary = summary_row[0] if summary_row else None
+    
+    conn.commit()
+    conn.close()
+    
+    return {
+        "success": True,
+        "token": token,
+        "username": username,
+        "user_id": user_id,
+        "summary": summary
+    }
+
+@router.post("/logout")
+async def logout(req: LogoutRequest):
+    """Logout user and generate session summary"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    # Get session info before closing
+    cursor.execute(
+        "SELECT session_id, user_id FROM sessions WHERE token = ? AND is_active = 1",
+        (req.token,)
+    )
+    result = cursor.fetchone()
+    
+    if result:
+        session_id, user_id = result
+        
+        # Mark session as inactive
+        cursor.execute(
+            "UPDATE sessions SET is_active = 0 WHERE token = ?",
+            (req.token,)
+        )
+        conn.commit()
+        conn.close()
+        
+        # Generate summary on explicit logout
+        print(f"[Logout] 📝 Generating summary for user {user_id}...")
+        summary = await generate_session_summary(session_id, user_id)
+        if summary:
+            print(f"[Logout] ✅ Summary generated")
+        
+        return {"success": True, "message": "Logged out successfully"}
+    else:
+        conn.close()
+        return {"success": True, "message": "Session already closed"}

mrrrme/backend/config.py

@@ -0,0 +1,62 @@
+"""MrrrMe Backend - Configuration and Environment Setup"""
+import os
+
+# ===== SET CACHE DIRECTORIES FIRST =====
+os.environ['HF_HOME'] = '/tmp/huggingface'
+os.environ['TRANSFORMERS_CACHE'] = '/tmp/transformers'
+os.environ['HF_HUB_CACHE'] = '/tmp/huggingface/hub'
+os.environ['TORCH_HOME'] = '/tmp/torch'
+
+# Create cache directories
+for cache_dir in ['/tmp/huggingface', '/tmp/transformers', '/tmp/huggingface/hub', '/tmp/torch']:
+    os.makedirs(cache_dir, exist_ok=True)
+
+# ===== DATABASE CONFIGURATION =====
+# Use /data for Hugging Face Spaces (persistent) or /tmp for local dev
+if os.path.exists('/data'):
+    DB_PATH = "/data/mrrrme_users.db"
+    print("[Config] 📁 Using persistent storage: /data/mrrrme_users.db")
+else:
+    DB_PATH = "/tmp/mrrrme_users.db"
+    print("[Config] ⚠️ Using ephemeral storage: /tmp/mrrrme_users.db")
+    print("[Config] ⚠️ Data will reset on rebuild! Enable persistent storage in HF Spaces.")
+
+# ===== API KEYS =====
+GROQ_API_KEY = os.getenv("GROQ_API_KEY", "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")
+
+# ===== FUSION WEIGHTS =====
+FUSION_WEIGHTS = {
+    'face': 0.5,
+    'voice': 0.3,
+    'text': 0.2
+}
+
+# ===== EMOTION MAPPING =====
+EMOTION_MAP = {'Neutral': 0, 'Happy': 1, 'Sad': 2, 'Angry': 3}
+FUSE4 = ['Neutral', 'Happy', 'Sad', 'Angry']
+
+# ===== SPEECH PROCESSING =====
+AUDIO_BUFFER_SIZE = 5  # Number of audio chunks to buffer
+AUDIO_BUFFER_KEEP = 3  # Number of chunks to keep after processing
+
+# ===== MESSAGE FILTERING =====
+HALLUCINATION_PHRASES = {"thank you", "thanks", "okay", "ok", "you", "yeah", "yep"}
+MIN_TRANSCRIPTION_LENGTH = 2
+
+# ===== SUMMARY GENERATION =====
+MIN_MESSAGES_FOR_SUMMARY = 3
+SUMMARY_MODEL = "llama-3.1-8b-instant"
+SUMMARY_MAX_TOKENS = 150
+SUMMARY_TEMPERATURE = 0.7
+
+# ===== GREETING MESSAGES =====
+GREETINGS = {
+    "en": {
+        "new": "Hey {username}! I'm MrrrMe, your emotion AI companion. How are you feeling today?",
+        "returning": "Welcome back, {username}! It's great to see you again. How have you been?"
+    },
+    "nl": {
+        "new": "Hoi {username}! Ik ben MrrrMe, jouw emotie AI-metgezel. Hoe voel je je vandaag?",
+        "returning": "Welkom terug, {username}! Fijn je weer te zien. Hoe gaat het met je?"
+    }
+}

mrrrme/backend/debug/__init__.py

@@ -0,0 +1,4 @@
+"""MrrrMe Backend - Debug Package"""
+from .routes import router
+
+__all__ = ['router']

mrrrme/backend/debug/routes.py

@@ -0,0 +1,60 @@
+"""MrrrMe Backend - Debug Routes"""
+from fastapi import APIRouter
+from ..auth.database import get_db_connection
+from ..config import DB_PATH
+
+router = APIRouter()
+
+@router.get("/users")
+async def debug_users():
+    """Debug endpoint - view all users and their summaries"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute("""
+        SELECT u.username, u.user_id, s.summary_text, s.updated_at
+        FROM users u
+        LEFT JOIN user_summaries s ON u.user_id = s.user_id
+        ORDER BY u.created_at DESC
+    """)
+    
+    users = []
+    for username, user_id, summary, updated in cursor.fetchall():
+        users.append({
+            "username": username,
+            "user_id": user_id,
+            "summary": summary,
+            "summary_updated": updated
+        })
+    
+    conn.close()
+    
+    return {"users": users, "database": DB_PATH}
+
+@router.get("/sessions")
+async def debug_sessions():
+    """Debug endpoint - view all active sessions"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute("""
+        SELECT s.session_id, s.token, u.username, s.is_active, s.created_at
+        FROM sessions s
+        JOIN users u ON s.user_id = u.user_id
+        ORDER BY s.created_at DESC
+        LIMIT 20
+    """)
+    
+    sessions = []
+    for session_id, token, username, is_active, created_at in cursor.fetchall():
+        sessions.append({
+            "session_id": session_id,
+            "token_preview": token[:10] + "..." if token else None,
+            "username": username,
+            "is_active": bool(is_active),
+            "created_at": created_at
+        })
+    
+    conn.close()
+    
+    return {"sessions": sessions, "database": DB_PATH}

mrrrme/backend/models/__init__.py

@@ -0,0 +1,26 @@
+"""MrrrMe Backend - Models Package"""
+from .loader import (
+    load_models,
+    get_models,
+    models_ready,
+    face_processor,
+    text_analyzer,
+    whisper_worker,
+    voice_worker,
+    llm_generator,
+    fusion_engine,
+    FusionEngine
+)
+
+__all__ = [
+    'load_models',
+    'get_models',
+    'models_ready',
+    'face_processor',
+    'text_analyzer',
+    'whisper_worker',
+    'voice_worker',
+    'llm_generator',
+    'fusion_engine',
+    'FusionEngine'
+]

mrrrme/backend/models/loader.py

@@ -0,0 +1,98 @@
+"""MrrrMe Backend - AI Model Loader"""
+import torch
+import numpy as np
+from typing import Optional
+
+# Global model variables
+face_processor = None
+text_analyzer = None
+whisper_worker = None
+voice_worker = None
+llm_generator = None
+fusion_engine = None
+models_ready = False
+
+class FusionEngine:
+    """Multi-modal emotion fusion engine"""
+    def __init__(self, alpha_face=0.5, alpha_voice=0.3, alpha_text=0.2):
+        self.alpha_face = alpha_face
+        self.alpha_voice = alpha_voice
+        self.alpha_text = alpha_text
+    
+    def fuse(self, face_probs, voice_probs, text_probs):
+        """Fuse emotion probabilities from multiple modalities"""
+        from ..config import FUSE4
+        
+        fused = (
+            self.alpha_face * face_probs +
+            self.alpha_voice * voice_probs +
+            self.alpha_text * text_probs
+        )
+        fused = fused / (np.sum(fused) + 1e-8)
+        fused_idx = int(np.argmax(fused))
+        fused_emotion = FUSE4[fused_idx]
+        intensity = float(np.max(fused))
+        return fused_emotion, intensity
+
+async def load_models():
+    """Load all AI models asynchronously"""
+    global face_processor, text_analyzer, whisper_worker, voice_worker
+    global llm_generator, fusion_engine, models_ready
+    
+    print("[Backend] 🚀 Initializing MrrrMe AI models in background...")
+    
+    try:
+        # Import modules (adjust paths based on your actual structure)
+        from mrrrme.vision.face_processor import FaceProcessor
+        from mrrrme.audio.voice_emotion import VoiceEmotionWorker
+        from mrrrme.audio.whisper_transcription import WhisperTranscriptionWorker
+        from mrrrme.nlp.text_sentiment import TextSentimentAnalyzer
+        from mrrrme.nlp.llm_generator_groq import LLMResponseGenerator
+        from ..config import GROQ_API_KEY
+        
+        # Load models
+        print("[Backend] Loading FaceProcessor...")
+        face_processor = FaceProcessor()
+        
+        print("[Backend] Loading TextSentiment...")
+        text_analyzer = TextSentimentAnalyzer()
+        
+        print("[Backend] Loading Whisper...")
+        whisper_worker = WhisperTranscriptionWorker(text_analyzer)
+        
+        print("[Backend] Loading VoiceEmotion...")
+        voice_worker = VoiceEmotionWorker(whisper_worker=whisper_worker)
+        
+        print("[Backend] Initializing LLM...")
+        llm_generator = LLMResponseGenerator(api_key=GROQ_API_KEY)
+        
+        # Initialize fusion engine
+        print("[Backend] Initializing FusionEngine...")
+        fusion_engine = FusionEngine()
+        
+        models_ready = True
+        
+        print("[Backend] ✅ All models loaded!")
+        
+        # GPU check
+        if torch.cuda.is_available():
+            print(f"[Backend] ✅ GPU available: {torch.cuda.get_device_name(0)}")
+        else:
+            print("[Backend] ⚠️ No GPU detected - using CPU mode")
+        
+    except Exception as e:
+        print(f"[Backend] ❌ Error loading models: {e}")
+        import traceback
+        traceback.print_exc()
+
+def get_models():
+    """Get loaded model instances"""
+    return {
+        'face_processor': face_processor,
+        'text_analyzer': text_analyzer,
+        'whisper_worker': whisper_worker,
+        'voice_worker': voice_worker,
+        'llm_generator': llm_generator,
+        'fusion_engine': fusion_engine,
+        'models_ready': models_ready
+    }

mrrrme/backend/processing/__init__.py

@@ -0,0 +1,14 @@
+"""MrrrMe Backend - Processing Package"""
+from .video import process_video_frame
+from .audio import process_audio_chunk
+from .speech import process_speech_end, filter_transcription
+from .fusion import calculate_fusion, adjust_fusion_weights
+
+__all__ = [
+    'process_video_frame',
+    'process_audio_chunk',
+    'process_speech_end',
+    'filter_transcription',
+    'calculate_fusion',
+    'adjust_fusion_weights'
+]

mrrrme/backend/processing/audio.py

@@ -0,0 +1,29 @@
+"""MrrrMe Backend - Audio Chunk Processing"""
+import base64
+from ..models.loader import voice_worker
+from ..config import AUDIO_BUFFER_SIZE, AUDIO_BUFFER_KEEP
+
+audio_buffer = []
+
+async def process_audio_chunk(audio_data_b64: str) -> dict:
+    """Process audio chunk for voice emotion detection"""
+    global audio_buffer
+    
+    try:
+        audio_data = base64.b64decode(audio_data_b64)
+        audio_buffer.append(audio_data)
+        
+        if len(audio_buffer) >= AUDIO_BUFFER_SIZE:
+            voice_probs, voice_emotion = voice_worker.get_probs()
+            audio_buffer = audio_buffer[-AUDIO_BUFFER_KEEP:]
+            
+            return {
+                "type": "voice_emotion",
+                "emotion": voice_emotion
+            }
+        
+        return None
+        
+    except Exception as e:
+        print(f"[Audio] Error: {e}")
+        return None

mrrrme/backend/processing/fusion.py

@@ -0,0 +1,53 @@
+"""MrrrMe Backend - Emotion Fusion Logic"""
+import numpy as np
+from ..config import EMOTION_MAP, FUSION_WEIGHTS
+
+def adjust_fusion_weights(face_quality: float, voice_active: bool, text_length: int) -> dict:
+    """
+    Adjust fusion weights based on quality metrics
+    
+    Returns:
+        Dict with adjusted weights and adjustment log
+    """
+    adjusted_weights = FUSION_WEIGHTS.copy()
+    adjustments = []
+    
+    # Reduce face weight if quality is poor
+    if face_quality < 0.5:
+        adjusted_weights['face'] *= 0.7
+        adjustments.append(f"Face weight reduced (low quality: {face_quality:.3f})")
+    
+    # Reduce voice weight if not active
+    if not voice_active:
+        adjusted_weights['voice'] *= 0.5
+        adjustments.append("Voice weight reduced (no recent speech)")
+    
+    # Reduce text weight if very short
+    if text_length < 10:
+        adjusted_weights['text'] *= 0.7
+        adjustments.append(f"Text weight reduced (short input: {text_length} chars)")
+    
+    # Normalize to sum to 1.0
+    total = sum(adjusted_weights.values())
+    final_weights = {k: v/total for k, v in adjusted_weights.items()}
+    
+    return {
+        'weights': final_weights,
+        'adjustments': adjustments
+    }
+
+def calculate_fusion(face_probs, voice_probs, text_probs, weights: dict):
+    """Calculate weighted fusion of emotion probabilities"""
+    fused_probs = (
+        weights['face'] * face_probs +
+        weights['voice'] * voice_probs +
+        weights['text'] * text_probs
+    )
+    fused_probs = fused_probs / (np.sum(fused_probs) + 1e-8)
+    
+    fused_idx = int(np.argmax(fused_probs))
+    emotions = ['Neutral', 'Happy', 'Sad', 'Angry']
+    fused_emotion = emotions[fused_idx]
+    intensity = float(np.max(fused_probs))
+    
+    return fused_emotion, intensity, fused_probs

mrrrme/backend/processing/video.py

@@ -0,0 +1,55 @@
+"""MrrrMe Backend - Video Frame Processing"""
+import base64
+import io
+import cv2
+import numpy as np
+from PIL import Image
+from ..models.loader import face_processor
+
+async def process_video_frame(frame_data: str) -> dict:
+    """
+    Process video frame for facial emotion detection
+    
+    Args:
+        frame_data: Base64 encoded image
+    
+    Returns:
+        Dict with emotion, confidence, probabilities, quality
+    """
+    try:
+        # Decode base64 image
+        img_data = base64.b64decode(frame_data.split(",")[1])
+        img = Image.open(io.BytesIO(img_data))
+        frame = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+        
+        # Process face emotion
+        try:
+            processed_frame, result = face_processor.process_frame(frame)
+            face_emotion = face_processor.get_last_emotion() or "Neutral"
+            face_confidence = face_processor.get_last_confidence() or 0.0
+            face_probs = face_processor.get_last_probs()
+            face_quality = face_processor.get_last_quality() if hasattr(face_processor, 'get_last_quality') else 0.5
+        except Exception as proc_err:
+            print(f"[FaceProcessor] Error: {proc_err}")
+            face_emotion = "Neutral"
+            face_confidence = 0.0
+            face_probs = np.array([0.25, 0.25, 0.25, 0.25])
+            face_quality = 0.0
+        
+        return {
+            "type": "face_emotion",
+            "emotion": face_emotion,
+            "confidence": face_confidence,
+            "probabilities": face_probs.tolist(),
+            "quality": face_quality
+        }
+        
+    except Exception as e:
+        print(f"[Video] Error: {e}")
+        return {
+            "type": "face_emotion",
+            "emotion": "Neutral",
+            "confidence": 0.0,
+            "probabilities": [0.25, 0.25, 0.25, 0.25],
+            "quality": 0.0
+        }

mrrrme/backend/session/__init__.py

@@ -0,0 +1,16 @@
+"""MrrrMe Backend - Session Package"""
+from .manager import (
+    validate_token,
+    get_user_summary,
+    load_user_history,
+    save_message
+)
+from .summary import generate_session_summary
+
+__all__ = [
+    'validate_token',
+    'get_user_summary', 
+    'load_user_history',
+    'save_message',
+    'generate_session_summary'
+]

mrrrme/backend/session/manager.py

@@ -0,0 +1,84 @@
+"""MrrrMe Backend - Session Management"""
+import sqlite3
+from typing import Optional, Tuple, Dict
+from ..auth.database import get_db_connection
+
+def validate_token(token: str) -> Optional[Dict[str, str]]:
+    """
+    Validate session token and return session data
+    
+    Returns:
+        Dict with session_id, user_id, username if valid
+        None if invalid
+    """
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute(
+        """SELECT s.session_id, s.user_id, u.username 
+           FROM sessions s 
+           JOIN users u ON s.user_id = u.user_id 
+           WHERE s.token = ? AND s.is_active = 1""",
+        (token,)
+    )
+    
+    result = cursor.fetchone()
+    conn.close()
+    
+    if not result:
+        return None
+    
+    session_id, user_id, username = result
+    return {
+        'session_id': session_id,
+        'user_id': user_id,
+        'username': username
+    }
+
+def get_user_summary(user_id: str) -> Optional[str]:
+    """Get user's conversation summary"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute(
+        "SELECT summary_text FROM user_summaries WHERE user_id = ?",
+        (user_id,)
+    )
+    
+    summary_row = cursor.fetchone()
+    conn.close()
+    
+    return summary_row[0] if summary_row else None
+
+def load_user_history(user_id: str, limit: int = 10) -> list:
+    """Load recent conversation history for user"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute(
+        """SELECT role, content FROM messages 
+           WHERE session_id IN (
+               SELECT session_id FROM sessions WHERE user_id = ?
+           )
+           ORDER BY timestamp DESC 
+           LIMIT ?""",
+        (user_id, limit)
+    )
+    
+    history = cursor.fetchall()
+    conn.close()
+    
+    return list(reversed(history))  # Return chronological order
+
+def save_message(session_id: str, role: str, content: str, emotion: Optional[str] = None):
+    """Save message to database"""
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    cursor.execute(
+        "INSERT INTO messages (session_id, role, content, emotion) VALUES (?, ?, ?, ?)",
+        (session_id, role, content, emotion)
+    )
+    
+    conn.commit()
+    conn.close()

mrrrme/backend/session/summary.py

@@ -0,0 +1,110 @@
+"""MrrrMe Backend - AI Session Summary Generation"""
+from datetime import datetime
+from typing import Optional
+from groq import Groq
+from ..auth.database import get_db_connection
+from ..config import (
+    GROQ_API_KEY, 
+    MIN_MESSAGES_FOR_SUMMARY,
+    SUMMARY_MODEL,
+    SUMMARY_MAX_TOKENS,
+    SUMMARY_TEMPERATURE
+)
+
+async def generate_session_summary(session_id: str, user_id: str) -> Optional[str]:
+    """
+    Generate AI summary of conversation for THIS specific user
+    
+    Args:
+        session_id: Session to summarize
+        user_id: User who owns the session
+    
+    Returns:
+        Summary text if successful, None otherwise
+    """
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    
+    # Verify session belongs to user (security check)
+    cursor.execute(
+        "SELECT user_id FROM sessions WHERE session_id = ?",
+        (session_id,)
+    )
+    session_owner = cursor.fetchone()
+    
+    if not session_owner or session_owner[0] != user_id:
+        print(f"[Summary] ❌ Security error: session {session_id} doesn't belong to user {user_id}")
+        conn.close()
+        return None
+    
+    # Get messages from this session
+    cursor.execute(
+        "SELECT role, content, emotion FROM messages WHERE session_id = ? ORDER BY timestamp ASC",
+        (session_id,)
+    )
+    
+    messages = cursor.fetchall()
+    
+    # Get username for better logging
+    cursor.execute("SELECT username FROM users WHERE user_id = ?", (user_id,))
+    username_row = cursor.fetchone()
+    username = username_row[0] if username_row else user_id
+    
+    conn.close()
+    
+    # Skip if not enough messages
+    if len(messages) < MIN_MESSAGES_FOR_SUMMARY:
+        print(f"[Summary] ⏭️ Skipped for {username} (only {len(messages)} messages)")
+        return None
+    
+    # Build conversation text
+    conversation = ""
+    for role, content, emotion in messages:
+        speaker = "User" if role == "user" else "AI"
+        emo_tag = f" [{emotion}]" if emotion else ""
+        conversation += f"{speaker}{emo_tag}: {content}\n"
+    
+    try:
+        # Generate summary using Groq
+        groq_client = Groq(api_key=GROQ_API_KEY)
+        
+        prompt = f"""Analyze this conversation and create a 2-3 sentence summary about THIS SPECIFIC USER.
+
+DO NOT include information about other users or other conversations.
+ONLY summarize what THIS user said and their patterns.
+
+Conversation ({len(messages)} messages):
+{conversation}
+
+Create a concise summary including: topics this user discussed, their emotional patterns, personal details THEY mentioned, and their preferences."""
+        
+        response = groq_client.chat.completions.create(
+            model=SUMMARY_MODEL,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=SUMMARY_MAX_TOKENS,
+            temperature=SUMMARY_TEMPERATURE
+        )
+        
+        summary = response.choices[0].message.content.strip()
+        
+        # Save summary FOR THIS USER ONLY
+        conn = get_db_connection()
+        cursor = conn.cursor()
+        
+        cursor.execute(
+            "INSERT OR REPLACE INTO user_summaries (user_id, summary_text, updated_at) VALUES (?, ?, ?)",
+            (user_id, summary, datetime.now())
+        )
+        
+        conn.commit()
+        conn.close()
+        
+        print(f"[Summary] ✅ Generated for {username} (user_id: {user_id})")
+        print(f"[Summary] 📝 Content: {summary}")
+        return summary
+        
+    except Exception as e:
+        print(f"[Summary] ❌ Error for {username}: {e}")
+        import traceback
+        traceback.print_exc()
+        return None

mrrrme/backend/utils/__init__.py

@@ -0,0 +1,5 @@
+"""MrrrMe Backend - Utils Package"""
+from .helpers import get_avatar_api_url, check_avatar_service
+from .patches import apply_all_patches
+
+__all__ = ['get_avatar_api_url', 'check_avatar_service', 'apply_all_patches']

mrrrme/backend/utils/helpers.py

@@ -0,0 +1,29 @@
+"""MrrrMe Backend - Utility Helper Functions"""
+import os
+import requests
+
+def get_avatar_api_url():
+    """Get correct avatar API URL based on environment"""
+    # For Hugging Face Spaces, use same host
+    if os.path.exists('/.dockerenv') or os.environ.get('SPACE_ID'):
+        # Running in Docker/HF Spaces - use internal networking
+        return "http://127.0.0.1:8765"
+    else:
+        # Local development
+        return "http://localhost:8765"
+
+async def check_avatar_service(avatar_api: str):
+    """Check if avatar TTS service is running"""
+    try:
+        response = requests.get(f"{avatar_api}/", timeout=2)
+        if response.status_code == 200:
+            print(f"[Backend] ✅ Avatar TTS service available at {avatar_api}")
+        else:
+            print(f"[Backend] ⚠️ Avatar TTS service responded with {response.status_code}")
+    except requests.exceptions.ConnectionError:
+        print(f"[Backend] ⚠️ Avatar TTS service NOT available at {avatar_api}")
+        print(f"[Backend] 💡 Text-only mode will be used (no avatar speech)")
+        print(f"[Backend] 📝 To enable avatar:")
+        print(f"[Backend]    cd avatar && python speak_server.py")
+    except Exception as e:
+        print(f"[Backend] ⚠️ Error checking avatar service: {e}")

mrrrme/backend/utils/patches.py

@@ -0,0 +1,49 @@
+"""MrrrMe Backend - GPU Fixes and Patches"""
+import os
+import logging
+
+# ===== GPU FIX: Patch TensorBoard =====
+class DummySummaryWriter:
+    """Dummy TensorBoard writer to prevent GPU issues"""
+    def __init__(self, *args, **kwargs): 
+        pass
+    
+    def __getattr__(self, name): 
+        return lambda *args, **kwargs: None
+
+def patch_tensorboard():
+    """Patch TensorBoard to avoid GPU conflicts"""
+    try:
+        import tensorboardX
+        tensorboardX.SummaryWriter = DummySummaryWriter
+        print("[Patches] ✅ TensorBoard patched")
+    except ImportError:
+        pass  # TensorBoard not installed
+
+# ===== GPU FIX: Patch Logging to redirect /work paths =====
+_original_FileHandler = logging.FileHandler
+
+class RedirectingFileHandler(_original_FileHandler):
+    """File handler that redirects /work paths to /tmp"""
+    def __init__(self, filename, mode='a', encoding=None, delay=False, errors=None):
+        if isinstance(filename, str) and filename.startswith('/work'):
+            filename = '/tmp/openface_log.txt'
+        
+        # Ensure directory exists
+        dirname = os.path.dirname(filename)
+        if dirname:
+            os.makedirs(dirname, exist_ok=True)
+        else:
+            os.makedirs('/tmp', exist_ok=True)
+        
+        super().__init__(filename, mode, encoding, delay, errors)
+
+def patch_logging():
+    """Patch logging FileHandler to redirect paths"""
+    logging.FileHandler = RedirectingFileHandler
+    print("[Patches] ✅ Logging FileHandler patched")
+
+def apply_all_patches():
+    """Apply all GPU and system patches"""
+    patch_tensorboard()
+    patch_logging()