michon commited on
Commit
4d1ecfd
·
1 Parent(s): 11b882b

Delete unused/old files

Browse files
app.py DELETED
@@ -1,9 +0,0 @@
1
- """
2
- Entry point for Hugging Face Spaces
3
- """
4
- import uvicorn
5
- # This imports the 'app' from your new modular folder structure
6
- from mrrrme.backend.app import app
7
-
8
- if __name__ == "__main__":
9
- uvicorn.run(app, host="0.0.0.0", port=8000)
 
 
 
 
 
 
 
 
 
 
model/AU_model.py DELETED
@@ -1,112 +0,0 @@
1
- import torch
2
- import torch.nn as nn
3
- import numpy as np
4
- import torch.nn.functional as F
5
- import math
6
-
7
- def normalize_digraph(A):
8
- b, n, _ = A.shape
9
- node_degrees = A.detach().sum(dim = -1)
10
- degs_inv_sqrt = node_degrees ** -0.5
11
- norm_degs_matrix = torch.eye(n)
12
- dev = A.get_device()
13
- if dev >= 0:
14
- norm_degs_matrix = norm_degs_matrix.to(dev)
15
- norm_degs_matrix = norm_degs_matrix.view(1, n, n) * degs_inv_sqrt.view(b, n, 1)
16
- norm_A = torch.bmm(torch.bmm(norm_degs_matrix,A),norm_degs_matrix)
17
- return norm_A
18
-
19
-
20
- class GNN(nn.Module):
21
- def __init__(self, in_channels, num_classes, neighbor_num=4, metric='dots'):
22
- super(GNN, self).__init__()
23
- # in_channels: dim of node feature
24
- # num_classes: num of nodes
25
- # neighbor_num: K in paper and we select the top-K nearest neighbors for each node feature.
26
- # metric: metric for assessing node similarity. Used in FGG module to build a dynamical graph
27
- # X' = ReLU(X + BN(V(X) + A x U(X)) )
28
-
29
- self.in_channels = in_channels
30
- self.num_classes = num_classes
31
- self.relu = nn.ReLU()
32
- self.metric = metric
33
- self.neighbor_num = neighbor_num
34
-
35
- # network
36
- self.U = nn.Linear(self.in_channels,self.in_channels)
37
- self.V = nn.Linear(self.in_channels,self.in_channels)
38
- self.bnv = nn.BatchNorm1d(num_classes)
39
-
40
- # init
41
- self.U.weight.data.normal_(0, math.sqrt(2. / self.in_channels))
42
- self.V.weight.data.normal_(0, math.sqrt(2. / self.in_channels))
43
- self.bnv.weight.data.fill_(1)
44
- self.bnv.bias.data.zero_()
45
-
46
- def forward(self, x):
47
- b, n, c = x.shape
48
-
49
- # build dynamical graph
50
- if self.metric == 'dots':
51
- si = x.detach()
52
- si = torch.einsum('b i j , b j k -> b i k', si, si.transpose(1, 2))
53
- threshold = si.topk(k=self.neighbor_num, dim=-1, largest=True)[0][:, :, -1].view(b, n, 1)
54
- adj = (si >= threshold).float()
55
-
56
- elif self.metric == 'cosine':
57
- si = x.detach()
58
- si = F.normalize(si, p=2, dim=-1)
59
- si = torch.einsum('b i j , b j k -> b i k', si, si.transpose(1, 2))
60
- threshold = si.topk(k=self.neighbor_num, dim=-1, largest=True)[0][:, :, -1].view(b, n, 1)
61
- adj = (si >= threshold).float()
62
-
63
- elif self.metric == 'l1':
64
- si = x.detach().repeat(1, n, 1).view(b, n, n, c)
65
- si = torch.abs(si.transpose(1, 2) - si)
66
- si = si.sum(dim=-1)
67
- threshold = si.topk(k=self.neighbor_num, dim=-1, largest=False)[0][:, :, -1].view(b, n, 1)
68
- adj = (si <= threshold).float()
69
-
70
- else:
71
- raise Exception("Error: wrong metric: ", self.metric)
72
-
73
- # GNN process
74
- A = normalize_digraph(adj)
75
- aggregate = torch.einsum('b i j, b j k->b i k', A, self.V(x))
76
- x = self.relu(x + self.bnv(aggregate + self.U(x)))
77
- return x
78
-
79
-
80
- class Head(nn.Module):
81
- def __init__(self, in_channels, num_classes, neighbor_num=4, metric='dots'):
82
- super(Head, self).__init__()
83
- self.in_channels = in_channels
84
- self.num_classes = num_classes
85
- class_linear_layers = []
86
- for i in range(self.num_classes):
87
- layer = nn.Linear(self.in_channels, self.in_channels)
88
- class_linear_layers += [layer]
89
- self.class_linears = nn.ModuleList(class_linear_layers)
90
- self.gnn = GNN(self.in_channels, self.num_classes,neighbor_num=neighbor_num,metric=metric)
91
- self.sc = nn.Parameter(torch.FloatTensor(torch.zeros(self.num_classes, self.in_channels)))
92
- self.relu = nn.ReLU()
93
-
94
- nn.init.xavier_uniform_(self.sc)
95
-
96
- def forward(self, x):
97
- # AFG
98
- f_u = []
99
- for i, layer in enumerate(self.class_linears):
100
- f_u.append(layer(x).unsqueeze(1))
101
- f_u = torch.cat(f_u, dim=1)
102
- # f_v = f_u.mean(dim=-2)
103
- # FGG
104
- f_v = self.gnn(f_u)
105
- # f_v = self.gnn(f_v)
106
- b, n, c = f_v.shape
107
- sc = self.sc
108
- sc = self.relu(sc)
109
- sc = F.normalize(sc, p=2, dim=-1)
110
- cl = F.normalize(f_v, p=2, dim=-1)
111
- cl = (cl * sc.view(1, n, c)).sum(dim=-1)
112
- return cl
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
model/AutomaticWeightedLoss.py DELETED
@@ -1,31 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- import torch
4
- import torch.nn as nn
5
-
6
- class AutomaticWeightedLoss(nn.Module):
7
- """automatically weighted multi-task loss
8
-
9
- Params:
10
- num: int,the number of loss
11
- x: multi-task loss
12
- Examples:
13
- loss1=1
14
- loss2=2
15
- awl = AutomaticWeightedLoss(2)
16
- loss_sum = awl(loss1, loss2)
17
- """
18
- def __init__(self, num=2):
19
- super(AutomaticWeightedLoss, self).__init__()
20
- params = torch.ones(num, requires_grad=True)
21
- self.params = torch.nn.Parameter(params)
22
-
23
- def forward(self, *x):
24
- loss_sum = 0
25
- for i, loss in enumerate(x):
26
- loss_sum += 0.5 / (self.params[i] ** 2) * loss + torch.log(1 + self.params[i] ** 2)
27
- return loss_sum
28
-
29
- if __name__ == '__main__':
30
- awl = AutomaticWeightedLoss(2)
31
- print(awl.parameters())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
model/MLT.py DELETED
@@ -1,38 +0,0 @@
1
- import torch
2
- import torch.nn as nn
3
- import timm
4
-
5
- from .AU_model import *
6
-
7
- class MLT(nn.Module):
8
- def __init__(self, base_model_name='tf_efficientnet_b0_ns', expr_classes=8, au_numbers=8):
9
- super(MLT, self).__init__()
10
- self.base_model = timm.create_model(base_model_name, pretrained=False)
11
- self.base_model.classifier = nn.Identity()
12
-
13
- feature_dim = self.base_model.num_features
14
-
15
- self.relu = nn.ReLU()
16
-
17
- self.fc_emotion = nn.Linear(feature_dim, feature_dim)
18
- self.fc_gaze = nn.Linear(feature_dim, feature_dim)
19
- self.fc_au = nn.Linear(feature_dim, feature_dim)
20
-
21
- self.emotion_classifier = nn.Linear(feature_dim, expr_classes)
22
- self.gaze_regressor = nn.Linear(feature_dim, 2)
23
- # self.au_regressor = nn.Linear(feature_dim, au_numbers)
24
- self.au_regressor = Head(in_channels=feature_dim, num_classes=au_numbers, neighbor_num=4, metric='dots')
25
-
26
- def forward(self, x):
27
- features = self.base_model(x)
28
-
29
- features_emotion = self.relu(self.fc_emotion(features))
30
- features_gaze = self.relu(self.fc_gaze(features))
31
- features_au = self.relu(self.fc_au(features))
32
-
33
- emotion_output = self.emotion_classifier(features_emotion)
34
- gaze_output = self.gaze_regressor(features_gaze)
35
- # au_output = torch.sigmoid(self.au_regressor(features_au))
36
- au_output = self.au_regressor(features_au)
37
-
38
- return emotion_output, gaze_output, au_output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mrrrme/audio/voice_assistant.py DELETED
@@ -1,227 +0,0 @@
1
- """Text-to-Speech using Coqui XTTS v2 (Multi-lingual)"""
2
- import os
3
- import time
4
- import tempfile
5
- import threading
6
- import pygame
7
- import torch
8
- import numpy as np
9
- from dotenv import load_dotenv
10
- from TTS.api import TTS # Coqui TTS
11
-
12
- load_dotenv()
13
-
14
- # XTTS v2 Default Speakers
15
- # Replaced "Andrew Chipper" with "Damien Black" (Confirmed Male)
16
- VOICE_MAP = {
17
- "female": "Ana Florence",
18
- "male": "Damien Black",
19
- "Happy": "Ana Florence",
20
- "Sad": "Ana Florence",
21
- "Angry": "Damien Black",
22
- "Neutral": "Ana Florence",
23
- }
24
-
25
- class VoiceAssistant:
26
- """Coqui XTTS v2 TTS"""
27
-
28
- def __init__(self, voice: str = "female", rate: float = 1.0, language: str = "en"):
29
- self.voice_key = voice
30
- self.voice_name = VOICE_MAP.get(voice, "Ana Florence")
31
- self.rate = rate
32
- self.language = language
33
-
34
- self.counter = 0
35
- self.is_speaking = False
36
- self.speaking_lock = threading.Lock()
37
- self.audio_workers = []
38
-
39
- print(f"[TTS] 🚀 Initializing Coqui XTTS v2...")
40
-
41
- # Initialize Coqui TTS with XTTS v2 model
42
- # gpu=True will use CUDA if available
43
- try:
44
- device = "cuda" if torch.cuda.is_available() else "cpu"
45
- print(f"[TTS] 📥 Loading XTTS v2 model on {device} (this may take time on first run)...")
46
-
47
- self.tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device)
48
-
49
- print(f"[TTS] ✅ XTTS v2 model loaded")
50
- except Exception as e:
51
- print(f"[TTS] ⚠️ XTTS init error: {e}")
52
- self.tts = None
53
-
54
- print("[TTS] 🔧 Initializing pygame...")
55
- try:
56
- pygame.mixer.quit()
57
- pygame.mixer.init(frequency=24000, size=-16, channels=1, buffer=2048)
58
- print(f"[TTS] ✅ Pygame ready")
59
- except Exception as e:
60
- print(f"[TTS] ⚠️ Pygame warning: {e}")
61
-
62
- print(f"[TTS] ✅ Ready ({self.voice_name}, {language})\n")
63
-
64
- def register_audio_worker(self, worker):
65
- self.audio_workers.append(worker)
66
- print(f"[TTS] ✅ Registered: {worker.__class__.__name__}")
67
-
68
- def set_voice(self, voice_key: str):
69
- """Switch between male/female voices"""
70
- if voice_key in VOICE_MAP:
71
- self.voice_name = VOICE_MAP[voice_key]
72
- self.voice_key = voice_key
73
- print(f"[TTS] 🎙️ Voice → {self.voice_name}")
74
- else:
75
- # If user passes a raw speaker name that exists in XTTS
76
- self.voice_name = voice_key
77
- print(f"[TTS] 🎙️ Voice → {self.voice_name} (Custom)")
78
-
79
- def set_language(self, language: str):
80
- """Set language (e.g., 'en', 'nl')"""
81
- self.language = language
82
- print(f"[TTS] 🌍 Language → {language}")
83
-
84
- def set_rate(self, rate: float):
85
- """
86
- Note: XTTS v2 does not natively support speed control via API in the same way.
87
- This is kept for compatibility but might not affect generation speed directly.
88
- """
89
- self.rate = max(0.5, min(2.0, rate))
90
- print(f"[TTS] 🎚️ Rate → {self.rate}x (XTTS may ignore this)")
91
-
92
- def apply_emotion_voice(self, emotion: str, intensity: float = 0.5):
93
- """
94
- Adjusts internal state based on emotion.
95
- Note: XTTS implies emotion via the input text or style transfer (if enabled).
96
- For now, we just log it or adjust simple parameters.
97
- """
98
- if emotion == "Happy":
99
- self.rate = 1.1
100
- elif emotion == "Sad":
101
- self.rate = 0.9
102
- elif emotion == "Angry":
103
- self.rate = 1.2
104
- else:
105
- self.rate = 1.0
106
-
107
- def stop(self):
108
- print("[TTS] 🛑 STOP")
109
- try:
110
- pygame.mixer.music.stop()
111
- pygame.mixer.music.unload()
112
- except:
113
- pass
114
-
115
- with self.speaking_lock:
116
- self.is_speaking = False
117
-
118
- for worker in self.audio_workers:
119
- if hasattr(worker, 'resume_listening'):
120
- try:
121
- worker.resume_listening()
122
- except:
123
- pass
124
-
125
- def _get_unique_filename(self, ext: str = ".wav"):
126
- self.counter += 1
127
- return os.path.join(tempfile.gettempdir(), f"xtts_{self.counter}_{int(time.time() * 1000)}{ext}")
128
-
129
- def _generate_speech(self, text: str, filename: str):
130
- """Generate speech using Coqui XTTS v2"""
131
- try:
132
- if self.tts is None:
133
- print("[TTS] ❌ Model not initialized")
134
- return False
135
-
136
- print(f"[TTS] 🔧 Generating with {self.voice_name} ({self.language})...")
137
- start = time.time()
138
-
139
- # XTTS v2 Generation
140
- self.tts.tts_to_file(
141
- text=text,
142
- file_path=filename,
143
- speaker=self.voice_name,
144
- language=self.language,
145
- split_sentences=True
146
- )
147
-
148
- gen_time = time.time() - start
149
- print(f"[TTS] ✅ Generated in {gen_time:.2f}s")
150
- return True
151
-
152
- except Exception as e:
153
- print(f"[TTS] ❌ Error: {e}")
154
- import traceback
155
- traceback.print_exc()
156
- return False
157
-
158
- def _play_audio(self, filename: str):
159
- try:
160
- if not os.path.exists(filename):
161
- return False
162
-
163
- print(f"[TTS] ▶️ Playing...")
164
- pygame.mixer.music.load(filename)
165
- pygame.mixer.music.play()
166
-
167
- while pygame.mixer.music.get_busy():
168
- pygame.time.Clock().tick(20)
169
-
170
- pygame.mixer.music.unload()
171
- print(f"[TTS] ✅ Done")
172
- return True
173
-
174
- except Exception as e:
175
- print(f"[TTS] ❌ Play error: {e}")
176
- return False
177
-
178
- def speak(self, text: str):
179
- if not text or not text.strip():
180
- return
181
-
182
- print(f"\n[TTS] 🔊 Speaking ({self.language}): '{text[:80]}...'")
183
-
184
- # Pause workers (listening)
185
- for worker in self.audio_workers:
186
- if hasattr(worker, 'pause_listening'):
187
- try:
188
- worker.pause_listening()
189
- except:
190
- pass
191
-
192
- with self.speaking_lock:
193
- self.is_speaking = True
194
-
195
- temp_file = self._get_unique_filename(".wav")
196
-
197
- try:
198
- if self._generate_speech(text, temp_file):
199
- self._play_audio(temp_file)
200
- try:
201
- if os.path.exists(temp_file):
202
- os.remove(temp_file)
203
- except:
204
- pass
205
-
206
- except Exception as e:
207
- print(f"[TTS] ❌ Error: {e}")
208
- finally:
209
- with self.speaking_lock:
210
- self.is_speaking = False
211
-
212
- time.sleep(0.2)
213
-
214
- # Resume workers
215
- for worker in self.audio_workers:
216
- if hasattr(worker, 'resume_listening'):
217
- try:
218
- worker.resume_listening()
219
- except:
220
- pass
221
-
222
- def speak_async(self, text: str):
223
- threading.Thread(target=self.speak, args=(text,), daemon=True).start()
224
-
225
- def get_is_speaking(self) -> bool:
226
- with self.speaking_lock:
227
- return self.is_speaking
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mrrrme/avatar/avatar_controller.py DELETED
@@ -1,127 +0,0 @@
1
- """Avatar Controller - Integrates avatar with MrrrMe pipeline"""
2
- import threading
3
- import time
4
- import requests
5
-
6
-
7
- class AvatarController:
8
- """Sends speech to avatar backend instead of playing locally"""
9
-
10
- def __init__(self, server_url: str = "http://localhost:8765"):
11
- self.server_url = server_url
12
- self.is_speaking = False
13
- self.speaking_lock = threading.Lock()
14
- self.audio_workers = []
15
-
16
- print(f"[AvatarController] Initializing...")
17
- print(f"[AvatarController] Backend: {server_url}")
18
-
19
- # Test connection
20
- try:
21
- response = requests.get(f"{server_url}/", timeout=2)
22
- data = response.json()
23
- print(f"[AvatarController] ✅ Backend connected")
24
- print(f"[AvatarController] Available voices: {data.get('voices_available', [])}")
25
- except:
26
- print(f"[AvatarController] ⚠️ Backend not responding!")
27
-
28
- def register_audio_worker(self, worker):
29
- self.audio_workers.append(worker)
30
- print(f"[AvatarController] Registered worker: {worker.__class__.__name__}")
31
-
32
- def apply_emotion_voice(self, emotion: str, intensity: float):
33
- pass
34
-
35
- def speak(self, text: str, voice: str = "female", language: str = "en"):
36
- """Send text to avatar backend with voice and language preferences"""
37
- if not text or not text.strip():
38
- return
39
-
40
- t_start = time.time()
41
- print(f"\n{'='*50}")
42
- print(f"[AvatarController] Starting TTS")
43
- print(f"[AvatarController] Text: '{text[:60]}...'")
44
- print(f"[AvatarController] Voice: {voice}, Language: {language}")
45
-
46
- # Pause workers
47
- paused_count = 0
48
- for worker in self.audio_workers:
49
- if hasattr(worker, 'pause_listening'):
50
- try:
51
- worker.pause_listening()
52
- paused_count += 1
53
- except Exception as e:
54
- print(f"[AvatarController] ⚠️ Failed to pause: {e}")
55
-
56
- print(f"[AvatarController] Paused {paused_count} workers")
57
- time.sleep(0.1)
58
-
59
- with self.speaking_lock:
60
- self.is_speaking = True
61
-
62
- try:
63
- print(f"[AvatarController] Sending to backend...")
64
-
65
- response = requests.post(
66
- f"{self.server_url}/speak",
67
- data={
68
- "text": text,
69
- "voice": voice,
70
- "language": language
71
- },
72
- timeout=45
73
- )
74
-
75
- if response.status_code == 200:
76
- data = response.json()
77
- duration = data.get('duration', len(text) * 0.05)
78
-
79
- print(f"[AvatarController] ✅ TTS generated")
80
- print(f"[AvatarController] Duration: {duration:.1f}s")
81
-
82
- # Wait for playback
83
- time.sleep(duration + 1.0)
84
-
85
- print(f"[AvatarController] ✅ Playback complete")
86
- else:
87
- print(f"[AvatarController] ❌ Backend error: {response.status_code}")
88
- print(f"[AvatarController] Response: {response.text}")
89
- time.sleep(2)
90
-
91
- except requests.exceptions.ConnectionError:
92
- print(f"[AvatarController] ❌ Cannot connect to {self.server_url}")
93
- time.sleep(2)
94
- except Exception as e:
95
- print(f"[AvatarController] ❌ Error: {e}")
96
- time.sleep(2)
97
-
98
- finally:
99
- with self.speaking_lock:
100
- self.is_speaking = False
101
-
102
- time.sleep(0.5)
103
-
104
- # Resume workers
105
- resumed_count = 0
106
- for worker in self.audio_workers:
107
- if hasattr(worker, 'resume_listening'):
108
- try:
109
- worker.resume_listening()
110
- resumed_count += 1
111
- except: pass
112
-
113
- t_end = time.time()
114
- print(f"[AvatarController] Resumed {resumed_count} workers")
115
- print(f"[AvatarController] Total time: {t_end-t_start:.2f}s")
116
- print(f"{'='*50}\n")
117
-
118
- def speak_async(self, text: str, voice: str = "female", language: str = "en"):
119
- """Speak asynchronously with voice and language"""
120
- threading.Thread(target=self.speak, args=(text, voice, language), daemon=True).start()
121
-
122
- def get_is_speaking(self) -> bool:
123
- with self.speaking_lock:
124
- return self.is_speaking
125
-
126
- def stop(self):
127
- pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mrrrme/backend_server_old.py DELETED
@@ -1,1123 +0,0 @@
1
- """MrrrMe Backend WebSocket Server - ENHANCED LOGGING VERSION"""
2
- import os
3
- import sys
4
-
5
- # ===== SET CACHE DIRECTORIES FIRST =====
6
- os.environ['HF_HOME'] = '/tmp/huggingface'
7
- os.environ['TRANSFORMERS_CACHE'] = '/tmp/transformers'
8
- os.environ['HF_HUB_CACHE'] = '/tmp/huggingface/hub'
9
- os.environ['TORCH_HOME'] = '/tmp/torch'
10
- os.makedirs('/tmp/huggingface', exist_ok=True)
11
- os.makedirs('/tmp/transformers', exist_ok=True)
12
- os.makedirs('/tmp/huggingface/hub', exist_ok=True)
13
- os.makedirs('/tmp/torch', exist_ok=True)
14
-
15
- # ===== GPU FIX: Patch TensorBoard =====
16
- class DummySummaryWriter:
17
- def __init__(self, *args, **kwargs): pass
18
- def __getattr__(self, name): return lambda *args, **kwargs: None
19
-
20
- try:
21
- import tensorboardX
22
- tensorboardX.SummaryWriter = DummySummaryWriter
23
- except: pass
24
-
25
- # ===== GPU FIX: Patch Logging to redirect /work paths =====
26
- import logging
27
- _original_FileHandler = logging.FileHandler
28
-
29
- class RedirectingFileHandler(_original_FileHandler):
30
- def __init__(self, filename, mode='a', encoding=None, delay=False, errors=None):
31
- if isinstance(filename, str) and filename.startswith('/work'):
32
- filename = '/tmp/openface_log.txt'
33
- os.makedirs(os.path.dirname(filename) if os.path.dirname(filename) else '/tmp', exist_ok=True)
34
- super().__init__(filename, mode, encoding, delay, errors)
35
-
36
- logging.FileHandler = RedirectingFileHandler
37
-
38
- # Now import everything else
39
- import asyncio
40
- import json
41
- import base64
42
- import numpy as np
43
- import cv2
44
- import io
45
- import torch
46
- from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
47
- from fastapi.middleware.cors import CORSMiddleware
48
- from pydantic import BaseModel
49
- import requests
50
- from PIL import Image
51
- from typing import Optional
52
- import sqlite3
53
- import secrets
54
- import hashlib
55
- from datetime import datetime
56
-
57
- # Check GPU
58
- if not torch.cuda.is_available():
59
- print("[Backend] ⚠️ No GPU detected - using CPU mode")
60
- else:
61
- print(f"[Backend] ✅ GPU available: {torch.cuda.get_device_name(0)}")
62
-
63
- app = FastAPI()
64
-
65
- # CORS for browser access
66
- app.add_middleware(
67
- CORSMiddleware,
68
- allow_origins=["*"],
69
- allow_credentials=True,
70
- allow_methods=["*"],
71
- allow_headers=["*"],
72
- )
73
-
74
- # Global model variables (will be loaded after startup)
75
- face_processor = None
76
- text_analyzer = None
77
- whisper_worker = None
78
- voice_worker = None
79
- llm_generator = None
80
- fusion_engine = None
81
- models_ready = False
82
-
83
- # Avatar backend URL - environment aware
84
- def get_avatar_api_url():
85
- """Get correct avatar API URL based on environment"""
86
- # For Hugging Face Spaces, use same host
87
- if os.path.exists('/.dockerenv') or os.environ.get('SPACE_ID'):
88
- # Running in Docker/HF Spaces - use internal networking
89
- return "http://127.0.0.1:8765"
90
- else:
91
- # Local development
92
- return "http://localhost:8765"
93
-
94
- AVATAR_API = get_avatar_api_url()
95
- print(f"[Backend] 🎭 Avatar API URL: {AVATAR_API}")
96
-
97
- # ===== AUTHENTICATION & DATABASE =====
98
- # Use /data for Hugging Face Spaces (persistent) or /tmp for local dev
99
- if os.path.exists('/data'):
100
- DB_PATH = "/data/mrrrme_users.db"
101
- print("[Backend] 📁 Using persistent storage: /data/mrrrme_users.db")
102
- else:
103
- DB_PATH = "/tmp/mrrrme_users.db"
104
- print("[Backend] ⚠️ Using ephemeral storage: /tmp/mrrrme_users.db (will reset on rebuild!)")
105
- print("[Backend] ⚠️ To persist data, enable persistent storage in HF Spaces settings")
106
-
107
- class SignupRequest(BaseModel):
108
- username: str
109
- password: str
110
-
111
- class LoginRequest(BaseModel):
112
- username: str
113
- password: str
114
-
115
- def init_db():
116
- conn = sqlite3.connect(DB_PATH)
117
- cursor = conn.cursor()
118
-
119
- cursor.execute("""
120
- CREATE TABLE IF NOT EXISTS users (
121
- user_id TEXT PRIMARY KEY,
122
- username TEXT UNIQUE NOT NULL,
123
- password_hash TEXT NOT NULL,
124
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
125
- )
126
- """)
127
-
128
- cursor.execute("""
129
- CREATE TABLE IF NOT EXISTS sessions (
130
- session_id TEXT PRIMARY KEY,
131
- user_id TEXT NOT NULL,
132
- token TEXT UNIQUE NOT NULL,
133
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
134
- is_active BOOLEAN DEFAULT 1
135
- )
136
- """)
137
-
138
- cursor.execute("""
139
- CREATE TABLE IF NOT EXISTS messages (
140
- message_id INTEGER PRIMARY KEY AUTOINCREMENT,
141
- session_id TEXT NOT NULL,
142
- role TEXT NOT NULL,
143
- content TEXT NOT NULL,
144
- emotion TEXT,
145
- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
146
- )
147
- """)
148
-
149
- cursor.execute("""
150
- CREATE TABLE IF NOT EXISTS user_summaries (
151
- user_id TEXT PRIMARY KEY,
152
- summary_text TEXT NOT NULL,
153
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
154
- )
155
- """)
156
-
157
- conn.commit()
158
- conn.close()
159
-
160
- init_db()
161
-
162
- def hash_password(pw: str) -> str:
163
- return hashlib.sha256(pw.encode()).hexdigest()
164
-
165
- @app.post("/api/signup")
166
- async def signup(req: SignupRequest):
167
- conn = sqlite3.connect(DB_PATH)
168
- cursor = conn.cursor()
169
-
170
- try:
171
- user_id = secrets.token_urlsafe(16)
172
- cursor.execute(
173
- "INSERT INTO users (user_id, username, password_hash) VALUES (?, ?, ?)",
174
- (user_id, req.username, hash_password(req.password))
175
- )
176
- conn.commit()
177
- conn.close()
178
- return {"success": True, "message": "Account created!"}
179
- except sqlite3.IntegrityError:
180
- conn.close()
181
- raise HTTPException(status_code=400, detail="Username already exists")
182
-
183
- @app.post("/api/login")
184
- async def login(req: LoginRequest):
185
- conn = sqlite3.connect(DB_PATH)
186
- cursor = conn.cursor()
187
-
188
- cursor.execute(
189
- "SELECT user_id, username FROM users WHERE username = ? AND password_hash = ?",
190
- (req.username, hash_password(req.password))
191
- )
192
-
193
- result = cursor.fetchone()
194
-
195
- if not result:
196
- conn.close()
197
- raise HTTPException(status_code=401, detail="Invalid credentials")
198
-
199
- user_id, username = result
200
-
201
- session_id = secrets.token_urlsafe(16)
202
- token = secrets.token_urlsafe(32)
203
-
204
- cursor.execute(
205
- "INSERT INTO sessions (session_id, user_id, token) VALUES (?, ?, ?)",
206
- (session_id, user_id, token)
207
- )
208
-
209
- cursor.execute(
210
- "SELECT summary_text FROM user_summaries WHERE user_id = ?",
211
- (user_id,)
212
- )
213
- summary_row = cursor.fetchone()
214
- summary = summary_row[0] if summary_row else None
215
-
216
- conn.commit()
217
- conn.close()
218
-
219
- return {
220
- "success": True,
221
- "token": token,
222
- "username": username,
223
- "user_id": user_id,
224
- "summary": summary
225
- }
226
-
227
- class LogoutRequest(BaseModel):
228
- token: str
229
-
230
- @app.post("/api/logout")
231
- async def logout(req: LogoutRequest):
232
- conn = sqlite3.connect(DB_PATH)
233
- cursor = conn.cursor()
234
-
235
- # Get session info before closing
236
- cursor.execute(
237
- "SELECT session_id, user_id FROM sessions WHERE token = ? AND is_active = 1",
238
- (req.token,)
239
- )
240
- result = cursor.fetchone()
241
-
242
- if result:
243
- session_id, user_id = result
244
-
245
- # Mark session as inactive
246
- cursor.execute(
247
- "UPDATE sessions SET is_active = 0 WHERE token = ?",
248
- (req.token,)
249
- )
250
- conn.commit()
251
- conn.close()
252
-
253
- # Generate summary on explicit logout
254
- print(f"[Logout] 📝 Generating summary for user {user_id}...")
255
- summary = await generate_session_summary(session_id, user_id)
256
- if summary:
257
- print(f"[Logout] ✅ Summary generated")
258
-
259
- return {"success": True, "message": "Logged out successfully"}
260
- else:
261
- conn.close()
262
- return {"success": True, "message": "Session already closed"}
263
-
264
- async def generate_session_summary(session_id: str, user_id: str):
265
- """Generate AI summary of conversation for THIS specific user"""
266
- conn = sqlite3.connect(DB_PATH)
267
- cursor = conn.cursor()
268
-
269
- # Verify session belongs to user
270
- cursor.execute(
271
- "SELECT user_id FROM sessions WHERE session_id = ?",
272
- (session_id,)
273
- )
274
- session_owner = cursor.fetchone()
275
-
276
- if not session_owner or session_owner[0] != user_id:
277
- print(f"[Summary] ❌ Security error: session {session_id} doesn't belong to user {user_id}")
278
- conn.close()
279
- return None
280
-
281
- # Get messages from this session
282
- cursor.execute(
283
- "SELECT role, content, emotion FROM messages WHERE session_id = ? ORDER BY timestamp ASC",
284
- (session_id,)
285
- )
286
-
287
- messages = cursor.fetchall()
288
-
289
- # Get username for better logging
290
- cursor.execute("SELECT username FROM users WHERE user_id = ?", (user_id,))
291
- username_row = cursor.fetchone()
292
- username = username_row[0] if username_row else user_id
293
-
294
- conn.close()
295
-
296
- if len(messages) < 3:
297
- print(f"[Summary] ⏭️ Skipped for {username} (only {len(messages)} messages)")
298
- return None
299
-
300
- conversation = ""
301
- for role, content, emotion in messages:
302
- speaker = "User" if role == "user" else "AI"
303
- emo_tag = f" [{emotion}]" if emotion else ""
304
- conversation += f"{speaker}{emo_tag}: {content}\n"
305
-
306
- try:
307
- from groq import Groq
308
- groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
309
-
310
- prompt = f"""Analyze this conversation and create a 2-3 sentence summary about THIS SPECIFIC USER.
311
-
312
- DO NOT include information about other users or other conversations.
313
- ONLY summarize what THIS user said and their patterns.
314
-
315
- Conversation ({len(messages)} messages):
316
- {conversation}
317
-
318
- Create a concise summary including: topics this user discussed, their emotional patterns, personal details THEY mentioned, and their preferences."""
319
-
320
- response = groq_client.chat.completions.create(
321
- model="llama-3.1-8b-instant",
322
- messages=[{"role": "user", "content": prompt}],
323
- max_tokens=150,
324
- temperature=0.7
325
- )
326
-
327
- summary = response.choices[0].message.content.strip()
328
-
329
- # Save summary FOR THIS USER ONLY
330
- conn = sqlite3.connect(DB_PATH)
331
- cursor = conn.cursor()
332
-
333
- cursor.execute(
334
- "INSERT OR REPLACE INTO user_summaries (user_id, summary_text, updated_at) VALUES (?, ?, ?)",
335
- (user_id, summary, datetime.now())
336
- )
337
-
338
- conn.commit()
339
- conn.close()
340
-
341
- print(f"[Summary] ✅ Generated for {username} (user_id: {user_id})")
342
- print(f"[Summary] 📝 Content: {summary}")
343
- return summary
344
-
345
- except Exception as e:
346
- print(f"[Summary] ❌ Error for {username}: {e}")
347
- import traceback
348
- traceback.print_exc()
349
- return None
350
-
351
- @app.on_event("startup")
352
- async def startup_event():
353
- """Start loading models in background after server is ready"""
354
- print("[Backend] 🚀 Starting up...")
355
-
356
- # Check if avatar service is running
357
- try:
358
- response = requests.get(f"{AVATAR_API}/", timeout=2)
359
- if response.status_code == 200:
360
- print(f"[Backend] ✅ Avatar TTS service available at {AVATAR_API}")
361
- else:
362
- print(f"[Backend] ⚠️ Avatar TTS service responded with {response.status_code}")
363
- except requests.exceptions.ConnectionError:
364
- print(f"[Backend] ⚠️ Avatar TTS service NOT available at {AVATAR_API}")
365
- print(f"[Backend] 💡 Text-only mode will be used (no avatar speech)")
366
- print(f"[Backend] 📝 To enable avatar:")
367
- print(f"[Backend] cd avatar && python speak_server.py")
368
- except Exception as e:
369
- print(f"[Backend] ⚠️ Error checking avatar service: {e}")
370
-
371
- asyncio.create_task(load_models())
372
-
373
- async def load_models():
374
- """Load all AI models asynchronously"""
375
- global face_processor, text_analyzer, whisper_worker, voice_worker
376
- global llm_generator, fusion_engine, models_ready
377
-
378
- print("[Backend] 🚀 Initializing MrrrMe AI models in background...")
379
-
380
- try:
381
- # Import modules
382
- from mrrrme.vision.face_processor import FaceProcessor
383
- from mrrrme.audio.voice_emotion import VoiceEmotionWorker
384
- from mrrrme.audio.whisper_transcription import WhisperTranscriptionWorker
385
- from mrrrme.nlp.text_sentiment import TextSentimentAnalyzer
386
- from mrrrme.nlp.llm_generator_groq import LLMResponseGenerator
387
- from mrrrme.config import FUSE4
388
-
389
- # Load models
390
- print("[Backend] Loading FaceProcessor...")
391
- face_processor = FaceProcessor()
392
-
393
- print("[Backend] Loading TextSentiment...")
394
- text_analyzer = TextSentimentAnalyzer()
395
-
396
- print("[Backend] Loading Whisper...")
397
- whisper_worker = WhisperTranscriptionWorker(text_analyzer)
398
-
399
- print("[Backend] Loading VoiceEmotion...")
400
- voice_worker = VoiceEmotionWorker(whisper_worker=whisper_worker)
401
-
402
- print("[Backend] Initializing LLM...")
403
- groq_api_key = os.getenv("GROQ_API_KEY", "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")
404
- llm_generator = LLMResponseGenerator(api_key=groq_api_key)
405
-
406
- # Initialize fusion engine
407
- class FusionEngine:
408
- def __init__(self):
409
- self.alpha_face = 0.4
410
- self.alpha_voice = 0.3
411
- self.alpha_text = 0.3
412
-
413
- def fuse(self, face_probs, voice_probs, text_probs):
414
- fused = (
415
- self.alpha_face * face_probs +
416
- self.alpha_voice * voice_probs +
417
- self.alpha_text * text_probs
418
- )
419
- fused = fused / (np.sum(fused) + 1e-8)
420
- fused_idx = int(np.argmax(fused))
421
- fused_emotion = FUSE4[fused_idx]
422
- intensity = float(np.max(fused))
423
- return fused_emotion, intensity
424
-
425
- fusion_engine = FusionEngine()
426
- models_ready = True
427
-
428
- print("[Backend] ✅ All models loaded!")
429
-
430
- except Exception as e:
431
- print(f"[Backend] ❌ Error loading models: {e}")
432
- import traceback
433
- traceback.print_exc()
434
-
435
- @app.get("/")
436
- async def root():
437
- """Root endpoint"""
438
- return {
439
- "status": "running",
440
- "models_ready": models_ready,
441
- "message": "MrrrMe AI Backend"
442
- }
443
-
444
- @app.get("/health")
445
- async def health():
446
- """Health check - responds immediately"""
447
- return {
448
- "status": "healthy",
449
- "models_ready": models_ready
450
- }
451
-
452
- @app.get("/api/debug/users")
453
- async def debug_users():
454
- """Debug endpoint - view all users and their summaries"""
455
- conn = sqlite3.connect(DB_PATH)
456
- cursor = conn.cursor()
457
-
458
- cursor.execute("""
459
- SELECT u.username, u.user_id, s.summary_text, s.updated_at
460
- FROM users u
461
- LEFT JOIN user_summaries s ON u.user_id = s.user_id
462
- ORDER BY u.created_at DESC
463
- """)
464
-
465
- users = []
466
- for username, user_id, summary, updated in cursor.fetchall():
467
- users.append({
468
- "username": username,
469
- "user_id": user_id,
470
- "summary": summary,
471
- "summary_updated": updated
472
- })
473
-
474
- conn.close()
475
-
476
- return {"users": users, "database": DB_PATH}
477
-
478
- @app.get("/api/debug/sessions")
479
- async def debug_sessions():
480
- """Debug endpoint - view all active sessions"""
481
- conn = sqlite3.connect(DB_PATH)
482
- cursor = conn.cursor()
483
-
484
- cursor.execute("""
485
- SELECT s.session_id, s.token, u.username, s.is_active, s.created_at
486
- FROM sessions s
487
- JOIN users u ON s.user_id = u.user_id
488
- ORDER BY s.created_at DESC
489
- LIMIT 20
490
- """)
491
-
492
- sessions = []
493
- for session_id, token, username, is_active, created_at in cursor.fetchall():
494
- sessions.append({
495
- "session_id": session_id,
496
- "token_preview": token[:10] + "..." if token else None,
497
- "username": username,
498
- "is_active": bool(is_active),
499
- "created_at": created_at
500
- })
501
-
502
- conn.close()
503
-
504
- return {"sessions": sessions, "database": DB_PATH}
505
-
506
- @app.websocket("/ws")
507
- async def websocket_endpoint(websocket: WebSocket):
508
- await websocket.accept()
509
- print("[WebSocket] ✅ Client connected!")
510
-
511
- # ===== AUTHENTICATION =====
512
- session_data = None
513
- user_summary = None
514
- session_id = None
515
- user_id = None
516
- username = None
517
-
518
- try:
519
- auth_msg = await websocket.receive_json()
520
- print(f"[WebSocket] 📨 Auth message received: {auth_msg.get('type')}")
521
-
522
- if auth_msg.get("type") != "auth":
523
- print(f"[WebSocket] ❌ Wrong message type: {auth_msg.get('type')}")
524
- await websocket.send_json({"type": "error", "message": "Authentication required"})
525
- return
526
-
527
- token = auth_msg.get("token")
528
- print(f"[WebSocket] 🔑 Validating token: {token[:10] if token else 'None'}...")
529
-
530
- if not token:
531
- print(f"[WebSocket] ❌ No token provided!")
532
- await websocket.send_json({"type": "error", "message": "No token provided"})
533
- return
534
-
535
- # Validate token
536
- conn = sqlite3.connect(DB_PATH)
537
- cursor = conn.cursor()
538
-
539
- cursor.execute(
540
- "SELECT s.session_id, s.user_id, u.username FROM sessions s JOIN users u ON s.user_id = u.user_id WHERE s.token = ? AND s.is_active = 1",
541
- (token,)
542
- )
543
-
544
- result = cursor.fetchone()
545
-
546
- if not result:
547
- # Debug: Check if token exists at all
548
- cursor.execute("SELECT session_id, user_id, is_active FROM sessions WHERE token = ?", (token,))
549
- debug_result = cursor.fetchone()
550
-
551
- if debug_result:
552
- print(f"[WebSocket] ⚠️ Token found but session inactive or invalid: {debug_result}")
553
- else:
554
- print(f"[WebSocket] ❌ Token not found in database!")
555
-
556
- await websocket.send_json({"type": "error", "message": "Invalid session - please login again"})
557
- conn.close()
558
- return
559
-
560
- session_id, user_id, username = result
561
- print(f"[WebSocket] ✅ Token validated for user: {username} (session: {session_id})")
562
-
563
- # Get user-specific summary
564
- cursor.execute(
565
- "SELECT summary_text FROM user_summaries WHERE user_id = ?",
566
- (user_id,)
567
- )
568
- summary_row = cursor.fetchone()
569
- user_summary = summary_row[0] if summary_row else None
570
-
571
- conn.close()
572
-
573
- session_data = {
574
- 'session_id': session_id,
575
- 'user_id': user_id,
576
- 'username': username
577
- }
578
-
579
- # Send authenticated confirmation
580
- await websocket.send_json({
581
- "type": "authenticated",
582
- "username": username,
583
- "summary": user_summary
584
- })
585
-
586
- print(f"[WebSocket] ✅ Authenticated: {username} (user_id: {user_id})")
587
- if user_summary:
588
- print(f"[WebSocket] 📖 Loaded summary: {user_summary[:60]}...")
589
-
590
- # Clear LLM's conversation history
591
- if llm_generator:
592
- llm_generator.clear_history()
593
- print(f"[LLM] 🗑️ Conversation history cleared")
594
-
595
- # Load user's recent conversation history
596
- conn = sqlite3.connect(DB_PATH)
597
- cursor = conn.cursor()
598
- cursor.execute(
599
- """SELECT role, content FROM messages
600
- WHERE session_id IN (
601
- SELECT session_id FROM sessions WHERE user_id = ?
602
- )
603
- ORDER BY timestamp DESC
604
- LIMIT 10""",
605
- (user_id,)
606
- )
607
- user_history = cursor.fetchall()
608
- conn.close()
609
-
610
- # Load user-specific history into LLM
611
- for role, content in reversed(user_history):
612
- llm_generator.conversation_history.append({
613
- "role": role,
614
- "content": content
615
- })
616
-
617
- if user_history:
618
- print(f"[WebSocket] 📚 Loaded {len(user_history)} messages from {username}'s history")
619
-
620
- except Exception as auth_err:
621
- print(f"[WebSocket] ❌ Auth error: {auth_err}")
622
- return
623
-
624
- # Wait for models to load if needed
625
- if not models_ready:
626
- await websocket.send_json({
627
- "type": "status",
628
- "message": "AI models are loading, please wait..."
629
- })
630
-
631
- # Wait up to 15 minutes for models
632
- for _ in range(900):
633
- if models_ready:
634
- await websocket.send_json({
635
- "type": "status",
636
- "message": "Models loaded! Ready to chat."
637
- })
638
- break
639
- await asyncio.sleep(1)
640
-
641
- if not models_ready:
642
- await websocket.send_json({
643
- "type": "error",
644
- "message": "Models failed to load. Please refresh."
645
- })
646
- return
647
-
648
- # Session state
649
- audio_buffer = []
650
- user_preferences = {"voice": "female", "language": "en"}
651
-
652
- try:
653
- while True:
654
- data = await websocket.receive_json()
655
- msg_type = data.get("type")
656
-
657
- # ============ PREFERENCES UPDATE ============
658
- if msg_type == "preferences":
659
- if "voice" in data:
660
- user_preferences["voice"] = data.get("voice", "female")
661
- if "language" in data:
662
- user_preferences["language"] = data.get("language", "en")
663
- print(f"[Preferences] {username}: voice={user_preferences.get('voice')}, language={user_preferences.get('language')}")
664
- continue
665
-
666
- # ============ AUTO-GREETING REQUEST ============
667
- elif msg_type == "request_greeting":
668
- try:
669
- print(f"[WebSocket] 🤖 Generating initial greeting for {username}...")
670
-
671
- # Determine greeting based on user context
672
- greeting_prompts = {
673
- "new": f"Hey {username}! I'm MrrrMe, your emotion AI companion. How are you feeling today?",
674
- "returning": f"Welcome back, {username}! It's great to see you again. How have you been?"
675
- }
676
-
677
- # Check if user has summary (returning user)
678
- greeting_text = greeting_prompts["returning"] if user_summary else greeting_prompts["new"]
679
-
680
- # Add language context
681
- if user_preferences.get("language") == "nl":
682
- if user_summary:
683
- greeting_text = f"Welkom terug, {username}! Fijn je weer te zien. Hoe gaat het met je?"
684
- else:
685
- greeting_text = f"Hoi {username}! Ik ben MrrrMe, jouw emotie AI-metgezel. Hoe voel je je vandaag?"
686
-
687
- print(f"[Greeting] 👋 Sending: '{greeting_text}'")
688
-
689
- # Try to send to avatar for TTS
690
- audio_url = None
691
- visemes = None
692
-
693
- try:
694
- voice_preference = user_preferences.get("voice", "female")
695
- language_preference = user_preferences.get("language", "en")
696
-
697
- print(f"[Greeting] 🔊 Requesting TTS from avatar service...")
698
- avatar_response = requests.post(
699
- f"{AVATAR_API}/speak",
700
- data={
701
- "text": greeting_text,
702
- "voice": voice_preference,
703
- "language": language_preference
704
- },
705
- timeout=10
706
- )
707
-
708
- if avatar_response.status_code == 200:
709
- avatar_data = avatar_response.json()
710
- audio_url = avatar_data.get("audio_url")
711
- visemes = avatar_data.get("visemes")
712
- print(f"[Greeting] ✅ TTS generated successfully")
713
- else:
714
- print(f"[Greeting] ⚠️ TTS failed: {avatar_response.status_code}")
715
-
716
- except requests.exceptions.ConnectionError as conn_err:
717
- print(f"[Greeting] ⚠️ Avatar service not available (port 8765 not responding)")
718
- print(f"[Greeting] 📝 Sending text-only greeting (TTS will be skipped)")
719
-
720
- except Exception as tts_err:
721
- print(f"[Greeting] ⚠️ TTS error: {tts_err}")
722
- print(f"[Greeting] 📝 Sending text-only greeting")
723
-
724
- # Send greeting to client
725
- response_data = {
726
- "type": "llm_response",
727
- "text": greeting_text,
728
- "emotion": "Neutral",
729
- "intensity": 0.5,
730
- "is_greeting": True
731
- }
732
-
733
- # Add audio/visemes only if TTS succeeded
734
- if audio_url and visemes:
735
- response_data["audio_url"] = audio_url
736
- response_data["visemes"] = visemes
737
- else:
738
- response_data["text_only"] = True
739
- print(f"[Greeting] 📝 Sending text-only (no TTS)")
740
-
741
- await websocket.send_json(response_data)
742
-
743
- # Save greeting to history
744
- conn = sqlite3.connect(DB_PATH)
745
- cursor = conn.cursor()
746
- cursor.execute(
747
- "INSERT INTO messages (session_id, role, content, emotion) VALUES (?, ?, ?, ?)",
748
- (session_id, "assistant", greeting_text, "Neutral")
749
- )
750
- conn.commit()
751
- conn.close()
752
-
753
- print(f"[Greeting] ✅ Sent to {username}")
754
-
755
- except Exception as greeting_err:
756
- print(f"[Greeting] ❌ Error: {greeting_err}")
757
- import traceback
758
- traceback.print_exc()
759
-
760
- try:
761
- await websocket.send_json({
762
- "type": "error",
763
- "message": "Greeting failed - avatar service unavailable"
764
- })
765
- except:
766
- pass
767
-
768
- # ============ VIDEO FRAME - UPDATED WITH PROBABILITIES ============
769
- elif msg_type == "video_frame":
770
- try:
771
- # Decode base64 image
772
- img_data = base64.b64decode(data["frame"].split(",")[1])
773
- img = Image.open(io.BytesIO(img_data))
774
- frame = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
775
-
776
- # Process face emotion
777
- try:
778
- processed_frame, result = face_processor.process_frame(frame)
779
- face_emotion = face_processor.get_last_emotion() or "Neutral"
780
- face_confidence = face_processor.get_last_confidence() or 0.0
781
- face_probs = face_processor.get_last_probs()
782
- face_quality = face_processor.get_last_quality() if hasattr(face_processor, 'get_last_quality') else 0.5
783
- except Exception as proc_err:
784
- print(f"[FaceProcessor] Error: {proc_err}")
785
- face_emotion = "Neutral"
786
- face_confidence = 0.0
787
- face_probs = np.array([0.25, 0.25, 0.25, 0.25])
788
- face_quality = 0.0
789
-
790
- # Send face emotion to frontend with probabilities
791
- await websocket.send_json({
792
- "type": "face_emotion",
793
- "emotion": face_emotion,
794
- "confidence": face_confidence,
795
- "probabilities": face_probs.tolist(),
796
- "quality": face_quality
797
- })
798
-
799
- except Exception as e:
800
- print(f"[Video] Error: {e}")
801
-
802
- # ============ AUDIO CHUNK ============
803
- elif msg_type == "audio_chunk":
804
- try:
805
- audio_data = base64.b64decode(data["audio"])
806
- audio_buffer.append(audio_data)
807
-
808
- if len(audio_buffer) >= 5:
809
- voice_probs, voice_emotion = voice_worker.get_probs()
810
- await websocket.send_json({
811
- "type": "voice_emotion",
812
- "emotion": voice_emotion
813
- })
814
- audio_buffer = audio_buffer[-3:]
815
-
816
- except Exception as e:
817
- print(f"[Audio] Error: {e}")
818
-
819
- # ============ USER FINISHED SPEAKING (ENHANCED LOGGING) ============
820
- elif msg_type == "speech_end":
821
- transcription = data.get("text", "").strip()
822
-
823
- print(f"\n{'='*80}")
824
- print(f"[Speech End] 🎤 USER FINISHED SPEAKING: {username}")
825
- print(f"{'='*80}")
826
- print(f"[Transcription] '{transcription}'")
827
-
828
- # Filter short/meaningless
829
- if len(transcription) < 2:
830
- print(f"[Filter] ⏭️ Skipped: Too short ({len(transcription)} chars)")
831
- continue
832
-
833
- hallucinations = {"thank you", "thanks", "okay", "ok", "you", "yeah", "yep"}
834
- if transcription.lower().strip('.,!?') in hallucinations:
835
- print(f"[Filter] ⏭️ Skipped: Hallucination detected ('{transcription}')")
836
- continue
837
-
838
- # Save user message
839
- conn = sqlite3.connect(DB_PATH)
840
- cursor = conn.cursor()
841
- cursor.execute(
842
- "INSERT INTO messages (session_id, role, content) VALUES (?, ?, ?)",
843
- (session_id, "user", transcription)
844
- )
845
- conn.commit()
846
- conn.close()
847
-
848
- try:
849
- # ========== EMOTION DETECTION PIPELINE (ENHANCED LOGGING) ==========
850
- print(f"\n[Pipeline] 🔍 Starting emotion analysis pipeline...")
851
- print(f"{'─'*80}")
852
-
853
- # Step 1: Get face emotion
854
- print(f"[Step 1/4] 📸 FACIAL EXPRESSION ANALYSIS")
855
- face_emotion = face_processor.get_last_emotion()
856
- face_confidence = face_processor.get_last_confidence()
857
- face_quality = face_processor.get_last_quality() if hasattr(face_processor, 'get_last_quality') else 0.5
858
-
859
- # Create emotion probabilities
860
- emotion_map = {'Neutral': 0, 'Happy': 1, 'Sad': 2, 'Angry': 3}
861
- face_probs = np.array([0.25, 0.25, 0.25, 0.25], dtype=np.float32)
862
- if face_emotion in emotion_map:
863
- face_idx = emotion_map[face_emotion]
864
- face_probs[face_idx] = face_confidence
865
- face_probs = face_probs / face_probs.sum()
866
-
867
- print(f" Result: {face_emotion}")
868
- print(f" Confidence: {face_confidence:.3f}")
869
- print(f" Quality Score: {face_quality:.3f}")
870
- print(f" Distribution: Neutral={face_probs[0]:.3f} | Happy={face_probs[1]:.3f} | Sad={face_probs[2]:.3f} | Angry={face_probs[3]:.3f}")
871
-
872
- # Step 2: Get voice emotion
873
- print(f"\n[Step 2/4] 🎤 VOICE TONE ANALYSIS")
874
- voice_probs, voice_emotion = voice_worker.get_probs()
875
- voice_state = voice_worker.get_state()
876
- voice_active = voice_state.get('speech_active', False)
877
- voice_inferences = voice_state.get('inference_count', 0)
878
- voice_skipped = voice_state.get('skipped_inferences', 0)
879
-
880
- print(f" {'✅ ACTIVELY PROCESSING' if voice_active else '⚠️ IDLE (no recent speech)'}")
881
- print(f" Result: {voice_emotion}")
882
- print(f" Distribution: Neutral={voice_probs[0]:.3f} | Happy={voice_probs[1]:.3f} | Sad={voice_probs[2]:.3f} | Angry={voice_probs[3]:.3f}")
883
- print(f" Inferences completed: {voice_inferences}")
884
- print(f" Skipped (silence optimization): {voice_skipped}")
885
- efficiency = (voice_inferences / (voice_inferences + voice_skipped) * 100) if (voice_inferences + voice_skipped) > 0 else 0
886
- print(f" Processing efficiency: {efficiency:.1f}%")
887
-
888
- # Step 3: Analyze text sentiment
889
- print(f"\n[Step 3/4] 💬 TEXT SENTIMENT ANALYSIS")
890
- print(f" ✅ Using Whisper transcription")
891
- text_analyzer.analyze(transcription)
892
- text_probs, _ = text_analyzer.get_probs()
893
- text_emotion = ['Neutral', 'Happy', 'Sad', 'Angry'][int(np.argmax(text_probs))]
894
-
895
- print(f" Result: {text_emotion}")
896
- print(f" Distribution: Neutral={text_probs[0]:.3f} | Happy={text_probs[1]:.3f} | Sad={text_probs[2]:.3f} | Angry={text_probs[3]:.3f}")
897
- print(f" Text length: {len(transcription)} characters")
898
-
899
- # Step 4: Calculate fusion weights
900
- print(f"\n[Step 4/4] ⚖️ MULTI-MODAL FUSION")
901
- base_weights = {
902
- 'face': fusion_engine.alpha_face,
903
- 'voice': fusion_engine.alpha_voice,
904
- 'text': fusion_engine.alpha_text
905
- }
906
-
907
- # Adjust weights based on quality/confidence
908
- adjusted_weights = base_weights.copy()
909
- adjustments_made = []
910
-
911
- # Reduce face weight if quality is poor
912
- if face_quality < 0.5:
913
- adjusted_weights['face'] *= 0.7
914
- adjustments_made.append(f"Face weight reduced (low quality: {face_quality:.3f})")
915
-
916
- # Reduce voice weight if not active
917
- if not voice_active:
918
- adjusted_weights['voice'] *= 0.5
919
- adjustments_made.append(f"Voice weight reduced (no recent speech)")
920
-
921
- # Reduce text weight if very short
922
- if len(transcription) < 10:
923
- adjusted_weights['text'] *= 0.7
924
- adjustments_made.append(f"Text weight reduced (short input: {len(transcription)} chars)")
925
-
926
- # Normalize weights to sum to 1.0
927
- total_weight = sum(adjusted_weights.values())
928
- final_weights = {k: v/total_weight for k, v in adjusted_weights.items()}
929
-
930
- print(f" Base weights: Face={base_weights['face']:.3f} | Voice={base_weights['voice']:.3f} | Text={base_weights['text']:.3f}")
931
- if adjustments_made:
932
- print(f" Adjustments:")
933
- for adj in adjustments_made:
934
- print(f" - {adj}")
935
- print(f" Final weights: Face={final_weights['face']:.3f} | Voice={final_weights['voice']:.3f} | Text={final_weights['text']:.3f}")
936
-
937
- # Calculate weighted fusion
938
- fused_probs = (
939
- final_weights['face'] * face_probs +
940
- final_weights['voice'] * voice_probs +
941
- final_weights['text'] * text_probs
942
- )
943
- fused_probs = fused_probs / (np.sum(fused_probs) + 1e-8)
944
-
945
- fused_emotion, intensity = fusion_engine.fuse(face_probs, voice_probs, text_probs)
946
-
947
- # Calculate fusion accuracy metrics
948
- agreement_count = sum([
949
- face_emotion == fused_emotion,
950
- voice_emotion == fused_emotion,
951
- text_emotion == fused_emotion
952
- ])
953
- agreement_score = agreement_count / 3.0
954
-
955
- # Check for conflicts
956
- all_same = (face_emotion == voice_emotion == text_emotion)
957
- has_conflict = len({face_emotion, voice_emotion, text_emotion}) == 3
958
-
959
- print(f"\n {'─'*76}")
960
- print(f" FUSION RESULTS:")
961
- print(f" {'─'*76}")
962
- print(f" Input emotions:")
963
- print(f" Face: {face_emotion:7s} (confidence={face_probs[emotion_map.get(face_emotion, 0)]:.3f}, weight={final_weights['face']:.3f})")
964
- print(f" Voice: {voice_emotion:7s} (confidence={voice_probs[emotion_map.get(voice_emotion, 0)]:.3f}, weight={final_weights['voice']:.3f})")
965
- print(f" Text: {text_emotion:7s} (confidence={text_probs[emotion_map.get(text_emotion, 0)]:.3f}, weight={final_weights['text']:.3f})")
966
- print(f" {'─'*76}")
967
- print(f" FUSED EMOTION: {fused_emotion}")
968
- print(f" Intensity: {intensity:.3f}")
969
- print(f" Fused distribution: Neutral={fused_probs[0]:.3f} | Happy={fused_probs[1]:.3f} | Sad={fused_probs[2]:.3f} | Angry={fused_probs[3]:.3f}")
970
- print(f" {'─'*76}")
971
- print(f" Agreement: {agreement_count}/3 modalities ({agreement_score*100:.1f}%)")
972
-
973
- if all_same:
974
- print(f" Status: ✅ Perfect agreement - all modalities aligned")
975
- elif has_conflict:
976
- print(f" Status: ⚠️ Full conflict - weighted fusion resolved disagreement")
977
- else:
978
- print(f" Status: 📊 Partial agreement - majority vote with confidence weighting")
979
-
980
- print(f" {'─'*76}")
981
-
982
- # ========== LLM INPUT PREPARATION ==========
983
- print(f"\n[LLM Input] 🧠 Preparing context for language model...")
984
-
985
- # Language instruction
986
- user_language = user_preferences.get("language", "en")
987
-
988
- context_prefix = ""
989
- if user_summary:
990
- context_prefix = f"[User context for {username}: {user_summary}]\n\n"
991
- print(f"[LLM Input] - User context: YES ({len(user_summary)} chars)")
992
- else:
993
- print(f"[LLM Input] - User context: NO (new user)")
994
-
995
- # Add language instruction
996
- if user_language == "nl":
997
- context_prefix += "[BELANGRIJK: Antwoord ALTIJD in het Nederlands!]\n\n"
998
- print(f"[LLM Input] - Language: Dutch (Nederlands)")
999
- else:
1000
- context_prefix += "[IMPORTANT: ALWAYS respond in English!]\n\n"
1001
- print(f"[LLM Input] - Language: English")
1002
-
1003
- full_llm_input = context_prefix + transcription
1004
-
1005
- print(f"[LLM Input] - Fused emotion: {fused_emotion}")
1006
- print(f"[LLM Input] - Face emotion: {face_emotion}")
1007
- print(f"[LLM Input] - Voice emotion: {voice_emotion}")
1008
- print(f"[LLM Input] - Intensity: {intensity:.3f}")
1009
- print(f"[LLM Input] - User text: '{transcription}'")
1010
- print(f"[LLM Input] - Full prompt length: {len(full_llm_input)} chars")
1011
-
1012
- if len(context_prefix) > 50:
1013
- print(f"[LLM Input] - Context preview: '{context_prefix[:100]}...'")
1014
-
1015
- # Generate LLM response
1016
- print(f"\n[LLM] 🤖 Generating response...")
1017
- response_text = llm_generator.generate_response(
1018
- fused_emotion, face_emotion, voice_emotion,
1019
- full_llm_input, force=True, intensity=intensity
1020
- )
1021
-
1022
- print(f"[LLM] ✅ Response generated: '{response_text}'")
1023
-
1024
- # Save assistant message
1025
- conn = sqlite3.connect(DB_PATH)
1026
- cursor = conn.cursor()
1027
- cursor.execute(
1028
- "INSERT INTO messages (session_id, role, content, emotion) VALUES (?, ?, ?, ?)",
1029
- (session_id, "assistant", response_text, fused_emotion)
1030
- )
1031
- conn.commit()
1032
- conn.close()
1033
-
1034
- # ========== SEND TO AVATAR FOR TTS ==========
1035
- print(f"\n[TTS] 🎭 Sending to avatar backend...")
1036
-
1037
- try:
1038
- voice_preference = user_preferences.get("voice", "female")
1039
- language_preference = user_preferences.get("language", "en")
1040
-
1041
- print(f"[TTS] - Voice: {voice_preference}")
1042
- print(f"[TTS] - Language: {language_preference}")
1043
- print(f"[TTS] - Text: '{response_text}'")
1044
-
1045
- avatar_response = requests.post(
1046
- f"{AVATAR_API}/speak",
1047
- data={
1048
- "text": response_text,
1049
- "voice": voice_preference,
1050
- "language": language_preference
1051
- },
1052
- timeout=45
1053
- )
1054
- avatar_response.raise_for_status()
1055
- avatar_data = avatar_response.json()
1056
-
1057
- print(f"[TTS] ✅ Avatar TTS generated")
1058
- print(f"[TTS] - Audio URL: {avatar_data.get('audio_url', 'N/A')}")
1059
- print(f"[TTS] - Visemes: {len(avatar_data.get('visemes', []))} keyframes")
1060
-
1061
- await websocket.send_json({
1062
- "type": "llm_response",
1063
- "text": response_text,
1064
- "emotion": fused_emotion,
1065
- "intensity": intensity,
1066
- "audio_url": avatar_data.get("audio_url"),
1067
- "visemes": avatar_data.get("visemes")
1068
- })
1069
-
1070
- print(f"[Pipeline] ✅ Complete response sent to {username}")
1071
-
1072
- except requests.exceptions.ConnectionError:
1073
- print(f"[TTS] ⚠️ Avatar service not available - sending text-only")
1074
- await websocket.send_json({
1075
- "type": "llm_response",
1076
- "text": response_text,
1077
- "emotion": fused_emotion,
1078
- "intensity": intensity,
1079
- "text_only": True
1080
- })
1081
-
1082
- except Exception as avatar_err:
1083
- print(f"[TTS] ❌ Avatar error: {avatar_err}")
1084
- await websocket.send_json({
1085
- "type": "llm_response",
1086
- "text": response_text,
1087
- "emotion": fused_emotion,
1088
- "intensity": intensity,
1089
- "error": "Avatar TTS failed",
1090
- "text_only": True
1091
- })
1092
-
1093
- print(f"{'='*80}\n")
1094
-
1095
- except Exception as e:
1096
- print(f"[Pipeline] ❌ Error in emotion processing: {e}")
1097
- import traceback
1098
- traceback.print_exc()
1099
-
1100
- except WebSocketDisconnect:
1101
- print(f"[WebSocket] ❌ {username} disconnected (close/refresh)")
1102
-
1103
- except Exception as e:
1104
- print(f"[WebSocket] ❌ {username} error: {e}")
1105
- import traceback
1106
- traceback.print_exc()
1107
-
1108
- finally:
1109
- # Generate summary on disconnect
1110
- if session_data and session_id and user_id:
1111
- print(f"[WebSocket] 📝 Generating summary for {username} (session ended)...")
1112
- try:
1113
- summary = await generate_session_summary(session_id, user_id)
1114
- if summary:
1115
- print(f"[Summary] ✅ Saved for {username}")
1116
- else:
1117
- print(f"[Summary] ⏭️ Skipped (not enough messages)")
1118
- except Exception as summary_err:
1119
- print(f"[Summary] ❌ Error for {username}: {summary_err}")
1120
-
1121
- if __name__ == "__main__":
1122
- import uvicorn
1123
- uvicorn.run(app, host="0.0.0.0", port=8000)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mrrrme/database/db_manager.py DELETED
@@ -1,333 +0,0 @@
1
- """Database manager for user sessions, chat history, and summaries"""
2
- import sqlite3
3
- import json
4
- import hashlib
5
- import secrets
6
- from datetime import datetime, timedelta
7
- from pathlib import Path
8
- from typing import Optional, List, Dict
9
-
10
- DB_PATH = Path("/tmp/mrrrme_users.db")
11
-
12
-
13
- class DatabaseManager:
14
- """Manages user authentication, chat history, and AI-generated summaries"""
15
-
16
- def __init__(self, db_path: str = str(DB_PATH)):
17
- self.db_path = db_path
18
- self._init_database()
19
- print(f"[Database] ✅ Initialized at {db_path}")
20
-
21
- def _init_database(self):
22
- """Create tables if they don't exist"""
23
- conn = sqlite3.connect(self.db_path)
24
- cursor = conn.cursor()
25
-
26
- # Users table
27
- cursor.execute("""
28
- CREATE TABLE IF NOT EXISTS users (
29
- user_id TEXT PRIMARY KEY,
30
- username TEXT UNIQUE NOT NULL,
31
- password_hash TEXT NOT NULL,
32
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
33
- last_login TIMESTAMP,
34
- total_sessions INTEGER DEFAULT 0
35
- )
36
- """)
37
-
38
- # Sessions table
39
- cursor.execute("""
40
- CREATE TABLE IF NOT EXISTS sessions (
41
- session_id TEXT PRIMARY KEY,
42
- user_id TEXT NOT NULL,
43
- token TEXT UNIQUE NOT NULL,
44
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
45
- last_activity TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
46
- is_active BOOLEAN DEFAULT 1,
47
- FOREIGN KEY (user_id) REFERENCES users(user_id)
48
- )
49
- """)
50
-
51
- # Messages table
52
- cursor.execute("""
53
- CREATE TABLE IF NOT EXISTS messages (
54
- message_id INTEGER PRIMARY KEY AUTOINCREMENT,
55
- session_id TEXT NOT NULL,
56
- user_id TEXT NOT NULL,
57
- role TEXT NOT NULL,
58
- content TEXT NOT NULL,
59
- emotion TEXT,
60
- intensity REAL,
61
- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
62
- FOREIGN KEY (session_id) REFERENCES sessions(session_id),
63
- FOREIGN KEY (user_id) REFERENCES users(user_id)
64
- )
65
- """)
66
-
67
- # Summaries table (AI-generated user profiles)
68
- cursor.execute("""
69
- CREATE TABLE IF NOT EXISTS user_summaries (
70
- summary_id INTEGER PRIMARY KEY AUTOINCREMENT,
71
- user_id TEXT NOT NULL,
72
- session_id TEXT,
73
- summary_text TEXT NOT NULL,
74
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
75
- message_count INTEGER,
76
- FOREIGN KEY (user_id) REFERENCES users(user_id),
77
- FOREIGN KEY (session_id) REFERENCES sessions(session_id)
78
- )
79
- """)
80
-
81
- conn.commit()
82
- conn.close()
83
-
84
- def _hash_password(self, password: str) -> str:
85
- """Hash password with salt"""
86
- return hashlib.sha256(password.encode()).hexdigest()
87
-
88
- def create_user(self, username: str, password: str) -> Optional[str]:
89
- """Create new user, returns user_id"""
90
- try:
91
- conn = sqlite3.connect(self.db_path)
92
- cursor = conn.cursor()
93
-
94
- user_id = secrets.token_urlsafe(16)
95
- password_hash = self._hash_password(password)
96
-
97
- cursor.execute(
98
- "INSERT INTO users (user_id, username, password_hash) VALUES (?, ?, ?)",
99
- (user_id, username, password_hash)
100
- )
101
-
102
- conn.commit()
103
- conn.close()
104
-
105
- print(f"[Database] ✅ Created user: {username}")
106
- return user_id
107
- except sqlite3.IntegrityError:
108
- return None
109
- except Exception as e:
110
- print(f"[Database] ❌ Error creating user: {e}")
111
- return None
112
-
113
- def authenticate_user(self, username: str, password: str) -> Optional[Dict]:
114
- """Authenticate user and return user info"""
115
- conn = sqlite3.connect(self.db_path)
116
- cursor = conn.cursor()
117
-
118
- password_hash = self._hash_password(password)
119
-
120
- cursor.execute(
121
- "SELECT user_id, username FROM users WHERE username = ? AND password_hash = ?",
122
- (username, password_hash)
123
- )
124
-
125
- result = cursor.fetchone()
126
-
127
- if result:
128
- user_id, username = result
129
-
130
- # Update last login
131
- cursor.execute(
132
- "UPDATE users SET last_login = ?, total_sessions = total_sessions + 1 WHERE user_id = ?",
133
- (datetime.now(), user_id)
134
- )
135
- conn.commit()
136
-
137
- conn.close()
138
- return {'user_id': user_id, 'username': username}
139
-
140
- conn.close()
141
- return None
142
-
143
- def create_session(self, user_id: str) -> Optional[Dict]:
144
- """Create new session for user"""
145
- try:
146
- conn = sqlite3.connect(self.db_path)
147
- cursor = conn.cursor()
148
-
149
- session_id = secrets.token_urlsafe(16)
150
- token = secrets.token_urlsafe(32)
151
-
152
- cursor.execute(
153
- "INSERT INTO sessions (session_id, user_id, token) VALUES (?, ?, ?)",
154
- (session_id, user_id, token)
155
- )
156
-
157
- conn.commit()
158
- conn.close()
159
-
160
- print(f"[Database] ✅ Created session for user {user_id}")
161
- return {
162
- 'session_id': session_id,
163
- 'token': token,
164
- 'user_id': user_id
165
- }
166
- except Exception as e:
167
- print(f"[Database] ❌ Error creating session: {e}")
168
- return None
169
-
170
- def validate_token(self, token: str) -> Optional[Dict]:
171
- """Validate session token and return session info"""
172
- conn = sqlite3.connect(self.db_path)
173
- cursor = conn.cursor()
174
-
175
- cursor.execute(
176
- """
177
- SELECT s.session_id, s.user_id, u.username
178
- FROM sessions s
179
- JOIN users u ON s.user_id = u.user_id
180
- WHERE s.token = ? AND s.is_active = 1
181
- """,
182
- (token,)
183
- )
184
-
185
- result = cursor.fetchone()
186
-
187
- if result:
188
- session_id, user_id, username = result
189
-
190
- # Update last activity
191
- cursor.execute(
192
- "UPDATE sessions SET last_activity = ? WHERE session_id = ?",
193
- (datetime.now(), session_id)
194
- )
195
- conn.commit()
196
-
197
- conn.close()
198
- return {
199
- 'session_id': session_id,
200
- 'user_id': user_id,
201
- 'username': username
202
- }
203
-
204
- conn.close()
205
- return None
206
-
207
- def add_message(self, session_id: str, user_id: str, role: str, content: str,
208
- emotion: Optional[str] = None, intensity: Optional[float] = None):
209
- """Add message to chat history"""
210
- try:
211
- conn = sqlite3.connect(self.db_path)
212
- cursor = conn.cursor()
213
-
214
- cursor.execute(
215
- """
216
- INSERT INTO messages (session_id, user_id, role, content, emotion, intensity)
217
- VALUES (?, ?, ?, ?, ?, ?)
218
- """,
219
- (session_id, user_id, role, content, emotion, intensity)
220
- )
221
-
222
- conn.commit()
223
- conn.close()
224
- except Exception as e:
225
- print(f"[Database] ❌ Error adding message: {e}")
226
-
227
- def get_session_messages(self, session_id: str) -> List[Dict]:
228
- """Get all messages for a session"""
229
- conn = sqlite3.connect(self.db_path)
230
- cursor = conn.cursor()
231
-
232
- cursor.execute(
233
- """
234
- SELECT role, content, emotion, intensity, timestamp
235
- FROM messages
236
- WHERE session_id = ?
237
- ORDER BY timestamp ASC
238
- """,
239
- (session_id,)
240
- )
241
-
242
- messages = []
243
- for row in cursor.fetchall():
244
- messages.append({
245
- 'role': row[0],
246
- 'content': row[1],
247
- 'emotion': row[2],
248
- 'intensity': row[3],
249
- 'timestamp': row[4]
250
- })
251
-
252
- conn.close()
253
- return messages
254
-
255
- def get_user_summary(self, user_id: str) -> Optional[str]:
256
- """Get most recent summary for user"""
257
- conn = sqlite3.connect(self.db_path)
258
- cursor = conn.cursor()
259
-
260
- cursor.execute(
261
- """
262
- SELECT summary_text
263
- FROM user_summaries
264
- WHERE user_id = ?
265
- ORDER BY created_at DESC
266
- LIMIT 1
267
- """,
268
- (user_id,)
269
- )
270
-
271
- result = cursor.fetchone()
272
- conn.close()
273
-
274
- return result[0] if result else None
275
-
276
- def add_summary(self, user_id: str, session_id: str, summary_text: str, message_count: int):
277
- """Add AI-generated summary"""
278
- try:
279
- conn = sqlite3.connect(self.db_path)
280
- cursor = conn.cursor()
281
-
282
- cursor.execute(
283
- """
284
- INSERT INTO user_summaries (user_id, session_id, summary_text, message_count)
285
- VALUES (?, ?, ?, ?)
286
- """,
287
- (user_id, session_id, summary_text, message_count)
288
- )
289
-
290
- conn.commit()
291
- conn.close()
292
-
293
- print(f"[Database] ✅ Saved summary for user {user_id} ({message_count} messages)")
294
- except Exception as e:
295
- print(f"[Database] ❌ Error saving summary: {e}")
296
-
297
- def close_session(self, session_id: str):
298
- """Mark session as inactive"""
299
- try:
300
- conn = sqlite3.connect(self.db_path)
301
- cursor = conn.cursor()
302
-
303
- cursor.execute(
304
- "UPDATE sessions SET is_active = 0 WHERE session_id = ?",
305
- (session_id,)
306
- )
307
-
308
- conn.commit()
309
- conn.close()
310
- except Exception as e:
311
- print(f"[Database] ❌ Error closing session: {e}")
312
-
313
- def cleanup_old_sessions(self, days: int = 30):
314
- """Clean up sessions older than X days"""
315
- try:
316
- conn = sqlite3.connect(self.db_path)
317
- cursor = conn.cursor()
318
-
319
- cutoff = datetime.now() - timedelta(days=days)
320
-
321
- cursor.execute(
322
- "DELETE FROM sessions WHERE last_activity < ? AND is_active = 0",
323
- (cutoff,)
324
- )
325
-
326
- deleted = cursor.rowcount
327
- conn.commit()
328
- conn.close()
329
-
330
- if deleted > 0:
331
- print(f"[Database] 🗑️ Cleaned up {deleted} old sessions")
332
- except Exception as e:
333
- print(f"[Database] ❌ Error cleaning sessions: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mrrrme/utils/weight_finder.py DELETED
@@ -1,38 +0,0 @@
1
- """Utility for finding OpenFace weight files"""
2
- import os
3
- from pathlib import Path
4
- from glob import glob
5
-
6
-
7
- def find_weight(filename: str) -> str:
8
- """Find weight file in various possible locations"""
9
- # Check environment variable
10
- env_dir = os.environ.get("OPENFACE_WEIGHT_DIR")
11
- if env_dir:
12
- p = Path(env_dir) / filename
13
- if p.is_file():
14
- return str(p)
15
-
16
- # Check package installation
17
- try:
18
- import openface as _of
19
- site_w = Path(_of.__path__[0]) / "weights" / filename
20
- if site_w.is_file():
21
- return str(site_w)
22
- except:
23
- pass
24
-
25
- # Check HuggingFace cache
26
- user_home = Path(os.environ.get("USERPROFILE", str(Path.home())))
27
- hf_root = user_home / ".cache" / "huggingface" / "hub"
28
- if hf_root.exists():
29
- hits = glob(str(hf_root / "**" / filename), recursive=True)
30
- if hits:
31
- return hits[0]
32
-
33
- # Check local weights directory
34
- local = Path("weights") / filename
35
- if local.is_file():
36
- return str(local)
37
-
38
- raise FileNotFoundError(f"Weight not found: {filename}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
sync.bat DELETED
@@ -1,73 +0,0 @@
1
- @echo off
2
- echo ============================================
3
- echo Nuclear Option: Clean Git History
4
- echo ============================================
5
- echo.
6
- echo WARNING: This will rewrite Git history!
7
- echo Make sure you have no other important uncommitted changes.
8
- echo.
9
- set /p CONFIRM="Type YES to continue: "
10
-
11
- if /i NOT "%CONFIRM%"=="YES" (
12
- echo Cancelled.
13
- pause
14
- exit /b
15
- )
16
-
17
- REM Step 1: Remove file from entire Git history
18
- echo.
19
- echo Step 1: Removing idle-animation.glb from entire Git history...
20
- git filter-branch --force --index-filter "git rm --cached --ignore-unmatch avatar-frontend/public/idle-animation.glb" --prune-empty --tag-name-filter cat -- --all
21
-
22
- REM Step 2: Clean up
23
- echo.
24
- echo Step 2: Cleaning up Git...
25
- git reflog expire --expire=now --all
26
- git gc --prune=now --aggressive
27
-
28
- REM Step 3: Force push to clear history on both repos
29
- echo.
30
- echo Step 3: Force pushing clean history to GitHub...
31
- git push origin main --force
32
-
33
- echo.
34
- echo Step 4: Force pushing clean history to Hugging Face...
35
- git push https://huggingface.co/spaces/michon/mrrrme-emotion-ai main --force
36
-
37
- REM Step 5: Setup LFS
38
- echo.
39
- echo Step 5: Setting up Git LFS...
40
- git lfs install
41
- git lfs track "avatar-frontend/public/idle-animation.glb"
42
- git add .gitattributes
43
- git commit -m "Setup LFS tracking for animation"
44
-
45
- REM Step 6: Push LFS setup
46
- echo.
47
- echo Step 6: Pushing LFS setup to GitHub...
48
- git push origin main
49
-
50
- echo.
51
- echo Step 7: Pushing LFS setup to Hugging Face...
52
- git push https://huggingface.co/spaces/michon/mrrrme-emotion-ai main
53
-
54
- REM Step 7: Add the actual file
55
- echo.
56
- echo Step 8: Adding idle-animation.glb with LFS...
57
- git add avatar-frontend/public/idle-animation.glb
58
- git commit -m "Add idle animation via LFS (195KB)"
59
-
60
- REM Step 8: Push file to both repos
61
- echo.
62
- echo Step 9: Pushing to GitHub...
63
- git push origin main
64
-
65
- echo.
66
- echo Step 10: Pushing to Hugging Face...
67
- git push https://huggingface.co/spaces/michon/mrrrme-emotion-ai main
68
-
69
- echo.
70
- echo ============================================
71
- echo Done! File should be accepted now.
72
- echo ============================================
73
- pause
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
weights/ir50.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:62fcfa833776648f818b15fac4f5b760d76847316097e8e046f77ac445defb75
3
- size 122022895
 
 
 
 
weights/mobilefacenet_model_best.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b994af026bfddbafc507a6f1c8737a9896bab20ed2b0cfb6ae90b81736970313
3
- size 12281146
 
 
 
 
weights/raf-db-model_best.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d9bf1d0d88238966ce0d1a289a2bb5f927ec2fe635ef1ec4396c323028924701
3
- size 238971279