Update predictor.py

predictor.py

@@ -1,5 +1,7 @@
 import json
 import re
+import os
+import hashlib
 import onnxruntime as ort
 import numpy as np
 from typing import List, Dict, Set, Optional
@@ -22,6 +24,17 @@ class SentenceExtractor:
         word_score_plus_threshold: int = 1,
         word_score_minus_threshold: int = -1,
     ):
+        # 统一以文件所在目录为根，避免工作目录不同导致找不到资源
+        self.base_dir = os.path.dirname(os.path.abspath(__file__))
+        self.tokenizer_dir = self.base_dir
+
+        # 允许传相对路径：自动转绝对
+        if not os.path.isabs(model_path):
+            model_path = os.path.join(self.base_dir, model_path)
+
+        if not os.path.isabs(eval_keywords_path):
+            eval_keywords_path = os.path.join(self.base_dir, eval_keywords_path)
+
         self.eval_keywords = self._load_eval_keywords(eval_keywords_path)
         self.all_keywords = self._extract_all_keywords()
 
@@ -36,22 +49,51 @@ class SentenceExtractor:
             self.aggregation_mode = "max"
         self.word_score_plus_threshold = int(word_score_plus_threshold)
         self.word_score_minus_threshold = int(word_score_minus_threshold)
+        self.providers: Optional[List[str]] = None
+        self.tokenizer_loaded: bool = False
+        self.last_tokenizer_error: Optional[str] = None
         try:
-            self.ort_session = ort.InferenceSession(model_path)
+            # 强制使用 CPU provider，避免某些环境下选择到不可用的 GPU provider 导致加载失败
+            self.ort_session = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
             self.input_name = self.ort_session.get_inputs()[0].name
             self.output_name = self.ort_session.get_outputs()[0].name
+            try:
+                self.providers = self.ort_session.get_providers()
+            except Exception:
+                self.providers = None
             print("ONNX 模型加载成功")
+            self.model_loaded: bool = True
         except Exception as e:
             print(f"ONNX 模型加载失败: {e}")
             self.ort_session = None
+            self.model_loaded = False
+
+        # 记录模型文件信息，便于排查“用错模型”问题
+        try:
+            self.model_path_abs: Optional[str] = os.path.abspath(model_path)
+            self.model_sha256: Optional[str] = None
+            if os.path.exists(model_path):
+                sha = hashlib.sha256()
+                with open(model_path, 'rb') as f:
+                    for chunk in iter(lambda: f.read(8192), b''):
+                        sha.update(chunk)
+                self.model_sha256 = sha.hexdigest()
+        except Exception:
+            self.model_path_abs = None
+            self.model_sha256 = None
 
     def _preprocess_text(self, text: str) -> np.ndarray:
         try:
             from transformers import AutoTokenizer
+            # 1) 优先从与脚本同目录加载本地 tokenizer（部署一起带上 tokenizer.json 等文件）
             try:
-                tokenizer = AutoTokenizer.from_pretrained(".", local_files_only=True)
+                tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_dir, local_files_only=True)
             except Exception:
-                tokenizer = AutoTokenizer.from_pretrained("uer/chinese_roberta_L-4_H-256")
+                try:
+                    tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_dir)
+                except Exception:
+                    # 2) 兜底：在线模型（需要外网）
+                    tokenizer = AutoTokenizer.from_pretrained("uer/chinese_roberta_L-4_H-256")
             inputs = tokenizer(
                 text,
                 truncation=True,
@@ -59,24 +101,25 @@ class SentenceExtractor:
                 max_length=512,
                 return_tensors='np'
             )
+            self.tokenizer_loaded = True
+            self.last_tokenizer_error = None
             return inputs
         except Exception as e:
-            print(f"Tokenizer预处理失败: {e}")
-            max_seq_length = 128
-            features = np.zeros((1, max_seq_length), dtype=np.float32)
-            for i, ch in enumerate(text[:max_seq_length]):
-                features[0, i] = (ord(ch) % 256) / 255.0
-            return features
-
-    def _predict_grade_with_model(self, text: str) -> str:
+            self.tokenizer_loaded = False
+            self.last_tokenizer_error = str(e)
+            # 继续抛出异常，由上层捕获并回退，同时记录原因
+            raise
+
+    def _predict_grade_with_model(self, text: str) -> Dict[str, any]:
         try:
             if not self.ort_session:
                 word_score = self._calculate_word_scores(text)["total_score"]
+                grade = "C"
                 if word_score > 1:
-                    return "B"
+                    grade = "B"
                 if word_score < -1:
-                    return "D"
-                return "C"
+                    grade = "D"
+                return {"grade": grade, "source": "rule", "word_score_total": word_score}
 
             inputs = self._preprocess_text(text)
 
@@ -106,15 +149,36 @@ class SentenceExtractor:
             predictions = outputs[0]
             grade_index = int(np.argmax(predictions))
             grades = ['A', 'B', 'C', 'D', 'E']
-            return grades[grade_index]
+            probs = self._softmax(predictions)[0].tolist()
+            return {
+                "grade": grades[grade_index],
+                "source": "model",
+                "prob": float(probs[grade_index]),
+                "probs": probs,
+                "logits": predictions[0].tolist(),
+            }
         except Exception as e:
             print(f"模型预测出错: {e}")
             word_score = self._calculate_word_scores(text)["total_score"]
+            grade = "C"
             if word_score > 1:
-                return "B"
+                grade = "B"
             if word_score < -1:
-                return "D"
-            return "C"
+                grade = "D"
+            return {
+                "grade": grade,
+                "source": "rule",
+                "word_score_total": word_score,
+                "reason": str(e),
+                "tokenizer_loaded": self.tokenizer_loaded,
+                "last_tokenizer_error": self.last_tokenizer_error,
+            }
+
+    @staticmethod
+    def _softmax(x: np.ndarray) -> np.ndarray:
+        x = x - np.max(x, axis=-1, keepdims=True)
+        exp_x = np.exp(x)
+        return exp_x / np.sum(exp_x, axis=-1, keepdims=True)
 
     def _load_eval_keywords(self, file_path: str) -> Dict[str, Dict[str, List[str]]]:
         try:
@@ -315,9 +379,17 @@ class SentenceExtractor:
         scored_sentences = []
         total_sentence_score = 0
         for sentence in relevant_sentences:
-            grade = self._predict_grade_with_model(sentence)
+            info = self._predict_grade_with_model(sentence)
+            grade = info.get("grade", "C")
             score = score_map.get(grade, 3)
-            scored_sentences.append({"sentence": sentence, "grade": grade})
+            # 附带调试信息
+            scored_sentences.append({
+                "sentence": sentence,
+                "grade": grade,
+                "source": info.get("source", "unknown"),
+                "prob": info.get("prob"),
+                "word_score_total": info.get("word_score_total"),
+            })
             total_sentence_score += score
 
         comprehensive_grade = "C"
@@ -346,4 +418,22 @@ class SentenceExtractor:
             "neutral_word_count": word_scores["neutral_count"],
             "scored_sentences": scored_sentences,
             "count": len(relevant_sentences),
-        }
+            # 调试字段
+            "debug": {
+                "model_loaded": getattr(self, "model_loaded", False),
+                "model_path_abs": getattr(self, "model_path_abs", None),
+                "model_sha256": getattr(self, "model_sha256", None),
+                "providers": self.providers,
+                "tokenizer_loaded": self.tokenizer_loaded,
+                "last_tokenizer_error": self.last_tokenizer_error,
+                "aggregation_mode": self.aggregation_mode,
+                "min_sentence_char_len": self.min_sentence_char_len,
+                "merge_leading_punct": self.merge_leading_punct,
+                "word_score_plus_threshold": self.word_score_plus_threshold,
+                "word_score_minus_threshold": self.word_score_minus_threshold,
+                "relevant_sentences": relevant_sentences,
+                "word_score_total": word_scores["total_score"],
+            }
+        }
+
+