Replace table with risk-level filter and persist history view

README.md

@@ -26,29 +26,15 @@ short_description: 'three-switchable AIGC detectors for Chinese papers'
 - 默认全文检测（不再区分快速模式）
 - 三模型可切换（默认 `paperpass-v3`）
 - 推理进度百分比显示
-- 页眉页脚、页码、重复行基础清洗（PDF）
-- 段落详情显示完整段落（不再截断到 380 字）
+- PDF 页眉页脚、页码、重复行基础清洗
+- 风险筛选器：`全部 / 高风险 / 中风险 / 低风险`
+- 历史记录保存与查看
 - 可选实验模式：`mba-aigc-detector`（需本地模型包）
 
 ## 关于 mba-aigc-detector
 
 该模型不是单个 `transformers` 分类模型，而是 `RoBERTa 特征 + 多个树模型` 的融合方案。  
-因此需要额外的本地模型文件（`models/mba/*.pkl`）才能运行，Hugging Face Space 默认不会自带这些大文件。
-
-### 你选的方案1（仓库内置模型包）
-
-把以下文件提交到仓库目录 `models/mba/`：
-
-- `select5_tree_d2_model.pkl`
-- `select10_tree_d2_model.pkl`
-- `select15_tree_d3_model.pkl`
-- `select20_tree_d2_model.pkl`
-- `bert_tree_d1_model.pkl`
-
-备注：
-
-- 这些文件通常较大，推荐用 Hugging Face 网页端直接上传到 Space 仓库，避免本地 `git-lfs` 环境问题。
-- 上传完成后无需改代码，页面中直接切换到 `mba-aigc-detector（实验版，需本地模型包）` 即可。
+需要 `models/mba/` 下的真实模型文件；如果上传的是 Git LFS 指针文件，系统会自动提示。
 
 ## 校准
 

app.py

@@ -1,5 +1,7 @@
 import json
 import re
+import time
+from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Tuple
 
@@ -29,10 +31,12 @@ DEFAULT_MODEL_LABEL = "paperpass-v3（默认，论文场景优先）"
 
 MIN_PARAGRAPH_CHARS = 80
 WINDOW_MAX_LENGTH = 512
-WINDOW_STRIDE = 128
+WINDOW_STRIDE = 192
+MAX_HISTORY_ITEMS = 30
 
 CALIBRATION_PATH = Path("calibration/model.json")
 MBA_MODELS_DIR = Path("models/mba")
+HISTORY_PATH = Path("history/analysis_records.json")
 
 CURRENT_MODEL_NAME = None
 CURRENT_TOKENIZER = None
@@ -45,6 +49,12 @@ MBA_STATE = {
     "tree_models": {},
 }
 
+# Better CPU utilization for Torch inference.
+try:
+    torch.set_num_threads(max(1, (torch.get_num_threads() or 4)))
+except Exception:
+    pass
+
 
 def load_calibration_model() -> Dict:
     if not CALIBRATION_PATH.exists():
@@ -62,13 +72,52 @@ def load_calibration_model() -> Dict:
 CALIBRATION_MODEL = load_calibration_model()
 
 
+def ensure_history_file():
+    HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True)
+    if not HISTORY_PATH.exists():
+        HISTORY_PATH.write_text("[]", encoding="utf-8")
+
+
+def load_history() -> List[Dict]:
+    ensure_history_file()
+    try:
+        data = json.loads(HISTORY_PATH.read_text(encoding="utf-8"))
+        if isinstance(data, list):
+            return data
+    except Exception:
+        pass
+    return []
+
+
+def save_history_item(item: Dict):
+    items = load_history()
+    items.insert(0, item)
+    items = items[:MAX_HISTORY_ITEMS]
+    HISTORY_PATH.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def format_history_markdown() -> str:
+    items = load_history()
+    if not items:
+        return "暂无历史记录。"
+    lines = ["# 历史分析记录"]
+    for i, x in enumerate(items, 1):
+        lines.append(
+            f"{i}. `{x.get('time')}` | 文件: {x.get('source')} | 模型: {x.get('model')} | "
+            f"综合风险: {x.get('overall', 0):.2%} | 预测知网率: {x.get('kn_like', 0):.2%} | 段落数: {x.get('paragraphs', 0)}"
+        )
+    return "\n".join(lines)
+
+
 def get_or_load_model(model_name: str):
     global CURRENT_MODEL_NAME, CURRENT_TOKENIZER, CURRENT_MODEL
     if CURRENT_MODEL_NAME == model_name and CURRENT_TOKENIZER is not None and CURRENT_MODEL is not None:
         return CURRENT_TOKENIZER, CURRENT_MODEL
+
     tokenizer = AutoTokenizer.from_pretrained(model_name)
     model = AutoModelForSequenceClassification.from_pretrained(model_name)
     model.eval()
+
     CURRENT_MODEL_NAME = model_name
     CURRENT_TOKENIZER = tokenizer
     CURRENT_MODEL = model
@@ -94,17 +143,15 @@ def is_probable_page_number(line: str) -> bool:
 
 def clean_common_noise(line: str) -> str:
     line = normalize_text(line)
-    line = re.sub(r"[ 	]+", " ", line)
-    return line
+    return re.sub(r"[ \t]+", " ", line)
 
 
 def extract_pdf_text(file_path: str) -> Tuple[str, Dict]:
     doc = fitz.open(file_path)
     all_pages = len(doc)
-    page_limit = all_pages
     page_lines: List[List[str]] = []
 
-    for idx in range(page_limit):
+    for idx in range(all_pages):
         page = doc[idx]
         rect = page.rect
         top_cut = rect.height * 0.06
@@ -113,42 +160,31 @@ def extract_pdf_text(file_path: str) -> Tuple[str, Dict]:
 
         lines = []
         for b in sorted(blocks, key=lambda x: (round(x[1], 1), round(x[0], 1))):
-            x0, y0, x1, y1, text, *_ = b
+            _, y0, _, y1, text, *_ = b
             if y1 <= top_cut or y0 >= rect.height - bottom_cut:
                 continue
             for raw in text.splitlines():
                 line = clean_common_noise(raw)
-                if not line:
-                    continue
-                if is_probable_page_number(line):
+                if not line or is_probable_page_number(line):
                     continue
                 lines.append(line)
         page_lines.append(lines)
 
-    # Remove repeating header/footer lines across many pages.
     freq = {}
     for lines in page_lines:
         if not lines:
             continue
-        cand = set(lines[:2] + lines[-2:])
-        for c in cand:
+        for c in set(lines[:2] + lines[-2:]):
             if len(c) >= 4:
                 freq[c] = freq.get(c, 0) + 1
-
-    repeat_lines = {k for k, v in freq.items() if v >= max(3, int(0.4 * page_limit))}
+    repeat_lines = {k for k, v in freq.items() if v >= max(3, int(0.4 * all_pages))}
 
     merged_pages = []
     for lines in page_lines:
         cleaned = [ln for ln in lines if ln not in repeat_lines and not is_probable_page_number(ln)]
         merged_pages.append("\n".join(cleaned))
 
-    text = "\n\n".join(merged_pages)
-    meta = {
-        "total_pages": all_pages,
-        "used_pages": page_limit,
-        "page_truncated": page_limit < all_pages,
-    }
-    return text, meta
+    return "\n\n".join(merged_pages), {"total_pages": all_pages, "used_pages": all_pages, "page_truncated": False}
 
 
 def extract_docx_text(file_path: str) -> Tuple[str, Dict]:
@@ -218,7 +254,7 @@ def detector_score_transformer(text: str, model_name: str) -> float:
         padding=True,
         return_tensors="pt",
     )
-    with torch.no_grad():
+    with torch.inference_mode():
         outputs = model(**inputs)
     probs = torch.softmax(outputs.logits, dim=-1)[:, 1].cpu().numpy()
     return float(0.75 * np.mean(probs) + 0.25 * np.max(probs))
@@ -261,11 +297,20 @@ def _extract_stat_features(text: str) -> np.ndarray:
     ])
 
 
+def is_lfs_pointer(path: Path) -> bool:
+    try:
+        txt = path.read_text(encoding="utf-8", errors="ignore")
+        return txt.startswith("version https://git-lfs.github.com/spec/v1")
+    except Exception:
+        return False
+
+
 def init_mba_pack() -> Tuple[bool, str]:
     if MBA_STATE["ready"]:
         return True, ""
     if joblib is None:
         return False, "当前环境缺少 joblib，无法加载 mba-aigc-detector 本地模型包。"
+
     needed = [
         "select5_tree_d2_model.pkl",
         "select10_tree_d2_model.pkl",
@@ -275,12 +320,13 @@ def init_mba_pack() -> Tuple[bool, str]:
     ]
     missing = [f for f in needed if not (MBA_MODELS_DIR / f).exists()]
     if missing:
-        return (
-            False,
-            "缺少 mba 模型文件。请将以下文件放入 models/mba/ : "
-            + ", ".join(needed)
-            + f"（当前缺失示例: {', '.join(missing[:2])}）",
-        )
+        return False, "缺少 mba 模型文件，请将模型文件放入 models/mba/。"
+
+    # Detect git-lfs pointer files early.
+    for f in needed:
+        p = MBA_MODELS_DIR / f
+        if is_lfs_pointer(p):
+            return False, "检测到 mba 模型文件是 Git LFS 指针，不是真实权重。请在仓库中上传真实模型二进制文件。"
 
     try:
         tok = AutoTokenizer.from_pretrained("hfl/chinese-roberta-wwm-ext")
@@ -297,11 +343,12 @@ def detector_score_mba(text: str) -> float:
     ok, msg = init_mba_pack()
     if not ok:
         raise RuntimeError(msg)
+
     tok = MBA_STATE["extractor_tokenizer"]
     mdl = MBA_STATE["extractor_model"]
-
     inputs = tok(text[:512], return_tensors="pt", max_length=512, truncation=True, padding=True)
-    with torch.no_grad():
+
+    with torch.inference_mode():
         bert_feat = mdl(**inputs).last_hidden_state[:, 0, :].cpu().numpy()[0]
 
     stat_feat = _extract_stat_features(text)
@@ -320,6 +367,7 @@ def analyze_paragraph(text: str, model_name: str) -> Dict[str, float]:
         detector = float(min(max(detector_score_mba(text) * 0.3, 0.0), 1.0))
     else:
         detector = detector_score_transformer(text, model_name)
+
     repetition = calc_repetition(text)
     variance = calc_sentence_variance(text)
     risk = float(min(max(detector * 0.78 + repetition * 0.12 + (1 - variance) * 0.10, 0.0), 1.0))
@@ -349,33 +397,41 @@ def predict_kn_like_rate(features: Dict[str, float]) -> float:
     return clip01(y)
 
 
-def analyze_document(upload_file, pasted_text, model_label, risk_threshold, topk, progress=gr.Progress()):
+def build_filtered_details(blocks: List[Dict], level_filter: str) -> str:
+    if level_filter == "全部":
+        selected = blocks
+    else:
+        selected = [b for b in blocks if b["risk_level"] == level_filter]
+    if not selected:
+        return f"当前筛选 `{level_filter}` 下暂无段落。"
+    return "\n\n---\n\n".join([b["content"] for b in selected])
+
+
+def analyze_document(upload_file, pasted_text, model_label, risk_threshold, risk_filter, progress=gr.Progress()):
     if upload_file is None and not normalize_text(pasted_text or ""):
-        return "请先上传文件，或粘贴文本。", [], ""
+        return "请先上传文件，或粘贴文本。", "", format_history_markdown()
 
     model_name = MODEL_CHOICES.get(model_label, MODEL_CHOICES[DEFAULT_MODEL_LABEL])
     if model_name == "mba_local_pack":
         ok, msg = init_mba_pack()
         if not ok:
-            return f"# 当前模型: {model_label}\n\n{msg}\n\n请切回 paperpass-v3 / zhv3 / zhv2。", [], ""
+            return f"# 当前模型: {model_label}\n\n{msg}\n\n请切回其他模型。", "", format_history_markdown()
 
+    source = "pasted_text"
     if normalize_text(pasted_text or ""):
         raw_text = pasted_text
         extract_meta = {"total_pages": None, "used_pages": None, "page_truncated": False}
-        # Textbox mode: respect user text directly, avoid extra document cleaning.
         paragraphs = split_paragraphs(raw_text)
     else:
+        source = Path(upload_file.name).name
         raw_text, extract_meta = extract_document_text(upload_file)
         paragraphs = [p for p in split_paragraphs(raw_text) if not should_skip_paragraph(p)]
-    original_count = len(paragraphs)
-    para_truncated = False
 
     if not paragraphs:
-        return "未提取到可分析正文。请尝试文本层可复制的文件，或调整排版后再试。", [], ""
+        return "未提取到可分析正文。", "", format_history_markdown()
 
-    rows = []
-    risks = []
-    details = []
+    t0 = time.time()
+    risks, details = [], []
     total = len(paragraphs)
     for i, p in enumerate(paragraphs, 1):
         progress(i / total, desc=f"推理进度: {int(i * 100 / total)}%")
@@ -387,9 +443,16 @@ def analyze_document(upload_file, pasted_text, model_label, risk_threshold, topk
         elif score["risk"] > max(0.55, risk_threshold - 0.15):
             level = "🟡"
 
-        rows.append([i, f"{score['risk']:.2%}", f"{score['detector']:.2%}", f"{score['repetition']:.2%}", p[:120]])
+        risk_level = "低风险"
+        if score["risk"] > risk_threshold:
+            risk_level = "高风险"
+        elif score["risk"] > max(0.55, risk_threshold - 0.15):
+            risk_level = "中风险"
+
         details.append(
-            f"""
+            {
+                "risk_level": risk_level,
+                "content": f"""
 {level} 段落 {i} AI风险: {score['risk']:.2%}
 
 Detector: {score['detector']:.2%}
@@ -397,44 +460,63 @@ Detector: {score['detector']:.2%}
 句式稳定性: {1 - score['variance']:.2%}
 
 {p}
-"""
+""",
+            }
         )
 
     f = build_doc_features(risks)
-    mode_line = "当前模式: 原始风险率（未加载校准模型）" if not CALIBRATION_MODEL else "当前模式: 知网对齐预测率（已加载校准模型）"
+    kn_like = predict_kn_like_rate(f)
+    elapsed = time.time() - t0
+    speed = len(paragraphs) / max(elapsed, 1e-6)
 
     trunc_info = []
-    if extract_meta.get("page_truncated"):
-        trunc_info.append(f"页面截断: 是（{extract_meta.get('used_pages')}/{extract_meta.get('total_pages')} 页）")
-    elif extract_meta.get("total_pages") is not None:
+    if extract_meta.get("total_pages") is not None:
         trunc_info.append(f"页面截断: 否（{extract_meta.get('used_pages')}/{extract_meta.get('total_pages')} 页）")
-    trunc_info.append(f"段落截断: {'是' if para_truncated else '否'}（分析 {len(paragraphs)}/{original_count} 段）")
+    trunc_info.append(f"段落截断: 否（分析 {len(paragraphs)}/{len(paragraphs)} 段）")
 
+    mode_line = "当前模式: 原始风险率（未加载校准模型）" if not CALIBRATION_MODEL else "当前模式: 知网对齐预测率（已加载校准模型）"
     summary = f"""
 # 当前模型: {model_label}
 # 综合AI风险率: {f['overall']:.2%}
-# 预测知网AIGC率: {predict_kn_like_rate(f):.2%}
+# 预测知网AIGC率: {kn_like:.2%}
 高风险段落占比: {f['high_ratio']:.2%}
 中风险段落占比: {f['mid_ratio']:.2%}
 有效段落数: {len(paragraphs)}
 
-判定阈值: {risk_threshold:.2f}
+平均速度: {speed:.2f} 段/秒
 {mode_line}
 {' | '.join(trunc_info)}
 
 （说明：该结果为“风险分析与校准预测”，并非官方系统结果）
 """
 
-    rows_sorted = sorted(rows, key=lambda r: float(r[1].strip("%")), reverse=True)
-    top_rows = rows_sorted[: int(topk)]
-    return summary, top_rows, "\n\n---\n\n".join(details)
+    filtered_details = build_filtered_details(details, risk_filter)
+
+    save_history_item(
+        {
+            "time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            "source": source,
+            "model": model_label,
+            "overall": f["overall"],
+            "kn_like": kn_like,
+            "paragraphs": len(paragraphs),
+            "high_ratio": f["high_ratio"],
+            "mid_ratio": f["mid_ratio"],
+            "elapsed_sec": elapsed,
+            "speed_para_per_sec": speed,
+        }
+    )
+
+    return summary, filtered_details, format_history_markdown()
 
 
 with gr.Blocks(theme=gr.themes.Soft(primary_hue="emerald", secondary_hue="slate"), title="论文AIGC风险检测系统") as demo:
-    gr.Markdown("""
+    gr.Markdown(
+        """
 # 论文AIGC风险检测系统
 支持 `PDF / Word(.docx) / 文本(.txt, .md)`，默认全文检测，支持直接粘贴文本。
-""")
+"""
+    )
 
     with gr.Row():
         with gr.Column(scale=1):
@@ -443,24 +525,19 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="emerald", secondary_hue="slate"
             model = gr.Dropdown(list(MODEL_CHOICES.keys()), value=DEFAULT_MODEL_LABEL, label="选择检测模型")
             with gr.Accordion("高级参数", open=False):
                 risk_threshold = gr.Slider(0.5, 0.9, value=0.75, step=0.01, label="高风险阈值")
-                topk = gr.Slider(5, 60, value=15, step=1, label="表格显示 Top-K 高风险段落")
+            risk_filter = gr.Radio(["全部", "高风险", "中风险", "低风险"], value="全部", label="风险筛选")
             run_btn = gr.Button("开始分析", variant="primary")
 
         with gr.Column(scale=2):
             summary_out = gr.Markdown(label="总览")
-            table_out = gr.Dataframe(
-                headers=["段落", "风险", "Detector", "重复度", "片段预览"],
-                datatype=["number", "str", "str", "str", "str"],
-                label="高风险段落列表",
-                wrap=True,
-            )
 
     details_out = gr.Markdown(label="段落详情")
+    history_out = gr.Markdown(label="历史记录", value=format_history_markdown())
 
     run_btn.click(
         fn=analyze_document,
-        inputs=[file_input, pasted_text, model, risk_threshold, topk],
-        outputs=[summary_out, table_out, details_out],
+        inputs=[file_input, pasted_text, model, risk_threshold, risk_filter],
+        outputs=[summary_out, details_out, history_out],
     )
 
 

models/mba/bert_tree_d1_meta.json

@@ -0,0 +1,7 @@
+{
+  "name": "bert_tree_d1",
+  "depth": 1,
+  "feature_dim": 768,
+  "has_selector": false,
+  "use_bert": true
+}

models/mba/model_manifest.json

@@ -0,0 +1,31 @@
+{
+  "models": [
+    {
+      "name": "select5_tree_d2",
+      "type": "feature_tree"
+    },
+    {
+      "name": "select10_tree_d2",
+      "type": "feature_tree"
+    },
+    {
+      "name": "select15_tree_d3",
+      "type": "feature_tree"
+    },
+    {
+      "name": "select20_tree_d2",
+      "type": "feature_tree"
+    },
+    {
+      "name": "bert_tree_d1",
+      "type": "bert_tree"
+    }
+  ],
+  "training_info": {
+    "original_samples": 15770,
+    "variant_samples": 150,
+    "total_samples": 15920,
+    "human_samples": 13191,
+    "ai_samples": 2729
+  }
+}

models/mba/select10_tree_d2_meta.json

@@ -0,0 +1,8 @@
+{
+  "name": "select10_tree_d2",
+  "k": 10,
+  "depth": 2,
+  "feature_dim": 10,
+  "has_selector": true,
+  "use_bert": false
+}

models/mba/select15_tree_d3_meta.json

@@ -0,0 +1,8 @@
+{
+  "name": "select15_tree_d3",
+  "k": 15,
+  "depth": 3,
+  "feature_dim": 15,
+  "has_selector": true,
+  "use_bert": false
+}

models/mba/select20_tree_d2_meta.json

@@ -0,0 +1,8 @@
+{
+  "name": "select20_tree_d2",
+  "k": 20,
+  "depth": 2,
+  "feature_dim": 20,
+  "has_selector": true,
+  "use_bert": false
+}

models/mba/select5_tree_d2_meta.json

@@ -0,0 +1,8 @@
+{
+  "name": "select5_tree_d2",
+  "k": 5,
+  "depth": 2,
+  "feature_dim": 5,
+  "has_selector": true,
+  "use_bert": false
+}