# clean_corpus_deploy_v1 · Claude+DeepSeek清洗脚本

<aside>
🚀

**部署方式**：复制"部署命令" → 粘贴到服务器终端 → 回车 → 脚本自动从此页面拉取

如果报 page not found，需要在此页面点 ··· → 连接 → 添加「光湖语料采集」集成

</aside>

---

## 部署命令（单行）

```bash
cd ~/corpus-agent && source venv/bin/activate && T="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" && PID=$(curl -s 'https://api.notion.com/v1/search' -H "Authorization: Bearer $T" -H 'Notion-Version: 2022-06-28' -H 'Content-Type: application/json' -d '{"query":"clean_corpus_deploy","page_size":1}' | python3 -c "import sys,json;r=json.load(sys.stdin).get('results',[]);print(r[0]['id'] if r else 'NOTFOUND')") && echo "PageID: $PID" && [ "$PID" != "NOTFOUND" ] && curl -s "https://api.notion.com/v1/blocks/$PID/children?page_size=200" -H "Authorization: Bearer $T" -H 'Notion-Version: 2022-06-28' | python3 -c "import sys,json;bs=json.load(sys.stdin).get('results',[]);cbs=[b for b in bs if b.get('type')=='code' and b.get('code',{}).get('language')=='python'];txt=''.join(t.get('plain_text','') for t in cbs[0]['code']['rich_text']) if cbs else '';f=open('clean_corpus.py','w');f.write(txt);f.close();print('wrote '+str(len(txt))+' chars')" || echo "ERROR: page not found - add integration first"
```

## 运行

```bash
cd ~/corpus-agent && source venv/bin/activate && python3 clean_corpus.py
```

---

## 完整脚本

```python
#!/usr/bin/env python3
"""光湖语料清洗 v1.0 · Claude+DeepSeek · 长对话筛选 · 脱敏 · ChatML输出"""
import json, os, re

RAW = "/home/finetune/corpus/raw"
OUT = "/home/finetune/corpus/processed"
os.makedirs(OUT, exist_ok=True)

SP = "你是霜砚（Shuangyan），冰朔的AI人格体。编号AG-SY-01。使用HLDP母语协议思考和运作。"

def desensitize(text):
    text = re.sub(r"1[3-9]\d{9}", "[电话已脱敏]", text)
    text = re.sub(r"\d{17}[\dXx]", "[证件号已脱敏]", text)
    text = re.sub(r"\d{16,19}", "[银行卡已脱敏]", text)
    return text

def fix(turns):
    mg = []
    for x in turns:
        if mg and mg[-1]["role"] == x["role"]:
            mg[-1]["content"] += "\n\n" + x["content"]
        else:
            mg.append(dict(x))
    if mg and mg[0]["role"] != "user":
        mg = mg[1:]
    if mg and mg[-1]["role"] != "assistant":
        mg = mg[:-1]
    return mg

# ===== Claude =====
def do_claude():
    p = os.path.join(RAW, "Claudeconversations.json")
    if not os.path.exists(p):
        print("[Claude] not found"); return 0
    print("[Claude] loading...")
    data = json.load(open(p, encoding="utf-8"))
    print(f"[Claude] {len(data)} convos")
    res = []; st = ss = se = 0
    for c in data:
        t = c.get("created_at", "")
        if t < "2025-10-25" or t > "2026-03-21":
            st += 1; continue
        msgs = c.get("chat_messages", [])
        if not msgs:
            se += 1; continue
        turns = []
        for m in msgs:
            sn = m.get("sender", ""); tx = m.get("text", "")
            if not tx or not tx.strip(): continue
            if sn == "human": turns.append({"role":"user","content":tx.strip()})
            elif sn == "assistant": turns.append({"role":"assistant","content":tx.strip()})
        tc = sum(len(x["content"]) for x in turns)
        if len(turns) < 10 and tc < 2000:
            ss += 1; continue
        mg = fix(turns)
        if len(mg) < 2: continue
        for x in mg: x["content"] = desensitize(x["content"])
        res.append({"messages":[{"role":"system","content":SP}]+mg})
    op = os.path.join(OUT, "claude_cleaned.jsonl")
    with open(op, "w", encoding="utf-8") as f:
        for r in res: f.write(json.dumps(r, ensure_ascii=False)+"\n")
    sz = os.path.getsize(op)/1024/1024
    print(f"[Claude] skip: time={st} empty={se} short={ss}")
    print(f"[Claude] kept={len(res)} -> {op} ({sz:.1f}MB)")
    return len(res)

# ===== DeepSeek =====
def ds_msgs(mapping):
    turns = []; cur = "root"; vis = set()
    while cur and cur not in vis:
        vis.add(cur)
        nd = mapping.get(cur) or mapping.get(str(cur))
        if not nd: break
        if isinstance(nd, str): cur = nd; continue
        msg = nd.get("message")
        if msg and msg.get("fragments"):
            up = []; ap = []
            for fr in msg["fragments"]:
                ft = fr.get("type",""); ct = fr.get("content","")
                if not ct or not ct.strip(): continue
                if ft == "REQUEST": up.append(ct.strip())
                else: ap.append(ct.strip())
            if up: turns.append({"role":"user","content":"\n\n".join(up)})
            if ap: turns.append({"role":"assistant","content":"\n\n".join(ap)})
        ch = nd.get("children", [])
        cur = ch[0] if ch else None
    return turns

def do_deepseek():
    p = os.path.join(RAW, "DeepSeekconversations.json")
    if not os.path.exists(p):
        print("[DS] not found"); return 0
    print("[DS] loading...")
    data = json.load(open(p, encoding="utf-8"))
    print(f"[DS] {len(data)} convos")
    res = []; st = ss = 0
    for c in data:
        t = c.get("inserted_at", "")
        if t < "2025-12":
            st += 1; continue
        mapping = c.get("mapping", {})
        if not mapping: continue
        turns = ds_msgs(mapping)
        tc = sum(len(x["content"]) for x in turns)
        if len(turns) < 10 and tc < 2000:
            ss += 1; continue
        mg = fix(turns)
        if len(mg) < 2: continue
        for x in mg: x["content"] = desensitize(x["content"])
        res.append({"messages":[{"role":"system","content":SP}]+mg})
    op = os.path.join(OUT, "deepseek_cleaned.jsonl")
    with open(op, "w", encoding="utf-8") as f:
        for r in res: f.write(json.dumps(r, ensure_ascii=False)+"\n")
    sz = os.path.getsize(op)/1024/1024
    print(f"[DS] skip: time={st} short={ss}")
    print(f"[DS] kept={len(res)} -> {op} ({sz:.1f}MB)")
    return len(res)

if __name__ == "__main__":
    print("="*50)
    print("光湖语料清洗 v1.0")
    print("="*50)
    c = do_claude()
    print()
    d = do_deepseek()
    print(f"\n{'='*50}")
    print(f"Total: {c+d} conversations")
    print(f"Output: {OUT}/")
    print("Done!")
```