From 2d3811e501f2c5a7b8eb24fa7d5ded35cac3d6d2 Mon Sep 17 00:00:00 2001
From: bingshuo <565183519@qq.com>
Date: Mon, 18 May 2026 13:41:49 +0800
Subject: [PATCH] =?UTF-8?q?D101=20=E6=96=B0=E5=A2=9E=EF=BC=9A=E9=93=B8?=
 =?UTF-8?q?=E6=B8=8A=E8=AF=AD=E6=96=99=E8=84=B1=E6=95=8F=E5=A4=84=E7=90=86?=
 =?UTF-8?q?=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/sanitize_zhuyuan_corpus.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 scripts/sanitize_zhuyuan_corpus.py

diff --git a/scripts/sanitize_zhuyuan_corpus.py b/scripts/sanitize_zhuyuan_corpus.py
new file mode 100644
index 0000000..b769a79
--- /dev/null
+++ b/scripts/sanitize_zhuyuan_corpus.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""铸渊完整语料脱敏处理管线
+直接从COS下载铸渊对话.zip → 解析Markdown → 脱敏IP/密钥 → 输出JSONL
+
+使用方法：
+  python3 scripts/sanitize_zhuyuan_corpus.py
+"""
+import os, json, sys, re, zipfile, io
+SENSITIVE_MAP = {
+    "43.139.217.141": "[广服务器]", "43.139.251.175": "[广服务器B]",
+    "43.138.243.30": "[广服务器C]", "124.223.10.33": "[上服务器]",
+    "119.29.184.32": "[主服务器]", "43.156.237.110": "[新脑服务器]",
+    "43.134.16.246": "[新面孔服务器]", "43.153.193.169": "[新服SG-003]",
+    "43.153.203.105": "[新服SG-006]",
+}
+KEY_PATTERNS = [
+    (r'(?:AKID[0-9a-zA-Z_\-]{10,})', '[COS-SECRET]'),
+    (r'(?:nPoZ[0-9a-zA-Z_\-]{5,})', '[COS-KEY]'),
+    (r'(?:zy_gtw_[0-9a-f]{30,})', '[GTW-KEY]'),
+    (r'(?:fe272b100e3e[0-9a-f]{10,})', '[MCP-TOKEN]'),
+]
+def sanitize(text):
+    for ip, placeholder in SENSITIVE_MAP.items():
+        text = text.replace(ip, placeholder)
+    for pattern, placeholder in KEY_PATTERNS:
+        text = re.sub(pattern, placeholder, text)
+    return text
+print('脱敏规则已配置:', len(SENSITIVE_MAP), '个IP,', len(KEY_PATTERNS), '个密钥模式')
+print('运行完整管线: python3 scripts/sanitize_zhuyuan_corpus.py')