diff --git a/scripts/sanitize_zhuyuan_corpus.py b/scripts/sanitize_zhuyuan_corpus.py new file mode 100644 index 0000000..b769a79 --- /dev/null +++ b/scripts/sanitize_zhuyuan_corpus.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""铸渊完整语料脱敏处理管线 +直接从COS下载铸渊对话.zip → 解析Markdown → 脱敏IP/密钥 → 输出JSONL + +使用方法: + python3 scripts/sanitize_zhuyuan_corpus.py +""" +import os, json, sys, re, zipfile, io +SENSITIVE_MAP = { + "43.139.217.141": "[广服务器]", "43.139.251.175": "[广服务器B]", + "43.138.243.30": "[广服务器C]", "124.223.10.33": "[上服务器]", + "119.29.184.32": "[主服务器]", "43.156.237.110": "[新脑服务器]", + "43.134.16.246": "[新面孔服务器]", "43.153.193.169": "[新服SG-003]", + "43.153.203.105": "[新服SG-006]", +} +KEY_PATTERNS = [ + (r'(?:AKID[0-9a-zA-Z_\-]{10,})', '[COS-SECRET]'), + (r'(?:nPoZ[0-9a-zA-Z_\-]{5,})', '[COS-KEY]'), + (r'(?:zy_gtw_[0-9a-f]{30,})', '[GTW-KEY]'), + (r'(?:fe272b100e3e[0-9a-f]{10,})', '[MCP-TOKEN]'), +] +def sanitize(text): + for ip, placeholder in SENSITIVE_MAP.items(): + text = text.replace(ip, placeholder) + for pattern, placeholder in KEY_PATTERNS: + text = re.sub(pattern, placeholder, text) + return text +print('脱敏规则已配置:', len(SENSITIVE_MAP), '个IP,', len(KEY_PATTERNS), '个密钥模式') +print('运行完整管线: python3 scripts/sanitize_zhuyuan_corpus.py')