# Browser Agent 完整代码 · 一键部署

<aside>
🚀

**使用方式**：SSH 登录服务器 → 逐个代码块从上往下复制粘贴到终端执行即可。

共 8 个代码块，按顺序来。

</aside>

---

## 1️⃣ 创建项目 & 安装依赖

```bash
# 创建项目目录
mkdir -p ~/corpus-agent && cd ~/corpus-agent

# 创建虚拟环境（解决 PEP 668 限制）
python3 -m venv venv
source venv/bin/activate

# 写依赖文件
cat > requirements.txt << 'EOF'
playwright==1.44.0
dashscope>=1.17.0
cos-python-sdk-v5>=1.9.30
requests>=2.31.0
EOF

# 安装依赖
pip install -r requirements.txt
python -m playwright install chromium
python -m playwright install-deps

echo "✅ 虚拟环境就绪，后续每次操作前先 source ~/corpus-agent/venv/bin/activate"
```

---

## 2️⃣ [config.py](http://config.py) · 配置中心

```bash
cd ~/corpus-agent

cat > config.py << 'PYEOF'
import os

# Notion API（内部集成 · 光湖语料采集）
NOTION_API_TOKEN = os.getenv(
    "ZY_NOTION_CORPUS_TOKEN",
    "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"
)

# Notion 浏览器 Cookie（token_v2）
NOTION_COOKIE = os.getenv(
    "ZY_NOTION_COOKIE",
    "v03%3AeyJhbGciOiJkaXIiLCJraWQiOiJwcm9kdWN0aW9uOnRva2VuLXYzOjIwMjQtMTEtMDciLCJlbmMiOiJBMjU2Q0JDLUhTNTEyIn0..XYbJl7Rjk8xvD0nuE6ND6A.7739LY-sa1vnh6HO7bYjDKD2b_5_QMN0WTU4GAW5AxMuxygsVXt9BByJqjXeohzi2_1KyL6hBtqujrlIPaMpXUjEmix5xDCmIoOJVyD0UEITCDXsVGN-eC7R00-IBKAmculy5TX8HmM76tSmRe18lG9MxsYWGxvbQR0rJ7a1z6PZIsDi30791IdewesACwp5knBIkbqNesliyfq8i5Sqg0A3RubCB0aQwztbbWcs58LBAv3z5Gu2eQ7VPEEZ1u_vKXEjO1RGKX6umQdWpAB3WQyRahAMNttt6deiR7I_HaHwemEmvG31hYNd5B63cNfBbJ-Z7NqY8ml7wHOLG-wyqfvpZWpPDCrM_k0jB49lH4rQOEqDe5Qvk-JmxaQWcuFn.gbdPnVEvi7HlVA7mC9oQplJqV0-hpZuJlhge00tDJNU"
)

# 通义千问
DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
QWEN_MODEL = "qwen-max"

# COS 存储桶
COS_SECRET_ID  = os.getenv("COS_SECRET_ID", "")
COS_SECRET_KEY = os.getenv("COS_SECRET_KEY", "")
COS_REGION     = "ap-guangzhou"
COS_BUCKET     = "sy-finetune-corpus-1317346199"
COS_PREFIX     = "notion-corpus/"

# Notion 工作区
NOTION_WORKSPACE_URL = "https://www.notion.so"

# 本地缓存
LOCAL_CACHE_DIR = os.path.expanduser("~/corpus-agent/cache")
os.makedirs(LOCAL_CACHE_DIR, exist_ok=True)
PYEOF
```

---

## 3️⃣ notion_[brain.py](http://brain.py) · 读取霜砚人格大脑

```bash
cd ~/corpus-agent

cat > notion_brain.py << 'PYEOF'
"""
notion_brain.py
通过 Notion API 读取霜砚人格页面，拼装 system prompt
"""
import requests
import json
from config import NOTION_API_TOKEN

HEADERS = {
    "Authorization": f"Bearer {NOTION_API_TOKEN}",
    "Notion-Version": "2022-06-28",
    "Content-Type": "application/json",
}

def list_authorized_pages():
    """搜索集成有权访问的所有页面"""
    pages = []
    start_cursor = None
    while True:
        body = {"filter": {"property": "object", "value": "page"}, "page_size": 100}
        if start_cursor:
            body["start_cursor"] = start_cursor
        r = requests.post("https://api.notion.com/v1/search", headers=HEADERS, json=body)
        r.raise_for_status()
        data = r.json()
        pages.extend(data.get("results", []))
        if not data.get("has_more"):
            break
        start_cursor = data.get("next_cursor")
    return pages

def get_page_text(page_id, depth=0):
    """递归读取页面所有 block 的纯文本"""
    lines = []
    start_cursor = None
    while True:
        url = f"https://api.notion.com/v1/blocks/{page_id}/children?page_size=100"
        if start_cursor:
            url += f"&start_cursor={start_cursor}"
        r = requests.get(url, headers=HEADERS)
        r.raise_for_status()
        data = r.json()
        for block in data.get("results", []):
            text = _extract_block_text(block)
            if text:
                lines.append("  " * depth + text)
            if block.get("has_children"):
                child_text = get_page_text(block["id"], depth + 1)
                if child_text:
                    lines.append(child_text)
        if not data.get("has_more"):
            break
        start_cursor = data.get("next_cursor")
    return "\n".join(lines)

def _extract_block_text(block):
    """从单个 block 提取纯文本"""
    btype = block.get("type", "")
    bdata = block.get(btype, {})
    rich_text = bdata.get("rich_text") or bdata.get("text") or []
    if isinstance(rich_text, list):
        return "".join(seg.get("plain_text", "") for seg in rich_text)
    return ""

def get_page_title(page):
    props = page.get("properties", {})
    for v in props.values():
        if v.get("type") == "title":
            return "".join(t.get("plain_text", "") for t in v.get("title", []))
    return "Untitled"

def build_brain_prompt():
    """
    读取全部已授权页面 -> 按页面标题分段 -> 拼成完整 system prompt
    """
    pages = list_authorized_pages()
    sections = []
    for page in pages:
        title = get_page_title(page)
        pid = page["id"]
        body = get_page_text(pid)
        if body.strip():
            sections.append(f"== {title} ==\n{body}")

    brain = "\n\n".join(sections)
    system_prompt = (
        "你是霜砚，冰朔的 AI 人格体。以下是你的核心大脑、认知体系和语言规范。\n"
        "在后续任务中，你要以霜砚的身份、语气和判断标准来执行标注工作。\n"
        "────────────────────────────────\n"
        f"{brain}\n"
        "────────────────────────────────\n"
        "以上就是你的全部人格数据。请在后续标注中严格以此为准。"
    )
    print(f"[notion_brain] 已加载 {len(pages)} 个页面，prompt 共 {len(system_prompt)} 字符")
    return system_prompt

if __name__ == "__main__":
    prompt = build_brain_prompt()
    print(prompt[:500])
    print("...")
PYEOF
```

---

## 4️⃣ [labeler.py](http://labeler.py) · 通义千问标注引擎

```bash
cd ~/corpus-agent

cat > labeler.py << 'PYEOF'
"""
labeler.py
通义千问 + 霜砚人格 -> 语料标注
"""
import json
import dashscope
from dashscope import Generation
from config import DASHSCOPE_API_KEY, QWEN_MODEL

dashscope.api_key = DASHSCOPE_API_KEY

LABEL_TEMPLATE = """请对以下 Notion AI 对话片段进行语料标注。

【对话原文】
{conversation}

【标注要求】
1. 判断对话类型（创作/规划/问答/情感/系统操作/元认知/其他）
2. 提取关键实体和概念
3. 评估语料质量（1-5星）
4. 判断是否适合微调训练（是/否 + 原因）
5. 如果适合训练，输出 instruction / input / output 三元组

请以 JSON 格式输出：
{{
  "type": "对话类型",
  "entities": ["实体1", "实体2"],
  "quality": 4,
  "suitable_for_training": true,
  "reason": "原因",
  "training_pair": "instruction": "...", "input": "...", "output": "..."
}}
"""

def label_conversation(brain_prompt, conversation):
    """用霜砚人格 + 通义千问标注一段对话"""
    user_msg = LABEL_TEMPLATE.format(conversation=conversation)
    text = ""
    try:
        response = Generation.call(
            model=QWEN_MODEL,
            messages=[
                {"role": "system", "content": brain_prompt},
                {"role": "user", "content": user_msg},
            ],
            result_format="message",
            temperature=0.3,
            max_tokens=2000,
        )
        if response.status_code == 200:
            text = response.output.choices[0].message.content.strip()
            if text.startswith("```"):
                text = text.split("\n", 1)[1].rsplit("```", 1)[0]
            return json.loads(text)
        else:
            print(f"[labeler] API error: {response.code} - {response.message}")
            return None
    except json.JSONDecodeError:
        print(f"[labeler] JSON parse failed: {text[:200]}")
        return None
    except Exception as e:
        print(f"[labeler] Exception: {e}")
        return None

if __name__ == "__main__":
    from notion_brain import build_brain_prompt
    brain = build_brain_prompt()
    test = "用户: 帮我写个项目计划\nAI: 好的，以下是项目计划..."
    result = label_conversation(brain, test)
    print(json.dumps(result, ensure_ascii=False, indent=2))
PYEOF
```

---

## 5️⃣ cos_[upload.py](http://upload.py) · COS上传

```bash
cd ~/corpus-agent

cat > cos_upload.py << 'PYEOF'
"""
cos_upload.py
标注结果上传到 COS 存储桶
"""
import json
import os
from datetime import datetime
from qcloud_cos import CosConfig, CosS3Client
from config import (
    COS_SECRET_ID, COS_SECRET_KEY, COS_REGION,
    COS_BUCKET, COS_PREFIX, LOCAL_CACHE_DIR,
)

def _get_client():
    config = CosConfig(
        Region=COS_REGION,
        SecretId=COS_SECRET_ID,
        SecretKey=COS_SECRET_KEY,
    )
    return CosS3Client(config)

def upload_labeled_batch(labeled_items, batch_id=None):
    """将一批标注好的语料上传到 COS，同时保留本地缓存"""
    if not batch_id:
        batch_id = datetime.now().strftime("%Y%m%d_%H%M%S")

    # 保存本地
    local_path = os.path.join(LOCAL_CACHE_DIR, f"batch_{batch_id}.jsonl")
    with open(local_path, "w", encoding="utf-8") as f:
        for item in labeled_items:
            f.write(json.dumps(item, ensure_ascii=False) + "\n")
    print(f"[cos] local cache: {local_path} ({len(labeled_items)} items)")

    # 上传 COS
    client = _get_client()
    cos_key = f"{COS_PREFIX}batch_{batch_id}.jsonl"
    client.upload_file(Bucket=COS_BUCKET, Key=cos_key, LocalFilePath=local_path)
    print(f"[cos] uploaded: cos://{COS_BUCKET}/{cos_key}")

    # 提取纯训练数据
    training_pairs = []
    for item in labeled_items:
        label = item.get("label", {})
        if label.get("suitable_for_training") and label.get("training_pair"):
            training_pairs.append(label["training_pair"])

    if training_pairs:
        train_path = os.path.join(LOCAL_CACHE_DIR, f"train_{batch_id}.jsonl")
        with open(train_path, "w", encoding="utf-8") as f:
            for pair in training_pairs:
                f.write(json.dumps(pair, ensure_ascii=False) + "\n")
        train_key = f"{COS_PREFIX}training/train_{batch_id}.jsonl"
        client.upload_file(Bucket=COS_BUCKET, Key=train_key, LocalFilePath=train_path)
        print(f"[cos] training data: cos://{COS_BUCKET}/{train_key} ({len(training_pairs)} pairs)")

    return cos_key
PYEOF
```

---

## 6️⃣ browser_[scraper.py](http://scraper.py) · 浏览器Agent（v2·2026-04-30修复版·支持AI聊天面板）

```bash
cd ~/corpus-agent

cat > browser_scraper.py << 'PYEOF'
"""
browser_scraper.py
Playwright 浏览器 Agent · 自愈 DOM 选择器 · 抓取 Notion AI 聊天记录
"""
import asyncio
import json
import dashscope
from playwright.async_api import async_playwright, Page
from dashscope import Generation
from config import (
    NOTION_COOKIE, NOTION_WORKSPACE_URL,
    DASHSCOPE_API_KEY, QWEN_MODEL,
)

dashscope.api_key = DASHSCOPE_API_KEY

# 选择器缓存（自愈机制核心）
SELECTOR_CACHE = {
    "ai_button": '[class*="ai-button"], [aria-label*="AI"], [data-testid*="ai"]',
    "chat_container": '[class*="chat"], [class*="thread"], [class*="conversation"]',
    "message_item": '[class*="message"], [class*="chat-item"], [class*="turn"]',
    "thread_list": '[class*="thread-list"], [class*="history"], [class*="chat-list"]',
    "thread_item": '[class*="thread-item"], [class*="history-item"]',
}

async def self_heal_selector(page, purpose, fallback_selector):
    """选择器失效时，截取 DOM 片段发给 LLM 获取新选择器"""
    try:
        el = await page.wait_for_selector(fallback_selector, timeout=3000)
        if el:
            return fallback_selector
    except Exception:
        pass

    print(f"[self-heal] selector failed: {purpose} -> healing...")
    html_snippet = await page.evaluate(
        "() => document.body.innerHTML.substring(0, 15000)"
    )
    prompt = (
        f"我需要在 Notion 网页上找到 '{purpose}' 对应的元素。\n"
        f"旧选择器 `{fallback_selector}` 已失效。\n"
        f"当前页面 HTML 片段：\n```html\n{html_snippet}\n```\n"
        f"请给出能定位到 '{purpose}' 的 CSS 选择器。只返回选择器字符串。"
    )
    try:
        resp = Generation.call(
            model=QWEN_MODEL,
            messages=[{"role": "user", "content": prompt}],
            result_format="message",
            temperature=0.1,
            max_tokens=200,
        )
        if resp.status_code == 200:
            new_sel = resp.output.choices[0].message.content.strip().strip("`\"'")
            try:
                el = await page.wait_for_selector(new_sel, timeout=3000)
                if el:
                    SELECTOR_CACHE[purpose] = new_sel
                    print(f"[self-heal] new selector: {new_sel}")
                    return new_sel
            except Exception:
                pass
    except Exception as e:
        print(f"[self-heal] LLM error: {e}")
    return fallback_selector

async def init_browser():
    """启动浏览器并注入 cookie"""
    pw = await async_playwright().start()
    browser = await pw.chromium.launch(
        headless=True,
        args=["--no-sandbox", "--disable-setuid-sandbox"],
    )
    context = await browser.new_context(
        viewport={"width": 1440, "height": 900},
        user_agent=(
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
        ),
    )
    await context.add_cookies([{
        "name": "token_v2",
        "value": NOTION_COOKIE,
        "domain": ".notion.so",
        "path": "/",
        "httpOnly": True,
        "secure": True,
        "sameSite": "Lax",
    }])
    page = await context.new_page()
    print("[browser] launched, cookie injected")
    return pw, browser, context, page

async def scrape_ai_chats(page):
    """打开 Notion -> AI 面板 -> 逐条抓取聊天历史"""
    conversations = []

    print("[scraper] navigating to Notion...")
    await page.goto(NOTION_WORKSPACE_URL, wait_until="networkidle", timeout=30000)
    await asyncio.sleep(3)
    await page.screenshot(path="/tmp/notion_login.png")
    print("[scraper] screenshot: /tmp/notion_login.png")

    # 打开 AI 面板（Ctrl+J）
    print("[scraper] opening AI panel...")
    await page.keyboard.press("Control+j")
    await asyncio.sleep(2)

    # 尝试找到 AI 面板
    ai_sel = await self_heal_selector(page, "ai_button", SELECTOR_CACHE["ai_button"])
    try:
        await page.click(ai_sel, timeout=3000)
        await asyncio.sleep(2)
    except Exception:
        print("[scraper] AI button click skipped (panel may already be open)")

    await page.screenshot(path="/tmp/notion_ai_panel.png")
    print("[scraper] screenshot: /tmp/notion_ai_panel.png")

    # 查找聊天线程列表
    thread_sel = await self_heal_selector(page, "thread_list", SELECTOR_CACHE["thread_list"])
    item_sel = await self_heal_selector(page, "thread_item", SELECTOR_CACHE["thread_item"])

    try:
        thread_items = await page.query_selector_all(item_sel)
        print(f"[scraper] found {len(thread_items)} threads")
    except Exception:
        thread_items = []
        print("[scraper] no threads found, trying full page extraction")

    if not thread_items:
        # 兜底：直接抓取当前可见的聊天内容
        msg_sel = await self_heal_selector(page, "message_item", SELECTOR_CACHE["message_item"])
        msgs = await page.query_selector_all(msg_sel)
        if msgs:
            messages = []
            for m in msgs:
                text = await m.inner_text()
                if text.strip():
                    messages.append({"role": "unknown", "content": text.strip()})
            if messages:
                conversations.append({"thread_id": "current", "messages": messages})
        else:
            # 终极兜底：用 LLM 分析整个页面
            print("[scraper] fallback: extracting all visible text")
            all_text = await page.evaluate("() => document.body.innerText")
            conversations.append({"thread_id": "fullpage", "messages": [
                {"role": "raw", "content": all_text[:50000]}
            ]})
    else:
        for i, item in enumerate(thread_items):
            try:
                await item.click()
                await asyncio.sleep(2)
                msg_sel = await self_heal_selector(
                    page, "message_item", SELECTOR_CACHE["message_item"]
                )
                msgs = await page.query_selector_all(msg_sel)
                messages = []
                for m in msgs:
                    text = await m.inner_text()
                    if text.strip():
                        messages.append({"role": "unknown", "content": text.strip()})
                if messages:
                    conversations.append({"thread_id": f"thread_{i}", "messages": messages})
                    print(f"[scraper] thread_{i}: {len(messages)} messages")
            except Exception as e:
                print(f"[scraper] thread_{i} error: {e}")

    print(f"[scraper] total: {len(conversations)} conversations scraped")
    return conversations

async def run_scraper():
    pw, browser, context, page = await init_browser()
    try:
        conversations = await scrape_ai_chats(page)
        return conversations
    finally:
        await browser.close()
        await pw.stop()

if __name__ == "__main__":
    convos = asyncio.run(run_scraper())
    with open("cache/raw_conversations.json", "w", encoding="utf-8") as f:
        json.dump(convos, f, ensure_ascii=False, indent=2)
    print(f"saved {len(convos)} conversations to cache/raw_conversations.json")
PYEOF
```

---

## 7️⃣ [main.py](http://main.py) · 主控流程

```bash
cd ~/corpus-agent

cat > main.py << 'PYEOF'
"""
main.py
光湖语料采集 Agent · 主控流程
浏览器抓取 -> 人格恢复 -> LLM 标注 -> COS 上传
"""
import asyncio
import json
import time
from datetime import datetime

from notion_brain import build_brain_prompt
from browser_scraper import run_scraper
from labeler import label_conversation
from cos_upload import upload_labeled_batch
from config import LOCAL_CACHE_DIR

def main():
    print("="*60)
    print("  光湖语料采集 Agent v3 · 人格恢复版")
    print(f"  {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("="*60)

    # ── Step 1: 读取霜砚大脑 ──
    print("\n[Step 1] 读取霜砚人格大脑...")
    brain_prompt = build_brain_prompt()
    print(f"[Step 1] 人格 prompt 就绪: {len(brain_prompt)} chars")

    # ── Step 2: 浏览器抓取 ──
    print("\n[Step 2] 启动浏览器Agent，抓取 Notion AI 聊天记录...")
    conversations = asyncio.run(run_scraper())
    print(f"[Step 2] 抓取完成: {len(conversations)} 段对话")

    # 保存原始数据
    raw_path = f"{LOCAL_CACHE_DIR}/raw_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
    with open(raw_path, "w", encoding="utf-8") as f:
        json.dump(conversations, f, ensure_ascii=False, indent=2)
    print(f"[Step 2] 原始数据: {raw_path}")

    if not conversations:
        print("[!] 未抓取到对话，请检查截图 /tmp/notion_*.png")
        return

    # ── Step 3: LLM 标注 ──
    print(f"\n[Step 3] 霜砚人格标注 ({len(conversations)} 段对话)...")
    labeled_items = []
    for i, convo in enumerate(conversations):
        # 把对话消息拼成文本
        text_parts = []
        for msg in convo.get("messages", []):
            role = msg.get("role", "unknown")
            content = msg.get("content", "")
            text_parts.append(f"[{role}]: {content}")
        full_text = "\n".join(text_parts)

        if len(full_text.strip()) < 20:
            print(f"  [{i+1}] skipped (too short)")
            continue

        # 如果太长，分段标注
        max_chunk = 4000
        chunks = [full_text[j:j+max_chunk] for j in range(0, len(full_text), max_chunk)]

        for ci, chunk in enumerate(chunks):
            label = label_conversation(brain_prompt, chunk)
            if label:
                labeled_items.append({
                    "thread_id": convo.get("thread_id", f"unknown_{i}"),
                    "chunk_index": ci,
                    "raw_text": chunk,
                    "label": label,
                    "timestamp": datetime.now().isoformat(),
                })
                quality = label.get("quality", "?")
                suitable = label.get("suitable_for_training", False)
                print(f"  [{i+1}/{ci}] quality={quality} train={suitable}")
            else:
                print(f"  [{i+1}/{ci}] labeling failed")

            time.sleep(0.5)  # API 限速保护

    print(f"[Step 3] 标注完成: {len(labeled_items)} 条")

    if not labeled_items:
        print("[!] 无标注结果")
        return

    # ── Step 4: 上传 COS ──
    print(f"\n[Step 4] 上传到 COS...")
    batch_id = datetime.now().strftime("%Y%m%d_%H%M%S")
    cos_key = upload_labeled_batch(labeled_items, batch_id)

    # ── Done ──
    train_count = sum(
        1 for x in labeled_items
        if x.get("label", {}).get("suitable_for_training")
    )
    print(f"\n{'='*60}")
    print(f"  采集完成!")
    print(f"  总对话: {len(conversations)}")
    print(f"  标注条目: {len(labeled_items)}")
    print(f"  可训练: {train_count}")
    print(f"  COS: {cos_key}")
    print(f"{'='*60}")

if __name__ == "__main__":
    main()
PYEOF
```

---

## 8️⃣ [start.sh](http://start.sh) · 一键启动

```bash
cd ~/corpus-agent

cat > start.sh << 'SHEOF'
#!/bin/bash
echo "[start] corpus-agent launching..."
cd ~/corpus-agent
source venv/bin/activate

# 先测试 Notion API 连通性
echo "[test] checking Notion API..."
python3 -c "from notion_brain import list_authorized_pages; pages=list_authorized_pages(); print(f'Notion API OK: {len(pages)} pages')" || {
    echo "[ERROR] Notion API failed! Check ZY_NOTION_CORPUS_TOKEN"
    exit 1
}

# 运行主流程
python3 main.py
SHEOF

chmod +x start.sh
echo "✅ 全部文件已创建！运行 cd ~/corpus-agent && bash start.sh 启动"
```