# browser_scraper.py 修复版v3 · Claude实测DOM · 2026-04-30

<aside>
⚠️

**v3 · Claude浏览器插件实测DOM版** — 选择器全部经过实测验证

之前v1/v2的选择器是猜的，全部不对。这版是Claude在浏览器里直接验证DOM结构后写的。

</aside>

---

## 部署到服务器（从 Notion API 直接拉代码）

<aside>
🧠

**方案：正确代码已在本页面的Python代码块里。服务器上有NOTION_API_TOKEN，直接用API读取这个页面的代码块写入文件。一条命令。**

</aside>

### 部署命令

复制 → 粘贴到终端 → 回车

```bash
cd ~/corpus-agent && PID=$(curl -s 'https://api.notion.com/v1/search' -H 'Authorization: Bearer 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' -H 'Notion-Version: 2022-06-28' -H 'Content-Type: application/json' -d '{"query":"browser_scraper","page_size":1}' | python3 -c "import sys,json;print(json.load(sys.stdin)['results'][0]['id'])") && curl -s "https://api.notion.com/v1/blocks/$PID/children?page_size=200" -H 'Authorization: Bearer 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' -H 'Notion-Version: 2022-06-28' | python3 -c "import sys,json;bs=json.load(sys.stdin);cbs=[b for b in bs['results'] if b['type']=='code' and b['code'].get('language')=='python'];txt=''.join(t['plain_text'] for t in cbs[0]['code']['rich_text']);open('browser_scraper.py','w').write(txt);print('wrote',len(txt),'chars')"
```

### 测试

```bash
cd ~/corpus-agent && python browser_scraper.py
```

<aside>
⚠️

**注意：** cookie可能已过期（两天前的）。如果报Timeout错误，需要重新从浏览器导出新的token_v2。

</aside>

---

## 完整代码（让Claude帮你写到服务器）

```python
from __future__ import annotations

import asyncio
import json
import re
from datetime import datetime
from pathlib import Path
from typing import Any

from playwright.async_api import async_playwright, BrowserContext, Page

from config import NOTION_COOKIE, NOTION_WORKSPACE_URL

CACHE_DIR = Path("cache")
CACHE_DIR.mkdir(exist_ok=True)

EXTRACT_CONVERSATION_JS = r"""
() => {
  const userNodes = document.querySelectorAll('[data-agent-chat-user-step-id]');
  const users = Array.from(userNodes).map(el => ({
    role: 'user',
    stepId: el.getAttribute('data-agent-chat-user-step-id'),
    el,
    text: (el.innerText || '').trim()
  })).filter(x => x.text);

  const aiNodes = document.querySelectorAll(
    'div[role="group"][data-content-editable-root="true"].whenContentEditable'
  );
  const ais = Array.from(aiNodes).map(el => ({
    role: 'assistant',
    el,
    text: (el.innerText || '').trim()
  })).filter(x => x.text);

  const items = [...users, ...ais].sort((a, b) => {
    const p = a.el.compareDocumentPosition(b.el);
    if (p & Node.DOCUMENT_POSITION_FOLLOWING) return -1;
    if (p & Node.DOCUMENT_POSITION_PRECEDING) return 1;
    return 0;
  });

  const merged = [];
  for (const it of items) {
    const last = merged[merged.length - 1];
    if (last && last.role === 'assistant' && it.role === 'assistant') {
      last.text += '\n\n' + it.text;
    } else {
      merged.push({
        role: it.role,
        text: it.text,
        ...(it.stepId ? { stepId: it.stepId } : {})
      });
    }
  }

  return {
    title: document.title,
    url: location.href,
    scrapedAt: new Date().toISOString(),
    totalTurns: merged.length,
    totalChars: merged.reduce((s, m) => s + m.text.length, 0),
    messages: merged
  };
}
"""

LIST_AI_CHATS_JS = r"""
() => {
  const items = document.querySelectorAll(
    '.notion-scroller.vertical a[role="treeitem"][href]'
  );
  return Array.from(items)
    .map(a => ({
      title: (a.innerText || '').trim().replace(/\s+/g, ' '),
      href: a.href
    }))
    .filter(x => /[?&]wfv=chat\b/.test(x.href));
}
"""

async def _new_context(playwright) -> BrowserContext:
    browser = await playwright.chromium.launch(
        headless=True,
        args=["--no-sandbox", "--disable-setuid-sandbox"],
    )
    context = await browser.new_context(
        viewport={"width": 1440, "height": 900},
        user_agent=(
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
        ),
    )
    await context.add_cookies([{
        "name": "token_v2",
        "value": NOTION_COOKIE,
        "domain": ".notion.so",
        "path": "/",
        "httpOnly": True,
        "secure": True,
        "sameSite": "Lax",
    }])
    return context

async def _wait_for_chat_dom(page: Page, timeout_ms: int = 20000) -> None:
    await page.wait_for_selector(
        '[data-agent-chat-user-step-id], '
        'div[role="group"][data-content-editable-root="true"].whenContentEditable',
        timeout=timeout_ms,
    )
    await page.wait_for_timeout(1500)

async def _list_ai_chats(page: Page) -> list[dict[str, str]]:
    await page.goto(NOTION_WORKSPACE_URL, wait_until="domcontentloaded")
    await page.wait_for_selector(
        '.notion-scroller.vertical a[role="treeitem"]',
        timeout=20000,
    )
    await page.wait_for_timeout(1500)
    chats = await page.evaluate(LIST_AI_CHATS_JS)
    return chats

async def _extract_conversation(page: Page, url: str) -> dict[str, Any]:
    await page.goto(url, wait_until="domcontentloaded")
    await _wait_for_chat_dom(page)
    return await page.evaluate(EXTRACT_CONVERSATION_JS)

def _save(conversations: list[dict[str, Any]]) -> tuple[Path, Path]:
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    json_path = CACHE_DIR / f"notion_ai_chats_{ts}.json"
    jsonl_path = CACHE_DIR / f"notion_ai_chats_{ts}.jsonl"

    with json_path.open("w", encoding="utf-8") as f:
        json.dump(conversations, f, ensure_ascii=False, indent=2)

    with jsonl_path.open("w", encoding="utf-8") as f:
        for conv in conversations:
            f.write(json.dumps(conv, ensure_ascii=False) + "\n")

    return json_path, jsonl_path

async def run_scraper() -> list[dict[str, Any]]:
    async with async_playwright() as p:
        context = await _new_context(p)
        page = await context.new_page()

        chats = await _list_ai_chats(page)
        print(f"[scraper] discovered {len(chats)} AI chat conversations")

        conversations = []
        for idx, c in enumerate(chats, 1):
            try:
                conv = await _extract_conversation(page, c["href"])
                conv["sidebar_title"] = c["title"]
                conversations.append(conv)
                print(
                    f"[scraper] [{idx}/{len(chats)}] "
                    f"{conv['totalTurns']} turns / {conv['totalChars']} chars  "
                    f"— {c['title'][:40]}"
                )
            except Exception as e:
                print(f"[scraper] [{idx}/{len(chats)}] FAILED: {c['title'][:40]}  {e!r}")

        json_path, jsonl_path = _save(conversations)
        print(f"[scraper] saved -> {json_path}")
        print(f"[scraper] saved -> {jsonl_path}")

        await context.close()
        return conversations

if __name__ == "__main__":
    convos = asyncio.run(run_scraper())
    total_turns = sum(c.get("totalTurns", 0) for c in convos)
    total_chars = sum(c.get("totalChars", 0) for c in convos)
    print(f"\n[scraper] DONE  chats={len(convos)}  turns={total_turns}  chars={total_chars}")
```

---

## 选择器对照（Claude实测vs之前的猜测）

| 功能 | 之前的（猜的❌） | Claude实测（真的✅） |
| --- | --- | --- |
| 打开历史菜单 | button[title="history"] | 不需要！侧边栏直接可见 |
| 对话列表项 | [role="menuitem"] | a[role="treeitem"] + wfv=chat过滤 |
| 用户消息 | div[class*="message"] | [data-agent-chat-user-step-id] |
| AI回复 | div[class*="prose"] | div[role="group"][data-content-editable-root="true"].whenContentEditable |
| 打开AI面板 | Ctrl+J | 不需要！直接goto会话URL |

---

## 改动说明（vs Claude原版）

- `run_scraper()` 返回 `list` 而不是 `dict`，兼容现有 [main.py](http://main.py)
- 其余逻辑完全保留Claude实测版本