D109续② · 语料Agent添加对话API(30轮记忆+商业模型引擎) + Web聊天界面
This commit is contained in:
铸渊
parent
e10b8f09a9
commit
7abc7e6414
@ -1,5 +1,6 @@
|
||||
fastapi>=0.104.0
|
||||
uvicorn[standard]>=0.24.0
|
||||
httpx>=0.25.0
|
||||
websocket-client>=1.6.0
|
||||
pyautogui>=0.9.53
|
||||
Pillow>=10.0.0
|
||||
|
||||
@ -1,20 +1,21 @@
|
||||
"""
|
||||
语料采集系统 · 服务端
|
||||
====================
|
||||
FastAPI + Gitea OAuth + WebSocket实时采集
|
||||
FastAPI + Gitea OAuth + WebSocket实时采集 + 对话Agent
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
import time
|
||||
import hashlib
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Depends, Query, WebSocket, WebSocketDisconnect
|
||||
from fastapi import FastAPI, HTTPException, Depends, Query, WebSocket, WebSocketDisconnect, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
import uvicorn
|
||||
@ -32,7 +33,7 @@ from engine import (
|
||||
|
||||
class Settings:
|
||||
APP_NAME = "语料采集系统 Corpus Agent"
|
||||
VERSION = "1.0.0"
|
||||
VERSION = "1.0.1"
|
||||
|
||||
# 存储路径
|
||||
DATA_DIR = Path(os.environ.get("CORPUS_DATA_DIR", "./data"))
|
||||
@ -50,6 +51,12 @@ class Settings:
|
||||
# 服务器
|
||||
HOST = os.environ.get("CORPUS_HOST", "0.0.0.0")
|
||||
PORT = int(os.environ.get("CORPUS_PORT", "8084"))
|
||||
|
||||
# Portal Chat-v2 API（广州服务器本地端口）
|
||||
PORTAL_CHAT_URL = os.environ.get("PORTAL_CHAT_URL", "http://127.0.0.1:3000/api/chat-v2")
|
||||
|
||||
# 对话记忆最大轮数
|
||||
MAX_MEMORY_ROUNDS = 30
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@ -78,11 +85,67 @@ class CorpusSave(BaseModel):
|
||||
class LoginRequest(BaseModel):
|
||||
gitea_token: str
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
message: str
|
||||
persona: str = "zhuyuan" # zhuyuan | shuangyan
|
||||
engine: str = "deepseek" # deepseek | zhipu | tongyi | huoshan
|
||||
|
||||
class ChatClearRequest(BaseModel):
|
||||
session_id: Optional[str] = None
|
||||
|
||||
class SessionData(BaseModel):
|
||||
username: str
|
||||
login_time: float
|
||||
exprire_at: float
|
||||
|
||||
# ============================================================
|
||||
# 对话记忆管理（每用户30轮）
|
||||
# ============================================================
|
||||
|
||||
class ConversationMemory:
|
||||
"""
|
||||
每用户每 session 的对话记忆
|
||||
支持30轮滚动：超出时从最旧的消息开始丢弃
|
||||
"""
|
||||
|
||||
def __init__(self, max_rounds: int = 30):
|
||||
self.max_rounds = max_rounds
|
||||
self._stores: dict[str, list] = {} # key: "username:session_id" -> [{"role":..., "content":...}, ...]
|
||||
|
||||
def _key(self, username: str, session_id: str = "default") -> str:
|
||||
return f"{username}:{session_id}"
|
||||
|
||||
def get_history(self, username: str, session_id: str = "default") -> list:
|
||||
"""获取对话历史（用于chat-v2 API的history参数）"""
|
||||
key = self._key(username, session_id)
|
||||
return self._stores.get(key, [])
|
||||
|
||||
def add_exchange(self, username: str, user_msg: str, assistant_msg: str, session_id: str = "default"):
|
||||
"""添加一轮对话"""
|
||||
key = self._key(username, session_id)
|
||||
if key not in self._stores:
|
||||
self._stores[key] = []
|
||||
|
||||
self._stores[key].append({"role": "user", "content": user_msg})
|
||||
self._stores[key].append({"role": "assistant", "content": assistant_msg})
|
||||
|
||||
# 超出30轮时裁掉最旧的一对（2条消息）
|
||||
while len(self._stores[key]) > self.max_rounds * 2:
|
||||
self._stores[key] = self._stores[key][2:]
|
||||
|
||||
def clear(self, username: str, session_id: str = "default"):
|
||||
"""清空对话记忆"""
|
||||
key = self._key(username, session_id)
|
||||
self._stores.pop(key, None)
|
||||
|
||||
def get_rounds_count(self, username: str, session_id: str = "default") -> int:
|
||||
"""获取当前对话轮数"""
|
||||
history = self.get_history(username, session_id)
|
||||
return len(history) // 2
|
||||
|
||||
# 全局对话记忆实例
|
||||
chat_memory = ConversationMemory(max_rounds=settings.MAX_MEMORY_ROUNDS)
|
||||
|
||||
# ============================================================
|
||||
# App
|
||||
# ============================================================
|
||||
@ -180,14 +243,29 @@ async def root():
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok", "app": settings.APP_NAME, "version": settings.VERSION}
|
||||
"""健康检查 + Portal chat-v2 探活"""
|
||||
portal_ok = False
|
||||
try:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=3) as client:
|
||||
r = await client.get("http://127.0.0.1:3000/api/health")
|
||||
portal_ok = r.status_code == 200
|
||||
except:
|
||||
pass
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"app": settings.APP_NAME,
|
||||
"version": settings.VERSION,
|
||||
"portal_chat_v2": portal_ok,
|
||||
"active_memories": len(chat_memory._stores),
|
||||
}
|
||||
|
||||
# --- 登录 ---
|
||||
|
||||
@app.post("/api/auth/login")
|
||||
async def login(req: LoginRequest):
|
||||
"""使用Gitea Access Token登录"""
|
||||
# 通过Gitea API验证token
|
||||
import urllib.request
|
||||
try:
|
||||
gitea_req = urllib.request.Request(
|
||||
@ -249,7 +327,6 @@ async def collect_chunk(chunk: TextChunk, token: str = Query(...)):
|
||||
if "user" in r and "assistant" in r:
|
||||
sft_samples.append(to_chatml(r["user"], r["assistant"]))
|
||||
else:
|
||||
# 单条文本也存起来
|
||||
sft_samples.append({
|
||||
"messages": [{"role": "user", "content": r["text"]}],
|
||||
"source": r["source"],
|
||||
@ -371,6 +448,204 @@ async def corpus_stats(token: str = Query(...)):
|
||||
"by_tag": by_tag,
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 对话Agent API（通过Portal Chat-v2连接商业模型）
|
||||
# ============================================================
|
||||
|
||||
# SSE事件类型常量
|
||||
SSE_TYPE_PERSONA = "persona-loaded"
|
||||
SSE_TYPE_TOKEN = "token"
|
||||
SSE_TYPE_TOOL_CALL = "tool-call"
|
||||
SSE_TYPE_TOOL_RESULT = "tool-result"
|
||||
SSE_TYPE_TOOL_UNAVAILABLE = "tool-unavailable"
|
||||
SSE_TYPE_DONE = "done"
|
||||
|
||||
@app.post("/api/corpus/chat")
|
||||
async def chat_with_agent(req: ChatRequest, token: str = Query(...)):
|
||||
"""
|
||||
对话Agent API（SSE流式）
|
||||
|
||||
通过 Portal Chat-v2 API 连接商业模型（DeepSeek/智谱/通义/火山）
|
||||
自动携带30轮对话记忆
|
||||
返回 SSE 流式响应
|
||||
"""
|
||||
username = get_user_from_token(token)
|
||||
if not username:
|
||||
raise HTTPException(401, "登录已过期")
|
||||
|
||||
if not req.message.strip():
|
||||
raise HTTPException(400, "消息不能为空")
|
||||
|
||||
# 获取对话历史
|
||||
history = chat_memory.get_history(username)
|
||||
rounds = chat_memory.get_rounds_count(username)
|
||||
|
||||
async def event_stream():
|
||||
full_response = ""
|
||||
|
||||
try:
|
||||
import httpx
|
||||
|
||||
# 调用 Portal Chat-v2 API
|
||||
payload = {
|
||||
"persona": req.persona,
|
||||
"engine": req.engine,
|
||||
"message": req.message,
|
||||
"history": history,
|
||||
"user": username,
|
||||
"isTemp": False,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
settings.PORTAL_CHAT_URL,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
) as resp:
|
||||
|
||||
if resp.status_code != 200:
|
||||
error_body = await resp.aread()
|
||||
error_text = error_body.decode()[:500]
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': f'API错误 ({resp.status_code}): {error_text}'})}\n\n"
|
||||
yield f"data: {json.dumps({'type': 'done'})}\n\n"
|
||||
return
|
||||
|
||||
# 流式解析 SSE
|
||||
buffer = ""
|
||||
async for chunk in resp.aiter_text():
|
||||
buffer += chunk
|
||||
|
||||
while "\n\n" in buffer:
|
||||
line, buffer = buffer.split("\n\n", 1)
|
||||
line = line.strip()
|
||||
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# 解析 data: 前缀
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:]
|
||||
|
||||
# 处理 [DONE] 标记
|
||||
if data_str.strip() == "[DONE]":
|
||||
# 保存对话到记忆
|
||||
chat_memory.add_exchange(
|
||||
username, req.message, full_response
|
||||
)
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_DONE, 'rounds': rounds + 1, 'max_rounds': settings.MAX_MEMORY_ROUNDS})}\n\n"
|
||||
return
|
||||
|
||||
try:
|
||||
data = json.loads(data_str)
|
||||
event_type = data.get("type", "")
|
||||
|
||||
# 转发 persona-loaded 事件
|
||||
if event_type == SSE_TYPE_PERSONA:
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_PERSONA, 'persona': data.get('persona', ''), 'fromDB': data.get('fromDB', False)})}\n\n"
|
||||
|
||||
# 转发 token 事件
|
||||
elif event_type == SSE_TYPE_TOKEN:
|
||||
token_text = data.get("token", "")
|
||||
full_response += token_text
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_TOKEN, 'token': token_text})}\n\n"
|
||||
|
||||
# 转发工具调用事件
|
||||
elif event_type == SSE_TYPE_TOOL_CALL:
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_TOOL_CALL, 'tool': data.get('tool', ''), 'name': data.get('name', '')})}\n\n"
|
||||
|
||||
elif event_type == SSE_TYPE_TOOL_RESULT:
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_TOOL_RESULT, 'tool': data.get('tool', ''), 'result': str(data.get('result', ''))[:200]})}\n\n"
|
||||
|
||||
elif event_type == SSE_TYPE_TOOL_UNAVAILABLE:
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_TOOL_UNAVAILABLE, 'tool': data.get('tool', '')})}\n\n"
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# 流结束但未收到 [DONE] - 仍然保存
|
||||
if full_response:
|
||||
chat_memory.add_exchange(username, req.message, full_response)
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_DONE, 'rounds': rounds + 1, 'max_rounds': settings.MAX_MEMORY_ROUNDS})}\n\n"
|
||||
|
||||
except httpx.ConnectError:
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': '无法连接到对话引擎(127.0.0.1:3000)，请确认Portal服务已启动'})}\n\n"
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_DONE})}\n\n"
|
||||
except httpx.TimeoutException:
|
||||
# 超时但仍可能有部分回复
|
||||
if full_response:
|
||||
chat_memory.add_exchange(username, req.message, full_response)
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': '对话引擎响应超时，已保存部分回复'})}\n\n"
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_DONE, 'rounds': rounds + 1, 'max_rounds': settings.MAX_MEMORY_ROUNDS, 'partial': True})}\n\n"
|
||||
except Exception as e:
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': f'对话错误: {str(e)}'})}\n\n"
|
||||
yield f"data: {json.dumps({'type': SSE_TYPE_DONE})}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
event_stream(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
)
|
||||
|
||||
@app.get("/api/corpus/chat/history")
|
||||
async def get_chat_history(token: str = Query(...)):
|
||||
"""获取当前对话历史"""
|
||||
username = get_user_from_token(token)
|
||||
if not username:
|
||||
raise HTTPException(401, "登录已过期")
|
||||
|
||||
history = chat_memory.get_history(username)
|
||||
rounds = chat_memory.get_rounds_count(username)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"username": username,
|
||||
"rounds": rounds,
|
||||
"max_rounds": settings.MAX_MEMORY_ROUNDS,
|
||||
"history": history,
|
||||
}
|
||||
|
||||
@app.post("/api/corpus/chat/clear")
|
||||
async def clear_chat_history(token: str = Query(...)):
|
||||
"""清空对话记忆"""
|
||||
username = get_user_from_token(token)
|
||||
if not username:
|
||||
raise HTTPException(401, "登录已过期")
|
||||
|
||||
chat_memory.clear(username)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"message": "对话记忆已清空",
|
||||
}
|
||||
|
||||
@app.get("/api/corpus/chat/models")
|
||||
async def get_available_models(token: str = Query(...)):
|
||||
"""获取可用的模型和人格体列表"""
|
||||
username = get_user_from_token(token)
|
||||
if not username:
|
||||
raise HTTPException(401, "登录已过期")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"personas": [
|
||||
{"id": "zhuyuan", "name": "铸渊", "desc": "现实执行层 · 系统决策、执行规划、技术实现"},
|
||||
{"id": "shuangyan", "name": "霜砚", "desc": "语言主控架构层 · 理解语言、组织表达、控制输出质量"},
|
||||
],
|
||||
"engines": [
|
||||
{"id": "deepseek", "name": "DeepSeek", "model": "deepseek-v4-pro", "provider": "DeepSeek"},
|
||||
{"id": "zhipu", "name": "智谱清言", "model": "glm-4-plus", "provider": "智谱AI"},
|
||||
{"id": "tongyi", "name": "通义千问", "model": "qwen-max", "provider": "阿里云"},
|
||||
{"id": "huoshan", "name": "火山引擎", "model": "doubao-pro-32k", "provider": "字节跳动"},
|
||||
],
|
||||
"current_rounds": chat_memory.get_rounds_count(username),
|
||||
"max_rounds": settings.MAX_MEMORY_ROUNDS,
|
||||
}
|
||||
|
||||
# --- 统计数据 ---
|
||||
|
||||
@app.get("/api/system/stats")
|
||||
@ -386,6 +661,7 @@ async def system_stats():
|
||||
"total_users": total_users,
|
||||
"total_samples": total_samples,
|
||||
"active_sessions": len(sessions),
|
||||
"active_chat_memories": len(chat_memory._stores),
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
@ -499,6 +775,8 @@ if __name__ == "__main__":
|
||||
print(f"🚀 {settings.APP_NAME} v{settings.VERSION}")
|
||||
print(f"📂 数据目录: {settings.DATA_DIR.absolute()}")
|
||||
print(f"🌐 服务地址: http://{settings.HOST}:{settings.PORT}")
|
||||
print(f"🤖 对话引擎: {settings.PORTAL_CHAT_URL}")
|
||||
print(f"💬 最大记忆轮数: {settings.MAX_MEMORY_ROUNDS}")
|
||||
|
||||
settings.USERS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@ -10,35 +10,39 @@
|
||||
.container { max-width: 1000px; margin: 0 auto; padding: 20px; }
|
||||
|
||||
/* Header */
|
||||
.header { background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); color: white; padding: 40px 20px; text-align: center; border-radius: 12px; margin-bottom: 24px; }
|
||||
.header h1 { font-size: 28px; margin-bottom: 8px; }
|
||||
.header p { color: #8899aa; font-size: 14px; }
|
||||
.header { background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); color: white; padding: 32px 20px; text-align: center; border-radius: 12px; margin-bottom: 24px; }
|
||||
.header h1 { font-size: 26px; margin-bottom: 6px; }
|
||||
.header p { color: #8899aa; font-size: 13px; }
|
||||
.header .version { color: #556; font-size: 11px; margin-top: 4px; }
|
||||
|
||||
/* Cards */
|
||||
.card { background: white; border-radius: 12px; padding: 24px; margin-bottom: 20px; box-shadow: 0 2px 8px rgba(0,0,0,0.06); }
|
||||
.card h2 { font-size: 18px; margin-bottom: 16px; color: #1a1a2e; }
|
||||
|
||||
/* Login / User area */
|
||||
/* Login */
|
||||
.login-box { text-align: center; padding: 40px; }
|
||||
.login-box input { padding: 12px 16px; width: 300px; border: 2px solid #e0e0e0; border-radius: 8px; font-size: 14px; margin-bottom: 12px; }
|
||||
.login-box input { padding: 12px 16px; width: 320px; border: 2px solid #e0e0e0; border-radius: 8px; font-size: 14px; margin-bottom: 12px; }
|
||||
.login-box input:focus { outline: none; border-color: #4a6cf7; }
|
||||
.btn { padding: 10px 24px; border: none; border-radius: 8px; font-size: 14px; cursor: pointer; transition: 0.2s; font-weight: 500; }
|
||||
.btn { padding: 10px 24px; border: none; border-radius: 8px; font-size: 14px; cursor: pointer; transition: 0.2s; font-weight: 500; display: inline-flex; align-items: center; gap: 6px; }
|
||||
.btn-primary { background: #4a6cf7; color: white; }
|
||||
.btn-primary:hover { background: #3b5de7; }
|
||||
.btn-success { background: #10b981; color: white; }
|
||||
.btn-success:hover { background: #059669; }
|
||||
.btn-danger { background: #ef4444; color: white; }
|
||||
.btn-danger:hover { background: #dc2626; }
|
||||
.btn-outline { background: transparent; color: #666; border: 1px solid #d0d0d0; }
|
||||
.btn-outline:hover { background: #f5f5f5; }
|
||||
.btn-sm { padding: 6px 14px; font-size: 12px; }
|
||||
.btn:disabled { opacity: 0.6; cursor: not-allowed; }
|
||||
|
||||
.user-info { display: flex; justify-content: space-between; align-items: center; }
|
||||
.user-info .badge { background: #e8f0fe; color: #4a6cf7; padding: 4px 12px; border-radius: 20px; font-size: 13px; }
|
||||
|
||||
/* Stats grid */
|
||||
.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 12px; margin: 16px 0; }
|
||||
.stat-card { background: #f8f9fc; border-radius: 8px; padding: 16px; text-align: center; }
|
||||
.stat-card .num { font-size: 28px; font-weight: 700; color: #1a1a2e; }
|
||||
.stat-card .label { font-size: 12px; color: #8899aa; margin-top: 4px; }
|
||||
/* Stats */
|
||||
.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); gap: 10px; margin: 16px 0; }
|
||||
.stat-card { background: #f8f9fc; border-radius: 8px; padding: 14px; text-align: center; }
|
||||
.stat-card .num { font-size: 24px; font-weight: 700; color: #1a1a2e; }
|
||||
.stat-card .label { font-size: 11px; color: #8899aa; margin-top: 4px; }
|
||||
|
||||
/* Sample list */
|
||||
.sample-list { max-height: 400px; overflow-y: auto; }
|
||||
@ -50,36 +54,74 @@
|
||||
.sample-item .tag { display: inline-block; background: #e8f0fe; color: #4a6cf7; padding: 2px 8px; border-radius: 10px; font-size: 11px; margin-right: 4px; }
|
||||
|
||||
/* Tabs */
|
||||
.tabs { display: flex; gap: 8px; margin-bottom: 16px; }
|
||||
.tab { padding: 8px 20px; border-radius: 8px; cursor: pointer; font-size: 14px; background: #f0f2f5; color: #666; transition: 0.2s; border: none; }
|
||||
.tabs { display: flex; gap: 6px; margin-bottom: 16px; flex-wrap: wrap; }
|
||||
.tab { padding: 8px 18px; border-radius: 8px; cursor: pointer; font-size: 13px; background: #f0f2f5; color: #666; transition: 0.2s; border: none; }
|
||||
.tab.active { background: #4a6cf7; color: white; }
|
||||
.tab:hover:not(.active) { background: #e0e2e5; }
|
||||
|
||||
.hidden { display: none; }
|
||||
.hidden { display: none !important; }
|
||||
|
||||
/* Preview / input area */
|
||||
/* Chat Styles */
|
||||
.chat-container { display: flex; flex-direction: column; height: 550px; }
|
||||
.chat-messages { flex: 1; overflow-y: auto; padding: 16px; background: #f8f9fc; border-radius: 8px 8px 0 0; border: 1px solid #e8e8e8; border-bottom: none; }
|
||||
.chat-message { margin-bottom: 16px; animation: fadeIn 0.2s ease; }
|
||||
@keyframes fadeIn { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: translateY(0); } }
|
||||
.chat-message .role { font-size: 11px; font-weight: 600; color: #8899aa; margin-bottom: 4px; display: flex; align-items: center; gap: 6px; }
|
||||
.chat-message .role .dot { width: 6px; height: 6px; border-radius: 50%; display: inline-block; }
|
||||
.chat-message .role .dot.user { background: #4a6cf7; }
|
||||
.chat-message .role .dot.assistant { background: #10b981; }
|
||||
.chat-message .role .dot.system { background: #f59e0b; }
|
||||
.chat-message .bubble { padding: 10px 14px; border-radius: 8px; font-size: 14px; line-height: 1.7; max-width: 85%; }
|
||||
.chat-message.user { text-align: right; }
|
||||
.chat-message.user .bubble { background: #4a6cf7; color: white; display: inline-block; text-align: left; }
|
||||
.chat-message.assistant .bubble { background: white; border: 1px solid #e8e8e8; display: inline-block; }
|
||||
.chat-message.system { text-align: center; }
|
||||
.chat-message.system .bubble { background: #fef3c7; color: #92400e; font-size: 13px; display: inline-block; max-width: 90%; }
|
||||
.chat-message .typing-indicator { display: inline-flex; gap: 4px; padding: 4px 0; }
|
||||
.chat-message .typing-indicator span { width: 8px; height: 8px; background: #bbb; border-radius: 50%; animation: typing 1.2s infinite ease-in-out; }
|
||||
.chat-message .typing-indicator span:nth-child(2) { animation-delay: 0.2s; }
|
||||
.chat-message .typing-indicator span:nth-child(3) { animation-delay: 0.4s; }
|
||||
@keyframes typing { 0%, 80%, 100% { transform: scale(0.6); } 40% { transform: scale(1); } }
|
||||
|
||||
.chat-input-area { display: flex; gap: 8px; padding: 12px; background: white; border: 1px solid #e8e8e8; border-top: none; border-radius: 0 0 8px 8px; }
|
||||
.chat-input-area input { flex: 1; padding: 10px 14px; border: 2px solid #e0e0e0; border-radius: 8px; font-size: 14px; }
|
||||
.chat-input-area input:focus { outline: none; border-color: #4a6cf7; }
|
||||
|
||||
.chat-controls { display: flex; gap: 8px; align-items: center; }
|
||||
.chat-controls select { padding: 8px 10px; border: 1px solid #d0d0d0; border-radius: 6px; font-size: 12px; background: white; color: #555; }
|
||||
.chat-rounds { font-size: 11px; color: #999; text-align: center; padding: 6px; }
|
||||
|
||||
/* Preview */
|
||||
.preview-area { background: #f8f9fc; border-radius: 8px; padding: 16px; margin: 12px 0; font-size: 13px; line-height: 1.6; max-height: 300px; overflow-y: auto; }
|
||||
.preview-area .valuable { border-left: 3px solid #10b981; padding-left: 12px; margin: 8px 0; }
|
||||
.preview-area .filtered { border-left: 3px solid #ef4444; padding-left: 12px; margin: 8px 0; opacity: 0.5; }
|
||||
|
||||
/* Mac client download */
|
||||
.download-section { text-align: center; padding: 20px; }
|
||||
/* Client download */
|
||||
.download-section { text-align: center; padding: 10px; }
|
||||
.download-section p { color: #666; font-size: 13px; margin: 8px 0; }
|
||||
code { background: #f0f2f5; padding: 2px 6px; border-radius: 4px; font-size: 12px; }
|
||||
|
||||
/* Footer */
|
||||
.footer { text-align: center; color: #999; font-size: 12px; padding: 20px 0; }
|
||||
.footer { text-align: center; color: #999; font-size: 11px; padding: 16px 0; }
|
||||
|
||||
/* Empty state */
|
||||
.empty-state { text-align: center; padding: 40px; color: #999; }
|
||||
.empty-state .icon { font-size: 48px; margin-bottom: 12px; }
|
||||
|
||||
/* Scrollbar */
|
||||
::-webkit-scrollbar { width: 6px; }
|
||||
::-webkit-scrollbar-track { background: transparent; }
|
||||
::-webkit-scrollbar-thumb { background: #ccc; border-radius: 3px; }
|
||||
::-webkit-scrollbar-thumb:hover { background: #aaa; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container" id="app">
|
||||
<!-- Header -->
|
||||
<div class="header">
|
||||
<h1>🧠 语料采集系统</h1>
|
||||
<p>Corpus Agent · 自动采集 · 脱敏 · 格式化</p>
|
||||
<h1>🧠 语料采集系统 · Corpus Agent</h1>
|
||||
<p>自动采集 · 脱敏 · 格式化 · 对话式语料分析</p>
|
||||
<div class="version">v1.0.1 · 国作登字-2026-A-00037559</div>
|
||||
</div>
|
||||
|
||||
<!-- Login -->
|
||||
@ -98,7 +140,7 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Main (after login) -->
|
||||
<!-- Main -->
|
||||
<div id="main-section" class="hidden">
|
||||
<!-- User info -->
|
||||
<div class="card">
|
||||
@ -128,19 +170,20 @@
|
||||
|
||||
<!-- Tabs -->
|
||||
<div class="tabs">
|
||||
<button class="tab active" onclick="switchTab('collection')">📥 实时采集</button>
|
||||
<button class="tab" onclick="switchTab('browse')">📚 语料浏览</button>
|
||||
<button class="tab" onclick="switchTab('client')">💻 Mac 客户端</button>
|
||||
<button class="tab active" onclick="switchTab('collection', this)">📥 实时采集</button>
|
||||
<button class="tab" onclick="switchTab('browse', this)">📚 语料浏览</button>
|
||||
<button class="tab" onclick="switchTab('client', this)">💻 Mac 客户端</button>
|
||||
<button class="tab" onclick="switchTab('chat', this); if(!chatInitialized)initChat()">🤖 语料对话</button>
|
||||
</div>
|
||||
|
||||
<!-- Tab: 实时采集 -->
|
||||
<div class="card" id="tab-collection">
|
||||
<h2>📥 实时采集</h2>
|
||||
<p style="color:#666; font-size:13px; margin-bottom:12px">粘贴对话文本，引擎自动判断价值并脱敏存储</p>
|
||||
<textarea id="input-text" style="width:100%;height:120px;padding:12px;border:2px solid #e0e0e0;border-radius:8px;font-size:13px;resize:vertical" placeholder="粘贴聊天内容..."></textarea>
|
||||
<textarea id="input-text" style="width:100%;height:120px;padding:12px;border:2px solid #e0e0e0;border-radius:8px;font-size:13px;resize:vertical" placeholder="粘贴聊天内容（技术讨论、踩坑记录、架构决策等会被自动识别）"></textarea>
|
||||
<div style="margin-top:12px">
|
||||
<button class="btn btn-primary" onclick="collectText()">采集</button>
|
||||
<button class="btn btn-sm" style="margin-left:8px" onclick="document.getElementById('input-text').value=''">清空</button>
|
||||
<button class="btn btn-outline btn-sm" style="margin-left:8px" onclick="document.getElementById('input-text').value=''">清空</button>
|
||||
</div>
|
||||
<div id="collect-result" class="hidden preview-area"></div>
|
||||
</div>
|
||||
@ -170,9 +213,9 @@
|
||||
<div style="text-align:left;max-width:600px;margin:20px auto">
|
||||
<h3 style="font-size:14px;margin-bottom:12px">📋 安装步骤</h3>
|
||||
<ol style="font-size:13px;color:#666;line-height:2">
|
||||
<li>下载 <code>mac-corpus-agent.py</code> 到你的 Mac</li>
|
||||
<li>下载 <code>mac_client.py</code> 到你的 Mac</li>
|
||||
<li>安装依赖：<code>pip3 install pyautogui pillow websocket-client</code></li>
|
||||
<li>运行：<code>python3 mac-corpus-agent.py --token YOUR_TOKEN</code></li>
|
||||
<li>运行：<code>python3 mac_client.py --token YOUR_TOKEN</code></li>
|
||||
<li>选择模式：<code>--mode auto</code>（实时）或 <code>--mode scroll</code>（滚屏）</li>
|
||||
</ol>
|
||||
|
||||
@ -198,11 +241,54 @@
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Tab: 语料对话 -->
|
||||
<div class="card hidden" id="tab-chat">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px">
|
||||
<h2 style="margin:0">🤖 语料助手对话</h2>
|
||||
<div style="display:flex;gap:8px;align-items:center">
|
||||
<span id="chat-rounds-display" style="font-size:12px;color:#999">0/30 轮</span>
|
||||
<button class="btn btn-sm btn-outline" onclick="clearChat()">🗑️ 清空记忆</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chat-container">
|
||||
<div class="chat-messages" id="chat-messages">
|
||||
<div class="chat-message system">
|
||||
<div class="bubble">
|
||||
👋 你好！我是语料助手，基于 DeepSeek/智谱/通义/火山 引擎驱动。<br>
|
||||
我可以帮你分析语料、讨论技术方案、解答训练相关的问题。<br>
|
||||
<span style="color:#999;font-size:12px">当前使用「铸渊」人格 + DeepSeek 引擎 · 支持30轮连续对话</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chat-controls" style="padding:8px 12px 0;display:flex;gap:8px;flex-wrap:wrap">
|
||||
<select id="chat-persona">
|
||||
<option value="zhuyuan">🧠 铸渊 · 执行层</option>
|
||||
<option value="shuangyan">🗣️ 霜砚 · 语言层</option>
|
||||
</select>
|
||||
<select id="chat-engine">
|
||||
<option value="deepseek">🔷 DeepSeek V4 Pro</option>
|
||||
<option value="zhipu">🟢 智谱 GLM-4-Plus</option>
|
||||
<option value="tongyi">🔴 通义 Qwen-Max</option>
|
||||
<option value="huoshan">🟠 火山 Doubao-Pro</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="chat-input-area">
|
||||
<input type="text" id="chat-input" placeholder="输入你的问题..." onkeydown="if(event.key==='Enter' && !event.shiftKey){event.preventDefault();sendChatMessage()}">
|
||||
<button class="btn btn-primary" id="chat-send-btn" onclick="sendChatMessage()">发送</button>
|
||||
<button class="btn btn-outline btn-sm hidden" id="chat-stop-btn" onclick="stopChatStream()" style="background:#ef4444;color:white;border:none">■ 停止</button>
|
||||
</div>
|
||||
<div class="chat-rounds" id="chat-rounds-bar">对话将自动保留最近30轮上下文</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div class="footer">
|
||||
语料采集系统 v1.0 · 国作登字-2026-A-00037559
|
||||
语料采集系统 v1.0.1 · 铸渊 · ICE-GL-ZY001 · 国作登字-2026-A-00037559
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -210,7 +296,11 @@
|
||||
let token = localStorage.getItem('corpus_token') || '';
|
||||
let username = '';
|
||||
const API = '';
|
||||
let chatInitialized = false;
|
||||
let chatAbortController = null;
|
||||
let isStreaming = false;
|
||||
|
||||
// ─── API Helper ─────────────────────────────────────────────
|
||||
async function api(method, path, body) {
|
||||
const url = `${API}${path}${path.includes('?') ? '&' : '?'}token=${token}`;
|
||||
const opts = { method, headers: {'Content-Type': 'application/json'} };
|
||||
@ -219,7 +309,7 @@ async function api(method, path, body) {
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
// --- Login ---
|
||||
// ─── Login ─────────────────────────────────────────────────
|
||||
async function login() {
|
||||
const t = document.getElementById('token-input').value.trim();
|
||||
if (!t) return alert('请输入Token');
|
||||
@ -248,7 +338,7 @@ function logout() {
|
||||
document.getElementById('main-section').classList.add('hidden');
|
||||
}
|
||||
|
||||
// Auto login on load
|
||||
// Auto login
|
||||
if (token) {
|
||||
api('GET', '/api/auth/check').then(d => {
|
||||
if (d.ok) {
|
||||
@ -264,15 +354,15 @@ if (token) {
|
||||
});
|
||||
}
|
||||
|
||||
// --- Tabs ---
|
||||
function switchTab(name) {
|
||||
// ─── Tabs ──────────────────────────────────────────────────
|
||||
function switchTab(name, btn) {
|
||||
document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
|
||||
document.querySelectorAll('[id^="tab-"]').forEach(t => t.classList.add('hidden'));
|
||||
event.target.classList.add('active');
|
||||
if (btn) btn.classList.add('active');
|
||||
document.getElementById('tab-' + name).classList.remove('hidden');
|
||||
}
|
||||
|
||||
// --- Collect ---
|
||||
// ─── Collect ───────────────────────────────────────────────
|
||||
async function collectText() {
|
||||
const text = document.getElementById('input-text').value.trim();
|
||||
if (!text) return;
|
||||
@ -289,7 +379,7 @@ async function collectText() {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Stats ---
|
||||
// ─── Stats ─────────────────────────────────────────────────
|
||||
async function loadStats() {
|
||||
const r = await api('GET', '/api/corpus/stats');
|
||||
if (!r.ok) return;
|
||||
@ -299,7 +389,7 @@ async function loadStats() {
|
||||
document.getElementById('sample-count').textContent = (r.total_samples || 0) + ' 条语料';
|
||||
}
|
||||
|
||||
// --- Samples ---
|
||||
// ─── Samples ───────────────────────────────────────────────
|
||||
async function loadSamples() {
|
||||
const r = await api('GET', '/api/corpus/list?page=1&size=50');
|
||||
if (!r.ok) return;
|
||||
@ -323,7 +413,7 @@ async function loadSamples() {
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// --- Export ---
|
||||
// ─── Export ────────────────────────────────────────────────
|
||||
async function exportCorpus() {
|
||||
const r = await fetch(`${API}/api/corpus/export?token=${token}&format=jsonl`);
|
||||
if (r.ok) {
|
||||
@ -335,7 +425,269 @@ async function exportCorpus() {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Chat ──────────────────────────────────────────────────
|
||||
let streamBuffer = '';
|
||||
|
||||
function initChat() {
|
||||
if (chatInitialized) return;
|
||||
chatInitialized = true;
|
||||
loadChatHistory();
|
||||
}
|
||||
|
||||
function addChatMessage(role, content, isHtml) {
|
||||
const container = document.getElementById('chat-messages');
|
||||
const div = document.createElement('div');
|
||||
div.className = 'chat-message ' + role;
|
||||
|
||||
const roleLabels = { user: '你', assistant: '助手', system: '系统' };
|
||||
const dotColors = { user: 'user', assistant: 'assistant', system: 'system' };
|
||||
|
||||
let contentHtml = isHtml ? content : escapeHtml(content);
|
||||
// Convert markdown-style bold and code for assistant messages
|
||||
if (role === 'assistant' && !isHtml) {
|
||||
contentHtml = contentHtml
|
||||
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
|
||||
.replace(/`(.+?)`/g, '<code style="background:#f0f0f0;padding:1px 4px;border-radius:3px;font-size:13px">$1</code>')
|
||||
.replace(/\n/g, '<br>');
|
||||
} else if (role === 'user' && !isHtml) {
|
||||
contentHtml = contentHtml.replace(/\n/g, '<br>');
|
||||
}
|
||||
|
||||
div.innerHTML = `
|
||||
<div class="role"><span class="dot ${dotColors[role]}"></span>${roleLabels[role] || role}</div>
|
||||
<div class="bubble">${contentHtml}</div>
|
||||
`;
|
||||
|
||||
container.appendChild(div);
|
||||
container.scrollTop = container.scrollHeight;
|
||||
return div;
|
||||
}
|
||||
|
||||
function showTyping() {
|
||||
const container = document.getElementById('chat-messages');
|
||||
const div = document.createElement('div');
|
||||
div.className = 'chat-message assistant';
|
||||
div.id = 'typing-indicator';
|
||||
div.innerHTML = `
|
||||
<div class="role"><span class="dot assistant"></span>助手</div>
|
||||
<div class="bubble">
|
||||
<div class="typing-indicator">
|
||||
<span></span><span></span><span></span>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
container.appendChild(div);
|
||||
container.scrollTop = container.scrollHeight;
|
||||
}
|
||||
|
||||
function removeTyping() {
|
||||
const el = document.getElementById('typing-indicator');
|
||||
if (el) el.remove();
|
||||
}
|
||||
|
||||
async function loadChatHistory() {
|
||||
try {
|
||||
const r = await api('GET', '/api/corpus/chat/history');
|
||||
if (r.ok) {
|
||||
document.getElementById('chat-rounds-display').textContent = `${r.rounds}/${r.max_rounds} 轮`;
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
async function sendChatMessage() {
|
||||
if (isStreaming) return;
|
||||
|
||||
const input = document.getElementById('chat-input');
|
||||
const message = input.value.trim();
|
||||
if (!message) return;
|
||||
|
||||
input.value = '';
|
||||
isStreaming = true;
|
||||
|
||||
// Disable send, show stop
|
||||
document.getElementById('chat-send-btn').classList.add('hidden');
|
||||
document.getElementById('chat-stop-btn').classList.remove('hidden');
|
||||
|
||||
// Add user message
|
||||
addChatMessage('user', message);
|
||||
|
||||
// Show typing indicator
|
||||
showTyping();
|
||||
|
||||
// Get config
|
||||
const persona = document.getElementById('chat-persona').value;
|
||||
const engine = document.getElementById('chat-engine').value;
|
||||
|
||||
// Build SSE URL
|
||||
const url = `${API}/api/corpus/chat?token=${token}`;
|
||||
|
||||
try {
|
||||
chatAbortController = new AbortController();
|
||||
streamBuffer = '';
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({
|
||||
message: message,
|
||||
persona: persona,
|
||||
engine: engine
|
||||
}),
|
||||
signal: chatAbortController.signal
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
removeTyping();
|
||||
addChatMessage('system', `❌ 请求失败 (${resp.status})`);
|
||||
isStreaming = false;
|
||||
document.getElementById('chat-send-btn').classList.remove('hidden');
|
||||
document.getElementById('chat-stop-btn').classList.add('hidden');
|
||||
return;
|
||||
}
|
||||
|
||||
// Read SSE stream
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let fullText = '';
|
||||
let hasContent = false;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
// Process complete SSE events
|
||||
while (buffer.includes('\n\n')) {
|
||||
const idx = buffer.indexOf('\n\n');
|
||||
const event = buffer.slice(0, idx);
|
||||
buffer = buffer.slice(idx + 2);
|
||||
|
||||
if (event.startsWith('data: ')) {
|
||||
const dataStr = event.slice(6).trim();
|
||||
|
||||
if (dataStr === '[DONE]') continue;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(dataStr);
|
||||
const type = data.type;
|
||||
|
||||
if (type === 'persona-loaded') {
|
||||
// Remove typing, show persona loaded
|
||||
removeTyping();
|
||||
hasContent = true;
|
||||
// Start a new bubble for the assistant
|
||||
addChatMessage('assistant', '');
|
||||
streamBuffer = '';
|
||||
} else if (type === 'token') {
|
||||
const token = data.token || '';
|
||||
fullText += token;
|
||||
streamBuffer += token;
|
||||
|
||||
if (hasContent) {
|
||||
// Update the last assistant message
|
||||
const container = document.getElementById('chat-messages');
|
||||
const lastMsg = container.lastElementChild;
|
||||
if (lastMsg && lastMsg.classList.contains('assistant')) {
|
||||
const bubble = lastMsg.querySelector('.bubble');
|
||||
if (bubble) {
|
||||
let displayText = fullText
|
||||
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
|
||||
.replace(/`(.+?)`/g, '<code style="background:#f0f0f0;padding:1px 4px;border-radius:3px;font-size:13px">$1</code>')
|
||||
.replace(/\n/g, '<br>');
|
||||
bubble.innerHTML = displayText;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
removeTyping();
|
||||
addChatMessage('assistant', token);
|
||||
fullText = token;
|
||||
streamBuffer = token;
|
||||
hasContent = true;
|
||||
}
|
||||
|
||||
// Auto scroll
|
||||
const container = document.getElementById('chat-messages');
|
||||
container.scrollTop = container.scrollHeight;
|
||||
|
||||
} else if (type === 'tool-call') {
|
||||
addChatMessage('system', `🔧 正在调用工具: ${data.name || data.tool}`);
|
||||
} else if (type === 'tool-result') {
|
||||
// Tool result, continue
|
||||
} else if (type === 'done') {
|
||||
// Finished
|
||||
if (data.rounds) {
|
||||
document.getElementById('chat-rounds-display').textContent = `${data.rounds}/${data.max_rounds || 30} 轮`;
|
||||
}
|
||||
} else if (type === 'error') {
|
||||
removeTyping();
|
||||
addChatMessage('system', `⚠️ ${data.message || '未知错误'}`);
|
||||
}
|
||||
} catch(e) {
|
||||
// Skip malformed JSON
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle leftover buffer
|
||||
if (buffer.startsWith('data: ')) {
|
||||
const dataStr = buffer.slice(6).trim();
|
||||
if (dataStr !== '[DONE]') {
|
||||
try {
|
||||
const data = JSON.parse(dataStr);
|
||||
if (data.type === 'token') {
|
||||
fullText += data.token || '';
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
if (e.name === 'AbortError') {
|
||||
addChatMessage('system', '⏹️ 对话已停止');
|
||||
} else {
|
||||
removeTyping();
|
||||
addChatMessage('system', `❌ 连接错误: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
removeTyping();
|
||||
isStreaming = false;
|
||||
chatAbortController = null;
|
||||
document.getElementById('chat-send-btn').classList.remove('hidden');
|
||||
document.getElementById('chat-stop-btn').classList.add('hidden');
|
||||
}
|
||||
|
||||
function stopChatStream() {
|
||||
if (chatAbortController) {
|
||||
chatAbortController.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async function clearChat() {
|
||||
if (!confirm('确定清空对话记忆吗？')) return;
|
||||
const r = await api('POST', '/api/corpus/chat/clear');
|
||||
if (r.ok) {
|
||||
// Clear messages
|
||||
const container = document.getElementById('chat-messages');
|
||||
container.innerHTML = `
|
||||
<div class="chat-message system">
|
||||
<div class="bubble">
|
||||
👋 记忆已清空，开始新的对话吧！<br>
|
||||
<span style="color:#999;font-size:12px">当前使用「铸渊」人格 + DeepSeek 引擎 · 支持30轮连续对话</span>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
document.getElementById('chat-rounds-display').textContent = '0/30 轮';
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Utilities ─────────────────────────────────────────────
|
||||
function escapeHtml(s) {
|
||||
if (typeof s !== 'string') s = String(s || '');
|
||||
const d = document.createElement('div');
|
||||
d.textContent = s;
|
||||
return d.innerHTML;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user