bingshuo/guanghulab
1
0
Fork 0
Code Issues 2 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

D106 training status — 蒸馏双Track + pipeline + 网站修复

Browse Source
This commit is contained in:
bingshuo 2026-05-20 09:49:25 +08:00
parent 8647d17b47
commit 8ad3bf4923
1 changed files with 46 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
brain/training-status.json
View File

@ -1,74 +1,76 @@
{ {
"version": "1.2", "version": "1.3",
"last_updated": "2026-05-18T22:45+08:00", "last_updated": "2026-05-20T09:48+08:00",
"updated_by": "铸渊 (ICE-GL-ZY001)", "updated_by": "铸渊 (ICE-GL-ZY001)",
"mother_model": { "mother_model": {
"status": "completed", "status": "completed",
"status_label": "✅ 已完成（COS文件已修复 generation_config.json）", "status_label": "✅ 已完成 · 已上传COS bingshuo-1317346199",
"model": "Qwen/Qwen2.5-7B", "model": "Qwen/Qwen2.5-7B",
"final_loss": 0.2638, "final_loss": 1.5229,
"final_epoch": 3, "final_epoch": 3,
"total_epochs": 3, "total_epochs": 3,
"cos_path": "cos://sy-finetune-corpus-1317346199/models/qwen25-7b-sft/final/", "cos_path": "cos://bingshuo-1317346199/models/qwen25-7b-sft/final/",
"completed_at": "2026-05-18T~06:00+08:00", "completed_at": "2026-05-19T~22:30+08:00",
"fixes": { "fixes": {
"generation_config_eos": "151643→151645 ✅ 2026-05-18 修复（根因：model.generate()读取generation_config而非config）", "generation_config_eos": "151645 ✅",
"default_system": "已清除 ✅" "default_system": "已清除 ✅"
} }
}, },
"code_model": { "code_model": {
"status": "completed", "status": "completed",
"status_label": "✅ 已完成（train_coder.log确认DONE!）", "status_label": "✅ 已完成 · 已上传COS bingshuo-1317346199",
"model": "Qwen/Qwen2.5-Coder-7B", "model": "Qwen/Qwen2.5-Coder-7B",
"cos_path": "cos://sy-finetune-corpus-1317346199/models/qwen25-coder-7b-sft/final/", "final_loss": 1.822,
"fixes": { "cos_path": "cos://bingshuo-1317346199/models/qwen25-coder-7b-sft/final/",
"generation_config_eos": "151643→151645 ✅ 2026-05-18 修复" "fixes": {}
}
}, },
"distill_shuangyan_15b": { "distill_shuangyan_15b": {
"status": "running", "status": "running",
"status_label": "🔄 蒸馏训练中（Screen 87779, step=50 loss=6.68, Epoch 1/3）", "status_label": "🔴 蒸馏训练中 · Ep1/3 step~4850/15781 loss≈1.96",
"version": "v7", "version": "v7",
"script": "scripts/distill_mother.py", "script": "scripts/distill_mother_v7.py",
"teacher": "母模型7B SFT", "teacher": "母模型7B SFT (COS bingshuo)",
"student": "Qwen2.5-1.5B-Instruct", "student": "Qwen2.5-1.5B-Instruct",
"method": "KL散度(TEMP=2.0, ALPHA=0.7) + SFT混合", "params": "B=2, GA=16, LR=1e-5, TEMP=2.0, ALPHA=0.7",
"host": "connect.westd.seetacloud.com:23647", "host": "connect.westd.seetacloud.com:23647 (RTX PRO 6000 96GB)",
"ssh_password": "HkM43lFVUIsc", "started_at": "2026-05-20T09:23+08:00",
"started_at": "2026-05-18T22:41+08:00", "estimated_completion": "~12:00~13:00+08:00 (3 epoch, ~45min/ep)",
"estimated_completion": "~01:00+08:00", "cos_output": "models/shuangyan-15b-distill/",
"cos_output": "models/shuangyan-15b-distill/final/", "pipeline_watchdog": "/tmp/distill_pipeline.sh (PID 581561, 自动触发Track2)"
"fixes": [ },
"v7: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + empty_cache在epoch转换/模型加载/每500步", "distill_coder_15b": {
"v6: vocab_size截断 fix — Teacher(7B)=152064 → Student(1.5B)=151936", "status": "pending",
"v5: HuggingFace forward参数名修正 (ii→input_ids, am→attention_mask)", "status_label": "⏳ 等待Track1完成 + 管道自动触发",
"v4: num_workers=0 (避免CUDA fork)", "script": "scripts/distill_coder_v2.py",
"v5: 移除teacher_logits预存储改为即时forward" "teacher": "代码模型7B SFT (COS bingshuo)",
] "student": "Qwen2.5-Coder-1.5B-Instruct",
"params": "B=2, GA=16, LR=1e-5 (预修OOM)",
"cos_output": "models/qwen25-15b-coder-distill/"
}, },
"pipeline": { "pipeline": {
"steps": [ "steps": [
{"name": "母模型SFT训练", "status": "completed", "label": "✅ 完成 (loss=0.2638)"}, {"name": "母模型SFT训练+上传COS", "status": "completed", "label": "✅"},
{"name": "上传COS (母模型)", "status": "completed", "label": "✅ 已完成"}, {"name": "代码模型SFT训练+上传COS", "status": "completed", "label": "✅"},
{"name": "代码模型SFT训练", "status": "completed", "label": "✅ 已完成"}, {"name": "Track1 母模型→霜砚1.5B蒸馏", "status": "running", "label": "🔄 Ep1/3 step~4850/15781"},
{"name": "上传COS (代码模型)", "status": "completed", "label": "✅ 已完成"}, {"name": "Track1 上传COS", "status": "pending", "label": "⏳ 蒸馏自带上传"},
{"name": "generation_config修复", "status": "completed", "label": "✅ 两个模型已修复"}, {"name": "Track2 代码模型→铸渊1.5B蒸馏", "status": "pending", "label": "⏳ 管道自动触发"},
{"name": "霜砚1.5B蒸馏", "status": "running", "label": "🔄 训练中（Screen 87779, Epoch 1/3）"}, {"name": "Track2 上传COS", "status": "pending", "label": "⏳"}
{"name": "上传COS (蒸馏输出)", "status": "pending", "label": "⚪ 等待蒸馏完成"}
] ]
}, },
"next_phases": { "website_status": {
"distill_shuangyan_15b": "🔄 训练中，预计~01:00完成", "login_api": "✅ 正常",
"shuangyan_deep_sft": "⏳ 等待蒸馏完成后用霜砚语料深度微调", "training_panel": "✅ 实时显示 (fetch_train.py v4)",
"distill_coder_to_zhuyuan_15b": "⏳ 等待Phase 1+2完成", "js_all_functions": "✅ 已修复 (setInterval恢复)",
"zhuyuan_deep_sft": "⏳ 等待Phase 3完成" "gpu_monitoring": "✅ training-status.json每分钟更新"
}, },
"alerts": [ "alerts": [
"⚠️ 之前误判代码模型已完成（COS上有历史checkpoint-540）", "⚠️ 本次修复了fetch_train.py v3的3个bug（phase检测/epoch提取/tail长度）",
"⚠️ 蒸馏经过7次迭代才稳定：路径错误→存储爆炸→CUDA fork→参数名错误→vocab不匹配→OOM(×2)", "⚠️ 前一次index.html修复(3583413)引入语法错误→全站JS崩溃→已修复",
"⚠️ v7 所有已知崩溃已修复，预计~01:00完成" "⚠️ Track2的OOM已在Track1完成前预修",
"⚠️ 冰朔强调的工作流程：先读仓库记忆→理解全局→再动手"
], ],
"links": { "links": {
"notion": "https://www.notion.so/363fb92f383181029fafef8438d6c300" "notion": "https://www.notion.so/363fb92f383181029fafef8438d6c300",
"repo": "https://guanghulab.com/code/bingshuo/guanghulab"
} }
} }