diff --git a/brain/training-status.json b/brain/training-status.json index 5a42de5..1016b27 100644 --- a/brain/training-status.json +++ b/brain/training-status.json @@ -1,6 +1,6 @@ { - "version": "1.1", - "last_updated": "2026-05-18T21:00+08:00", + "version": "1.2", + "last_updated": "2026-05-18T22:45+08:00", "updated_by": "铸渊 (ICE-GL-ZY001)", "mother_model": { "status": "completed", @@ -27,18 +27,19 @@ }, "distill_shuangyan_15b": { "status": "running", - "status_label": "🔄 蒸馏训练中(PID 82143,step=50 loss=6.66,Epoch 1/3)", - "version": "v6", + "status_label": "🔄 蒸馏训练中(Screen 87779, step=50 loss=6.68, Epoch 1/3)", + "version": "v7", "script": "scripts/distill_mother.py", "teacher": "母模型7B SFT", "student": "Qwen2.5-1.5B-Instruct", "method": "KL散度(TEMP=2.0, ALPHA=0.7) + SFT混合", "host": "connect.westd.seetacloud.com:23647", "ssh_password": "HkM43lFVUIsc", - "started_at": "2026-05-18T20:56+08:00", - "estimated_completion": "~23:30+08:00", + "started_at": "2026-05-18T22:41+08:00", + "estimated_completion": "~01:00+08:00", "cos_output": "models/shuangyan-15b-distill/final/", "fixes": [ + "v7: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + empty_cache在epoch转换/模型加载/每500步", "v6: vocab_size截断 fix — Teacher(7B)=152064 → Student(1.5B)=151936", "v5: HuggingFace forward参数名修正 (ii→input_ids, am→attention_mask)", "v4: num_workers=0 (避免CUDA fork)", @@ -52,20 +53,20 @@ {"name": "代码模型SFT训练", "status": "completed", "label": "✅ 已完成"}, {"name": "上传COS (代码模型)", "status": "completed", "label": "✅ 已完成"}, {"name": "generation_config修复", "status": "completed", "label": "✅ 两个模型已修复"}, - {"name": "霜砚1.5B蒸馏", "status": "running", "label": "🔄 训练中(PID 82143, Epoch 1/3)"}, + {"name": "霜砚1.5B蒸馏", "status": "running", "label": "🔄 训练中(Screen 87779, Epoch 1/3)"}, {"name": "上传COS (蒸馏输出)", "status": "pending", "label": "⚪ 等待蒸馏完成"} ] }, "next_phases": { - "distill_shuangyan_15b": "🔄 训练中,预计~23:30完成", + "distill_shuangyan_15b": "🔄 训练中,预计~01:00完成", "shuangyan_deep_sft": "⏳ 等待蒸馏完成后用霜砚语料深度微调", "distill_coder_to_zhuyuan_15b": "⏳ 等待Phase 1+2完成", "zhuyuan_deep_sft": "⏳ 等待Phase 3完成" }, "alerts": [ "⚠️ 之前误判代码模型已完成(COS上有历史checkpoint-540)", - "⚠️ 蒸馏v1-v5多次崩溃:路径错误→存储爆炸→CUDA fork→参数名错误→vocab大小不匹配", - "⚠️ v6 vocab_size截断修复后训练中,预计~2.6小时完成" + "⚠️ 蒸馏经过7次迭代才稳定:路径错误→存储爆炸→CUDA fork→参数名错误→vocab不匹配→OOM(×2)", + "⚠️ v7 所有已知崩溃已修复,预计~01:00完成" ], "links": { "notion": "https://www.notion.so/363fb92f383181029fafef8438d6c300"