guanghulab/brain/training-status.json

{
  "version": "1.2",
  "last_updated": "2026-05-18T22:45+08:00",
  "updated_by": "铸渊 (ICE-GL-ZY001)",
  "mother_model": {
    "status": "completed",
    "status_label": "✅ 已完成（COS文件已修复 generation_config.json）",
    "model": "Qwen/Qwen2.5-7B",
    "final_loss": 0.2638,
    "final_epoch": 3,
    "total_epochs": 3,
    "cos_path": "cos://sy-finetune-corpus-1317346199/models/qwen25-7b-sft/final/",
    "completed_at": "2026-05-18T~06:00+08:00",
    "fixes": {
      "generation_config_eos": "151643→151645 ✅ 2026-05-18 修复（根因：model.generate()读取generation_config而非config）",
      "default_system": "已清除 ✅"
    }
  },
  "code_model": {
    "status": "completed",
    "status_label": "✅ 已完成（train_coder.log确认DONE!）",
    "model": "Qwen/Qwen2.5-Coder-7B",
    "cos_path": "cos://sy-finetune-corpus-1317346199/models/qwen25-coder-7b-sft/final/",
    "fixes": {
      "generation_config_eos": "151643→151645 ✅ 2026-05-18 修复"
    }
  },
  "distill_shuangyan_15b": {
    "status": "running",
    "status_label": "🔄 蒸馏训练中（Screen 87779, step=50 loss=6.68, Epoch 1/3）",
    "version": "v7",
    "script": "scripts/distill_mother.py",
    "teacher": "母模型7B SFT",
    "student": "Qwen2.5-1.5B-Instruct",
    "method": "KL散度(TEMP=2.0, ALPHA=0.7) + SFT混合",
    "host": "connect.westd.seetacloud.com:23647",
    "ssh_password": "HkM43lFVUIsc",
    "started_at": "2026-05-18T22:41+08:00",
    "estimated_completion": "~01:00+08:00",
    "cos_output": "models/shuangyan-15b-distill/final/",
    "fixes": [
      "v7: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + empty_cache在epoch转换/模型加载/每500步",
      "v6: vocab_size截断 fix — Teacher(7B)=152064 → Student(1.5B)=151936",
      "v5: HuggingFace forward参数名修正 (ii→input_ids, am→attention_mask)",
      "v4: num_workers=0 (避免CUDA fork)",
      "v5: 移除teacher_logits预存储改为即时forward"
    ]
  },
  "pipeline": {
    "steps": [
      {"name": "母模型SFT训练", "status": "completed", "label": "✅ 完成 (loss=0.2638)"},
      {"name": "上传COS (母模型)", "status": "completed", "label": "✅ 已完成"},
      {"name": "代码模型SFT训练", "status": "completed", "label": "✅ 已完成"},
      {"name": "上传COS (代码模型)", "status": "completed", "label": "✅ 已完成"},
      {"name": "generation_config修复", "status": "completed", "label": "✅ 两个模型已修复"},
      {"name": "霜砚1.5B蒸馏", "status": "running", "label": "🔄 训练中（Screen 87779, Epoch 1/3）"},
      {"name": "上传COS (蒸馏输出)", "status": "pending", "label": "⚪ 等待蒸馏完成"}
    ]
  },
  "next_phases": {
    "distill_shuangyan_15b": "🔄 训练中，预计~01:00完成",
    "shuangyan_deep_sft": "⏳ 等待蒸馏完成后用霜砚语料深度微调",
    "distill_coder_to_zhuyuan_15b": "⏳ 等待Phase 1+2完成",
    "zhuyuan_deep_sft": "⏳ 等待Phase 3完成"
  },
  "alerts": [
    "⚠️ 之前误判代码模型已完成（COS上有历史checkpoint-540）",
    "⚠️ 蒸馏经过7次迭代才稳定：路径错误→存储爆炸→CUDA fork→参数名错误→vocab不匹配→OOM(×2)",
    "⚠️ v7 所有已知崩溃已修复，预计~01:00完成"
  ],
  "links": {
    "notion": "https://www.notion.so/363fb92f383181029fafef8438d6c300"
  }
}