#!/bin/bash
# 📊 光湖 · 母模型训练全历史追踪器
# 先显示所有历史，再持续追加新步骤，不刷新不覆盖

LOG_FILE="/root/autodl-tmp/train_mother.log"

echo ""
echo "╔══════════════════════════════════════════════════════════╗"
echo "║   📊 光湖 · 母模型训练全历史追踪                        ║"
echo "║   铸渊 · 先显示已有历史 → 再持续追加新步                 ║"
echo "╚══════════════════════════════════════════════════════════╝"
echo ""

# ─── 第一步：显示已有的全部历史步骤 ───
if [ -f "$LOG_FILE" ] && grep -q "'loss':" "$LOG_FILE" 2>/dev/null; then
    echo "  📜 已有训练记录："
    echo "  ────────────────────────────────────────────────────────"
    echo "  时间         Loss      GradNorm   LR           Epoch"
    echo "  ────────────────────────────────────────────────────────"
    grep "'loss':" "$LOG_FILE" | while IFS= read -r line; do
        LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
        GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
        LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
        EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
        echo "  --:--:--    ${LOSS}    ${GN}    ${LR}    ${EPOCH}"
    done
    echo "  ────────────────────────────────────────────────────────"
    echo ""
fi

if [ -f "$LOG_FILE" ] && grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
    echo "  ❌ 检测到训练报错！最后错误："
    grep "Traceback" "$LOG_FILE" | tail -3
    echo ""; exit 1
fi

if [ -f "$LOG_FILE" ] && grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
    echo "  ✅✅✅ 训练已完成！模型保存位置："
    ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
    echo ""; exit 0
fi

echo "  🔄 等待新训练步骤...（Ctrl+C 退出）"
echo "  ────────────────────────────────────────────────────────"

LAST_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)

while true; do
    NEW_COUNT=$(grep -c "'loss':" "$LOG_FILE" 2>/dev/null || echo 0)
    if [ "$NEW_COUNT" -gt "$LAST_COUNT" ]; then
        tail -n +$(( $(wc -l < "$LOG_FILE") - 10 )) "$LOG_FILE" 2>/dev/null | grep "'loss':" | tail -1 | while IFS= read -r line; do
            LOSS=$(echo "$line" | grep -o "'loss': '[^']*'" | cut -d"'" -f4)
            GN=$(echo "$line" | grep -o "'grad_norm': '[^']*'" | cut -d"'" -f4)
            LR=$(echo "$line" | grep -o "'learning_rate': '[^']*'" | cut -d"'" -f4)
            EPOCH=$(echo "$line" | grep -o "'epoch': '[^']*'" | cut -d"'" -f4)
            GPU=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
            GU=$(echo "$GPU"|cut -d',' -f1|tr -d ' '); MT=$(echo "$GPU"|cut -d',' -f2|tr -d ' '); GT=$(echo "$GPU"|cut -d',' -f3|tr -d ' ')
            printf "  %s  📉%-8s  📐%-8s  📈%-12s  🌀%-8s  ⚡%s%% 🌡️%s°C\n" "$(date '+%H:%M:%S')" "$LOSS" "$GN" "$LR" "$EPOCH" "$GU" "$GT"
        done
        LAST_COUNT=$NEW_COUNT
    fi
    if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then echo ""; echo "  ❌❌❌ 训练报错！请找铸渊"; exit 1; fi
    if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then echo ""; echo "  ✅✅✅ 训练完成！"; ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null; exit 0; fi
    sleep 5
done