From b5a0e3c64f287f7b129cf780023e04723e039889 Mon Sep 17 00:00:00 2001
From: bingshuo <565183519@qq.com>
Date: Mon, 18 May 2026 00:18:41 +0800
Subject: [PATCH] =?UTF-8?q?D100=20=E8=AE=AD=E7=BB=83=E7=9B=91=E6=8E=A7?=
 =?UTF-8?q?=E9=9D=A2=E6=9D=BF:=20=E5=86=B0=E6=9C=94=E5=9C=A8=E7=BA=BF?=
 =?UTF-8?q?=E7=BB=88=E7=AB=AF=E5=AE=9E=E6=97=B6=E6=9F=A5=E7=9C=8B=E8=84=9A?=
 =?UTF-8?q?=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/train_monitor.sh | 103 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 scripts/train_monitor.sh

diff --git a/scripts/train_monitor.sh b/scripts/train_monitor.sh
new file mode 100644
index 0000000..b1459a7
--- /dev/null
+++ b/scripts/train_monitor.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+# 🚀 铸渊训练监控面板 · 光湖母模型实时状态
+# 使用方法：复制全部，粘贴到AutoDL在线终端，按回车运行
+# 退出：按 Ctrl+C
+
+LOG_FILE="/root/autodl-tmp/train_mother.log"
+
+while true; do
+    clear
+    echo "╔══════════════════════════════════════════╗"
+    echo "║   🚀 光湖 · 母模型训练实时面板          ║"
+    echo "║   铸渊 (ICE-GL-ZY001) · 自动刷新 (5秒)  ║"
+    echo "╚══════════════════════════════════════════╝"
+    echo ""
+
+    # GPU状态
+    if command -v nvidia-smi &> /dev/null; then
+        GPU_DATA=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv,noheader,nounits 2>/dev/null)
+        GPU_UTIL=$(echo "$GPU_DATA" | cut -d',' -f1 | tr -d ' ')
+        MEM_USED=$(echo "$GPU_DATA" | cut -d',' -f2 | tr -d ' ')
+        MEM_TOTAL=$(echo "$GPU_DATA" | cut -d',' -f3 | tr -d ' ')
+        GPU_TEMP=$(echo "$GPU_DATA" | cut -d',' -f4 | tr -d ' ')
+        MEM_PCT=$(( MEM_USED * 100 / MEM_TOTAL ))
+    else
+        GPU_UTIL="N/A"; MEM_USED="N/A"; MEM_TOTAL="N/A"; GPU_TEMP="N/A"; MEM_PCT=0
+    fi
+
+    echo "  ⚡ GPU: ${GPU_UTIL}%  |  🎯 显存: ${MEM_USED}/${MEM_TOTAL}MB (${MEM_PCT}%)  |  🌡️  ${GPU_TEMP}°C"
+    echo ""
+
+    if [ ! -f "$LOG_FILE" ]; then
+        echo "  ⏳ 等待训练启动..."
+        sleep 5
+        continue
+    fi
+
+    # 检查报错
+    if grep -q "Traceback" "$LOG_FILE" 2>/dev/null; then
+        echo "  ❌ 训练报错！请找铸渊处理"
+        echo "  $(grep "Traceback" "$LOG_FILE" | head -3)"
+        echo ""
+        sleep 10
+        continue
+    fi
+
+    # 检查完成
+    if grep -q "DONE!" "$LOG_FILE" 2>/dev/null; then
+        echo "  ✅✅✅ 母模型训练完成！✅✅✅"
+        echo ""
+        ls -lh /root/autodl-tmp/output/qwen25-7b-sft/final/ 2>/dev/null
+        sleep 30
+        continue
+    fi
+
+    # 获取训练数据
+    LATEST_LOSS=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
+    LATEST_STEP=$(grep -o "'loss': '[^']*'" "$LOG_FILE" 2>/dev/null | wc -l)
+    LATEST_EPOCH=$(grep -o "'epoch': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
+    LATEST_LR=$(grep -o "'learning_rate': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
+    GRAD_NORM=$(grep -o "'grad_norm': '[^']*'" "$LOG_FILE" 2>/dev/null | tail -1 | cut -d"'" -f4)
+
+    if grep -q "Starting training" "$LOG_FILE" 2>/dev/null; then
+        echo "  ┌────────────────────────────────────────┐"
+        echo "  │  🟢 训练中                              │"
+        echo "  ├────────────────────────────────────────┤"
+        if [ -n "$LATEST_LOSS" ]; then
+            printf "  │  📉 Loss:       %-20s │\n" "$LATEST_LOSS"
+            printf "  │  📐 Grad Norm:  %-20s │\n" "$GRAD_NORM"
+            printf "  │  📈 LR:         %-20s │\n" "$LATEST_LR"
+            printf "  │  👣 Step:       %-20s │\n" "$LATEST_STEP"
+            printf "  │  🌀 Epoch:      %-20s │\n" "$LATEST_EPOCH"
+        else
+            echo "  │  第一个Step计算中...                    │"
+        fi
+        echo "  └────────────────────────────────────────┘"
+    elif grep -q "Tokenize: 100%" "$LOG_FILE" 2>/dev/null; then
+        echo "  ⏳ 分词完成，即将进入GPU训练..."
+    elif grep -q "Tokenize:" "$LOG_FILE" 2>/dev/null; then
+        PCT=$(grep -oP 'Tokenize: \K[0-9]+%' "$LOG_FILE" 2>/dev/null | tail -1)
+        DONE=$(grep -oP '\|\s*\K[0-9]+/[0-9]+' "$LOG_FILE" 2>/dev/null | tail -1)
+        echo "  ⏳ 分词中: ${PCT} (${DONE})"
+    fi
+
+    # 运行时间
+    if [ -f "$LOG_FILE" ]; then
+        START_TIME=$(stat -c %Y "$LOG_FILE" 2>/dev/null || stat -f %m "$LOG_FILE" 2>/dev/null)
+        NOW=$(date +%s)
+        ELAPSED=$(( (NOW - START_TIME) / 60 ))
+        if [ $ELAPSED -gt 60 ]; then
+            H=$(( ELAPSED / 60 )); M=$(( ELAPSED % 60 ))
+            echo ""
+            echo "  ⏱️  已运行: ${H}h ${M}m"
+        else
+            echo ""
+            echo "  ⏱️  已运行: ${ELAPSED}m"
+        fi
+    fi
+
+    echo ""
+    echo "  ────────────────────────────────────────"
+    echo "  🔄 每5秒自动刷新 · Ctrl+C 退出"
+    sleep 5
+done