From b87be581d3fad4fc9d567f6c5a59437cf38fe69b Mon Sep 17 00:00:00 2001
From: root <root@localhost.localdomain>
Date: Wed, 20 May 2026 09:26:07 +0800
Subject: [PATCH] feat: distill panel with GPU status + pipeline watchdog

---
 homepage/fetch_train.py | 120 +++++++++++++++++++++++++++-------------
 homepage/index.html     |   4 +-
 2 files changed, 85 insertions(+), 39 deletions(-)

diff --git a/homepage/fetch_train.py b/homepage/fetch_train.py
index 40804df..a43ec68 100644
--- a/homepage/fetch_train.py
+++ b/homepage/fetch_train.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
-"""fetch_train.py v2 — 从AutoDL实时同步代码模型训练进度
-
-修复: 排除checkpoint保存时的"0/1"假进度，从trainer_state.json解析loss
+"""
+fetch_train.py v3 — 蒸馏+GPU实时进度同步
 """
 import subprocess, json, re
 from datetime import datetime, timezone
@@ -10,48 +9,95 @@ DATA_FILE = '/opt/guanghulab-repo/homepage/training-status.json'
 SSH_BASE = ['sshpass','-p','HkM43lFVUIsc','ssh','-o','StrictHostKeyChecking=no',
             '-o','ConnectTimeout=10','-p','23647','root@connect.westd.seetacloud.com']
 
-def fetch_step():
-    """从train_coder.log解析最后几步的step/total"""
+def ssh(cmd_list):
     try:
-        r = subprocess.run(SSH_BASE + ['tail','-50','/root/autodl-tmp/train_coder.log'],
-            capture_output=True, text=True, timeout=30).stdout
+        r = subprocess.run(SSH_BASE + cmd_list, capture_output=True, text=True, timeout=30)
+        return r.stdout.strip()
     except:
-        return 0, 11838
-    
-    matches = list(re.finditer(r'(\d+)/(\d+)', r))
-    for m in reversed(matches):
-        s, t = int(m.group(1)), int(m.group(2))
-        if t == 11838 and 0 < s <= t:
-            return s, t
-    return 0, 11838
+        return ''
 
+def get_distill_status():
+    raw = ssh(['tail','-50','/root/autodl-tmp/distill_mother.log'])
+    if not raw:
+        return {}, 'no_log'
+    lines = raw.replace('\r', '\n').split('\n')
 
-def fetch_loss():
-    """从trainer_state.json读取最新loss"""
-    try:
-        cmd = 'cat /root/autodl-tmp/output/qwen25-coder-7b-sft/checkpoint-*/trainer_state.json'
-        r = subprocess.run(SSH_BASE + [cmd], capture_output=True, text=True, timeout=15).stdout
-        state = json.loads(r)
-        for entry in reversed(state.get('log_history', [])):
-            if 'loss' in entry:
-                return f"{entry['loss']:.3f}"
-    except:
-        pass
-    return '--'
+    if 'DONE' in raw:
+        return {'phase': 'completed'}, 'done'
+    if 'Train' in raw:
+        phase = 'training'
+    elif 'Load models' in raw or 'Loading weights' in raw:
+        phase = 'loading_models'
+    elif 'Tokenize' in raw:
+        phase = 'tokenizing'
+    else:
+        phase = 'unknown'
 
+    epoch = None; total_epoch = 3
+    for line in reversed(lines):
+        m = re.search(r'Epoch\s+(\d+)/(\d+)', line)
+        if m:
+            epoch, total_epoch = int(m.group(1)), int(m.group(2))
+            break
+
+    step = None; total_steps = None
+    for line in reversed(lines):
+        m = re.search(r'(\d+)/(\d+)\s+\[', line)
+        if m:
+            step, total_steps = int(m.group(1)), int(m.group(2))
+            break
+
+    loss = '--'
+    for line in reversed(lines):
+        m = re.search(r"loss[=:]?\s*'?([0-9.]+)'?", line)
+        if m:
+            loss = m.group(1)
+            break
+
+    eta = '--'
+    for line in reversed(lines):
+        m = re.search(r'<([0-9]+:[0-9]+:[0-9]+)', line)
+        if m:
+            eta = m.group(1)
+            break
+
+    return {'phase': phase, 'epoch': epoch, 'total_epoch': total_epoch,
+            'step': step, 'total_steps': total_steps, 'loss': loss, 'eta': eta}, phase
+
+def get_gpu_status():
+    raw = ssh(['nvidia-smi','--query-gpu=temperature.gpu,memory.used,memory.total,utilization.gpu,utilization.memory',
+               '--format=csv,noheader'])
+    parts = raw.split(', ')
+    if len(parts) >= 5:
+        return {'temp_c': parts[0].strip(),
+                'mem_used_gb': round(int(parts[1].strip().split()[0])/1024, 1),
+                'mem_total_gb': round(int(parts[2].strip().split()[0])/1024, 1),
+                'gpu_util_pct': parts[3].strip().split()[0],
+                'mem_util_pct': parts[4].strip().split()[0]}
+    return {}
 
 def main():
-    step, total = fetch_step()
-    loss = fetch_loss()
-    data = {
-        'step': step,
-        'total': total,
-        'loss': loss,
-        'updated': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000000+00:00Z')
-    }
+    info = {'mode': 'distill_shuangyan'}
+    status, phase = get_distill_status()
+    info.update(status)
+    info['gpu'] = get_gpu_status()
+
+    if info.get('phase') == 'completed':
+        info['display_step'] = '完成'
+        info['display_pct'] = 100.0
+    elif info.get('step') and info.get('total_steps') and info.get('epoch'):
+        ep_progress = (info['epoch'] - 1) * 100 / info['total_epoch']
+        ep_step = info['step'] / info['total_steps'] * 100 / info['total_epoch']
+        info['display_pct'] = round(ep_progress + ep_step, 1)
+        info['display_step'] = f"Ep{info['epoch']}/{info['total_epoch']} · {info['step']}/{info['total_steps']}"
+    else:
+        info['display_step'] = phase
+        info['display_pct'] = 0
+
+    info['updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000000+00:00Z')
     with open(DATA_FILE, 'w') as f:
-        json.dump(data, f)
-    print(f'Step {step} / {total}, Loss: {loss}')
+        json.dump(info, f)
+    print(json.dumps(info, ensure_ascii=False, indent=2))
 
 if __name__ == '__main__':
     main()
diff --git a/homepage/index.html b/homepage/index.html
index 05f6158..046aa7a 100644
--- a/homepage/index.html
+++ b/homepage/index.html
@@ -334,13 +334,13 @@ body{
 
     <!-- 训练进度 -->
     <div class="card bg-heart clickable" style="animation-delay:.5s;">
-      <div class="card-header"><span class="hd heart"></span>铸渊 · 代码模型训练中<span class="r">Qwen2.5-Coder-7B</span></div>
+      <div class="card-header"><span class="hd heart"></span>铸渊 · 母模型蒸馏霜砚中<span class="r">母模型7B→霜砚1.5B</span></div>
       <div class="train-metrics">
         <div class="train-m"><div class="mv cyan" id="train-step">—</div><div class="ml">当前步数 / 总步数</div></div>
         <div class="train-m"><div class="mv gold" id="train-loss">—</div><div class="ml">Loss</div></div>
         <div class="train-m"><div class="mv violet" id="train-pct">—</div><div class="ml">完成进度</div></div>
       </div>
-      <div class="train-bar-wrap"><div class="train-bar" id="train-bar" style="width:2%"></div></div>
+      <div id="gpu-info" style="text-align:center;font-size:14px;color:#8aa0b8;margin-bottom:10px">GPU: 等待数据...</div><div class="train-bar-wrap"><div class="train-bar" id="train-bar" style="width:2%"></div></div>
       <div class="train-foot" id="train-eta">训练完成 · 待上传COS ✅</div>
     </div>