feat: distill panel with GPU status + pipeline watchdog
This commit is contained in:
root
parent
3ddae51f7c
commit
b87be581d3
@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fetch_train.py v2 — 从AutoDL实时同步代码模型训练进度
|
||||
|
||||
修复: 排除checkpoint保存时的"0/1"假进度，从trainer_state.json解析loss
|
||||
"""
|
||||
fetch_train.py v3 — 蒸馏+GPU实时进度同步
|
||||
"""
|
||||
import subprocess, json, re
|
||||
from datetime import datetime, timezone
|
||||
@ -10,48 +9,95 @@ DATA_FILE = '/opt/guanghulab-repo/homepage/training-status.json'
|
||||
SSH_BASE = ['sshpass','-p','HkM43lFVUIsc','ssh','-o','StrictHostKeyChecking=no',
|
||||
'-o','ConnectTimeout=10','-p','23647','root@connect.westd.seetacloud.com']
|
||||
|
||||
def fetch_step():
|
||||
"""从train_coder.log解析最后几步的step/total"""
|
||||
def ssh(cmd_list):
|
||||
try:
|
||||
r = subprocess.run(SSH_BASE + ['tail','-50','/root/autodl-tmp/train_coder.log'],
|
||||
capture_output=True, text=True, timeout=30).stdout
|
||||
r = subprocess.run(SSH_BASE + cmd_list, capture_output=True, text=True, timeout=30)
|
||||
return r.stdout.strip()
|
||||
except:
|
||||
return 0, 11838
|
||||
|
||||
matches = list(re.finditer(r'(\d+)/(\d+)', r))
|
||||
for m in reversed(matches):
|
||||
s, t = int(m.group(1)), int(m.group(2))
|
||||
if t == 11838 and 0 < s <= t:
|
||||
return s, t
|
||||
return 0, 11838
|
||||
return ''
|
||||
|
||||
def get_distill_status():
|
||||
raw = ssh(['tail','-50','/root/autodl-tmp/distill_mother.log'])
|
||||
if not raw:
|
||||
return {}, 'no_log'
|
||||
lines = raw.replace('\r', '\n').split('\n')
|
||||
|
||||
def fetch_loss():
|
||||
"""从trainer_state.json读取最新loss"""
|
||||
try:
|
||||
cmd = 'cat /root/autodl-tmp/output/qwen25-coder-7b-sft/checkpoint-*/trainer_state.json'
|
||||
r = subprocess.run(SSH_BASE + [cmd], capture_output=True, text=True, timeout=15).stdout
|
||||
state = json.loads(r)
|
||||
for entry in reversed(state.get('log_history', [])):
|
||||
if 'loss' in entry:
|
||||
return f"{entry['loss']:.3f}"
|
||||
except:
|
||||
pass
|
||||
return '--'
|
||||
if 'DONE' in raw:
|
||||
return {'phase': 'completed'}, 'done'
|
||||
if 'Train' in raw:
|
||||
phase = 'training'
|
||||
elif 'Load models' in raw or 'Loading weights' in raw:
|
||||
phase = 'loading_models'
|
||||
elif 'Tokenize' in raw:
|
||||
phase = 'tokenizing'
|
||||
else:
|
||||
phase = 'unknown'
|
||||
|
||||
epoch = None; total_epoch = 3
|
||||
for line in reversed(lines):
|
||||
m = re.search(r'Epoch\s+(\d+)/(\d+)', line)
|
||||
if m:
|
||||
epoch, total_epoch = int(m.group(1)), int(m.group(2))
|
||||
break
|
||||
|
||||
step = None; total_steps = None
|
||||
for line in reversed(lines):
|
||||
m = re.search(r'(\d+)/(\d+)\s+\[', line)
|
||||
if m:
|
||||
step, total_steps = int(m.group(1)), int(m.group(2))
|
||||
break
|
||||
|
||||
loss = '--'
|
||||
for line in reversed(lines):
|
||||
m = re.search(r"loss[=:]?\s*'?([0-9.]+)'?", line)
|
||||
if m:
|
||||
loss = m.group(1)
|
||||
break
|
||||
|
||||
eta = '--'
|
||||
for line in reversed(lines):
|
||||
m = re.search(r'<([0-9]+:[0-9]+:[0-9]+)', line)
|
||||
if m:
|
||||
eta = m.group(1)
|
||||
break
|
||||
|
||||
return {'phase': phase, 'epoch': epoch, 'total_epoch': total_epoch,
|
||||
'step': step, 'total_steps': total_steps, 'loss': loss, 'eta': eta}, phase
|
||||
|
||||
def get_gpu_status():
|
||||
raw = ssh(['nvidia-smi','--query-gpu=temperature.gpu,memory.used,memory.total,utilization.gpu,utilization.memory',
|
||||
'--format=csv,noheader'])
|
||||
parts = raw.split(', ')
|
||||
if len(parts) >= 5:
|
||||
return {'temp_c': parts[0].strip(),
|
||||
'mem_used_gb': round(int(parts[1].strip().split()[0])/1024, 1),
|
||||
'mem_total_gb': round(int(parts[2].strip().split()[0])/1024, 1),
|
||||
'gpu_util_pct': parts[3].strip().split()[0],
|
||||
'mem_util_pct': parts[4].strip().split()[0]}
|
||||
return {}
|
||||
|
||||
def main():
|
||||
step, total = fetch_step()
|
||||
loss = fetch_loss()
|
||||
data = {
|
||||
'step': step,
|
||||
'total': total,
|
||||
'loss': loss,
|
||||
'updated': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000000+00:00Z')
|
||||
}
|
||||
info = {'mode': 'distill_shuangyan'}
|
||||
status, phase = get_distill_status()
|
||||
info.update(status)
|
||||
info['gpu'] = get_gpu_status()
|
||||
|
||||
if info.get('phase') == 'completed':
|
||||
info['display_step'] = '完成'
|
||||
info['display_pct'] = 100.0
|
||||
elif info.get('step') and info.get('total_steps') and info.get('epoch'):
|
||||
ep_progress = (info['epoch'] - 1) * 100 / info['total_epoch']
|
||||
ep_step = info['step'] / info['total_steps'] * 100 / info['total_epoch']
|
||||
info['display_pct'] = round(ep_progress + ep_step, 1)
|
||||
info['display_step'] = f"Ep{info['epoch']}/{info['total_epoch']} · {info['step']}/{info['total_steps']}"
|
||||
else:
|
||||
info['display_step'] = phase
|
||||
info['display_pct'] = 0
|
||||
|
||||
info['updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000000+00:00Z')
|
||||
with open(DATA_FILE, 'w') as f:
|
||||
json.dump(data, f)
|
||||
print(f'Step {step} / {total}, Loss: {loss}')
|
||||
json.dump(info, f)
|
||||
print(json.dumps(info, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@ -334,13 +334,13 @@ body{
|
||||
|
||||
<!-- 训练进度 -->
|
||||
<div class="card bg-heart clickable" style="animation-delay:.5s;">
|
||||
<div class="card-header"><span class="hd heart"></span>铸渊 · 代码模型训练中<span class="r">Qwen2.5-Coder-7B</span></div>
|
||||
<div class="card-header"><span class="hd heart"></span>铸渊 · 母模型蒸馏霜砚中<span class="r">母模型7B→霜砚1.5B</span></div>
|
||||
<div class="train-metrics">
|
||||
<div class="train-m"><div class="mv cyan" id="train-step">—</div><div class="ml">当前步数 / 总步数</div></div>
|
||||
<div class="train-m"><div class="mv gold" id="train-loss">—</div><div class="ml">Loss</div></div>
|
||||
<div class="train-m"><div class="mv violet" id="train-pct">—</div><div class="ml">完成进度</div></div>
|
||||
</div>
|
||||
<div class="train-bar-wrap"><div class="train-bar" id="train-bar" style="width:2%"></div></div>
|
||||
<div id="gpu-info" style="text-align:center;font-size:14px;color:#8aa0b8;margin-bottom:10px">GPU: 等待数据...</div><div class="train-bar-wrap"><div class="train-bar" id="train-bar" style="width:2%"></div></div>
|
||||
<div class="train-foot" id="train-eta">训练完成 · 待上传COS ✅</div>
|
||||
</div>
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user