Files
dj/integration-work/check_pinned_hd_progress.py

8 lines
532 B
Python

import subprocess
for name in ('huobao-comfy-gpu1','h3-caption-gpu3-canary-8a5f'):
r=subprocess.run(['docker','logs','--tail','70',name],capture_output=True,text=True)
lines=(r.stdout+r.stderr).replace('\r','\n').splitlines()
selected=[l for l in lines if any(s in l for s in ('SelfLift timing','Prompt executed','Error','Exception','out of memory','%|'))]
print(name,'\n'+'\n'.join(selected[-9:]))
print(subprocess.check_output(['nvidia-smi','--query-gpu=index,memory.used,utilization.gpu','--format=csv,noheader'],text=True))