28 lines
1.9 KiB
Python
28 lines
1.9 KiB
Python
import subprocess, json
|
|
names=['huobao-comfy-gpu1','comfyui-manju','comfyui','h3-caption-gpu3-canary-8a5f','canvas-h3-prompt-normalizer-healthy-20261026','huobao-vdn-normalizer']
|
|
for name in names:
|
|
r=subprocess.run(['docker','inspect',name],capture_output=True,text=True)
|
|
if r.returncode: print(name,'not found');continue
|
|
d=json.loads(r.stdout)[0]
|
|
print(json.dumps({'name':name,'status':d['State']['Status'],'started':d['State']['StartedAt'],'pid':d['State']['Pid']}))
|
|
print(subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader'],text=True))
|
|
code='''import asyncio,httpx,json,app,hashlib,inspect
|
|
print('app_sha',hashlib.sha256(open('/app/app.py','rb').read()).hexdigest())
|
|
print(inspect.getsource(app.provider_queue_depth))
|
|
async def main():
|
|
async with httpx.AsyncClient(timeout=8) as c:
|
|
for p in app.providers().values():
|
|
try:
|
|
q=await c.get(p.base_url+'/queue',headers=app.provider_headers(p));q.raise_for_status();q=q.json()
|
|
s=await c.get(p.base_url+'/system_stats',headers=app.provider_headers(p));s.raise_for_status();s=s.json()
|
|
print(json.dumps({'provider':p.id,'url':p.base_url,'running':[x[1] for x in q.get('queue_running',[])],'pending':[x[1] for x in q.get('queue_pending',[])],'devices':s.get('devices',[])}))
|
|
except Exception as e: print(json.dumps({'provider':p.id,'error':str(e)}))
|
|
asyncio.run(main())
|
|
'''
|
|
print(subprocess.check_output(['docker','exec','canvas-h3-prompt-normalizer-healthy-20261026','python','-c',code],text=True))
|
|
for name in ['huobao-comfy-gpu1','comfyui-manju']:
|
|
r=subprocess.run(['docker','logs','--tail','50',name],capture_output=True,text=True)
|
|
print('LOG',name)
|
|
for line in (r.stdout+r.stderr).replace('\r','\n').splitlines():
|
|
if any(x.lower() in line.lower() for x in ['%|','Prompt executed','error','memory','free','unload','traceback','GET /queue','POST /free']):print(line[:300])
|