Files
dj/integration-work/diagnose_video_timeouts.py

23 lines
1.9 KiB
Python

import subprocess,json
jobs={202:'Z3B1MV84MTg5OjA3MjIxMTkyLWViZTAtNDQ2OC04Zjg2LTA5YWRiMjdhMjhmYw',203:'Z3B1MV84MTg5OjliY2JiNmY4LWU1OWMtNDg5YS1hN2Y0LThmZDdkMTU3MmFlYg',204:'ZGVmYXVsdDo4MTgxY2IzOS1jYTg0LTQ2ODItODg2MC01ZjNjZWNmMDZhZjA',205:'Z3B1MzpkOTJkZmMxOS0zNWIxLTRmNDEtYjkzZS1mZTJmZjk2OGY0ODc'}
code='''import asyncio,httpx,json,app
jobs=JOBS
async def main():
async with httpx.AsyncClient(timeout=15) as c:
for appid,jid in jobs.items():
p,pid=app.decode_job(jid)
q=await c.get(p.base_url+'/queue',headers=app.provider_headers(p));q.raise_for_status();q=q.json()
pending=[x[1] for x in q.get('queue_pending',[])];running=[x[1] for x in q.get('queue_running',[])]
h=await c.get(p.base_url+'/history/'+pid,headers=app.provider_headers(p));h.raise_for_status();entry=h.json().get(pid,{})
info={'id':appid,'worker':p.id,'running':pid in running,'queue_position':pending.index(pid)+1 if pid in pending else None,'pending_count':len(pending),'status':entry.get('status'),'output_nodes':list(entry.get('outputs',{}))}
# Avoid prompts/tensors in failure diagnostics, retain the actual exception and node.
status=info.pop('status') or {};info['history_status']=status.get('status_str');info['completed']=status.get('completed')
info['errors']=[{k:e[1].get(k) for k in ['node_id','node_type','exception_type','exception_message','traceback'] if k in e[1]} for e in status.get('messages',[]) if e[0] in ['execution_error','execution_interrupted']]
try:
result=await app.get_job(jid);info['normalizer']={k:result.get(k) for k in ['status','error','providerId']}
except Exception as e:info['normalizer_error']=str(getattr(e,'detail',e))
print(json.dumps(info,ensure_ascii=False))
asyncio.run(main())'''.replace('JOBS',repr(jobs))
r=subprocess.run(['docker','exec','canvas-h3-prompt-normalizer-healthy-20261026','python','-c',code],capture_output=True,text=True)
print(r.stdout);print(r.stderr);raise SystemExit(r.returncode)