26 lines
2.0 KiB
Python
26 lines
2.0 KiB
Python
"""Read-only GPU3 history/log inspection for a previously failed task."""
|
|
import json, subprocess, re
|
|
job = 'Z3B1MzoxZTVhOTVkNC1hMjk5LTQ5OTAtOGU2Yi02M2M1N2EzMTJkMzM'
|
|
code = '''import asyncio,httpx,json,app
|
|
async def main():
|
|
p,pid=app.decode_job(JOB)
|
|
async with httpx.AsyncClient(timeout=15) as c:
|
|
q=(await c.get(p.base_url+'/queue',headers=app.provider_headers(p))).json()
|
|
h=(await c.get(p.base_url+'/history/'+pid,headers=app.provider_headers(p))).json().get(pid,{})
|
|
status=h.get('status',{})
|
|
errors=[{k:x[1].get(k) for k in ['node_id','node_type','exception_type','exception_message','traceback'] if k in x[1]} for x in status.get('messages',[]) if x[0] in ['execution_error','execution_interrupted']]
|
|
print(json.dumps({'prompt_id':pid,'provider':p.id,'running_ids':[x[1] for x in q.get('queue_running',[])],'pending_ids':[x[1] for x in q.get('queue_pending',[])],'history_present':bool(h),'status':status.get('status_str'),'errors':errors,'output_nodes':list(h.get('outputs',{}))},ensure_ascii=False))
|
|
asyncio.run(main())'''.replace('JOB',repr(job))
|
|
run=subprocess.run(['docker','exec','canvas-h3-prompt-normalizer-healthy-20261026','python','-c',code],capture_output=True,text=True)
|
|
print(run.stdout);print(run.stderr)
|
|
name='h3-caption-gpu3-canary-8a5f'
|
|
details=json.loads(subprocess.check_output(['docker','inspect',name],text=True))[0]
|
|
print(json.dumps({'container':name,'started_at':details['State']['StartedAt'],'running':details['State']['Running'],'oom_killed':details['State']['OOMKilled'],'restart_count':details['RestartCount']}))
|
|
logs=subprocess.run(['docker','logs','--since','25m','--tail','1800',name],capture_output=True,text=True)
|
|
lines=(logs.stdout+'\n'+logs.stderr).splitlines()
|
|
matches=[i for i,line in enumerate(lines) if re.search(r'1e5a95d4|execution failed|exception|error|out of memory|Traceback|prompt executed|H3_HISTORY',line,re.I)]
|
|
indices=sorted({j for i in matches for j in range(max(0,i-2),min(len(lines),i+6))})
|
|
for i in indices:
|
|
line=re.sub(r'(?i)(token|api[_-]?key|authorization)(\s*[=:]\s*)\S+',r'\1\2[redacted]',lines[i])
|
|
print(line[:700])
|