37 lines
2.8 KiB
Python
37 lines
2.8 KiB
Python
import pathlib,json,hashlib,subprocess,sqlite3,urllib.request,time
|
|
root=pathlib.Path('/home/ps/huobao-gpu-selection');folder=pathlib.Path(json.loads((root/'task-tracking-current.json').read_text())['directory']);manifest=json.loads((folder/'manifest.json').read_text())
|
|
for entry in manifest.values():assert hashlib.sha256(pathlib.Path(entry['path']).read_bytes()).hexdigest()==entry['before'],entry['path']+' changed since preparation'
|
|
for key,p in [('main','/home/ps/huobao-model-bridge/state/jobs.sqlite3'),('hd','/home/ps/huobao-vdn/bridge-state/jobs.sqlite3')]:
|
|
with sqlite3.connect(p) as src,sqlite3.connect(folder/(key+'-jobs.sqlite3')) as dest:src.backup(dest)
|
|
changes=[]
|
|
try:
|
|
for short in ['general','vdn']:
|
|
m=manifest[short];p=pathlib.Path(m['path']);p.write_bytes(pathlib.Path(m['candidate']).read_bytes());changes.append((p,folder/(short+'.before.py'),m['name']))
|
|
subprocess.run(['docker','restart','--timeout','60',m['name']],check=True,stdout=subprocess.DEVNULL)
|
|
for short,name,port,config in [('main','huobao-model-bridge',18792,'/home/ps/huobao-model-bridge/config.json'),('hd','huobao-vdn-bridge',18794,'/home/ps/huobao-vdn/bridge/config.json')]:
|
|
p=pathlib.Path(manifest[short+'-bridge']['path']);p.write_bytes((folder/'bridge.after.py').read_bytes());changes.append((p,folder/(short+'-bridge.before.py'),name))
|
|
subprocess.run(['docker','restart','--timeout','60',name],check=True,stdout=subprocess.DEVNULL)
|
|
cfg=json.loads(pathlib.Path(config).read_text())
|
|
for attempt in range(10):
|
|
try:
|
|
req=urllib.request.Request(f'http://192.168.110.111:{port}/v1/models',headers={'Authorization':'Bearer '+cfg['token']})
|
|
with urllib.request.urlopen(req,timeout=10) as r:assert r.status==200
|
|
break
|
|
except Exception:
|
|
if attempt==9:raise
|
|
time.sleep(1)
|
|
print('READY',name)
|
|
cfg=json.loads(pathlib.Path('/home/ps/huobao-model-bridge/config.json').read_text())
|
|
jid='Z3B1MzpkOTJkZmMxOS0zNWIxLTRmNDEtYjkzZS1mZTJmZjk2OGY0ODc'
|
|
req=urllib.request.Request('http://192.168.110.111:18792/v2/query/video_generation/'+jid,headers={'Authorization':'Bearer '+cfg['token']})
|
|
with urllib.request.urlopen(req,timeout=20) as r:task=json.load(r)['task']
|
|
print(json.dumps({'task':205,'state':task.get('status'),'worker_state':task.get('worker_state'),'routing':task.get('routing')}))
|
|
except Exception:
|
|
for p,before,name in reversed(changes):
|
|
p.write_bytes(before.read_bytes());subprocess.run(['docker','restart','--timeout','60',name],check=True,stdout=subprocess.DEVNULL)
|
|
raise
|
|
print('BACKUP',folder)
|
|
r=subprocess.run(['docker','logs','--tail','1200','comfyui-manju-before-qwentts-20261109'],capture_output=True,text=True)
|
|
for line in (r.stdout+r.stderr).splitlines():
|
|
if any(word in line for word in ['RuntimeError:','OutOfMemoryError:','ValueError:','CUDA error:','H3_SPEECH_MISMATCH','H3_SUBTITLE_']):print('PREVIOUS_GPU1_ERROR',line[:300])
|