Initial import of Huobao multi-GPU customizations
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
import pathlib,json,hashlib,subprocess,sqlite3,urllib.request,time
|
||||
root=pathlib.Path('/home/ps/huobao-gpu-selection');folder=pathlib.Path(json.loads((root/'task-tracking-current.json').read_text())['directory']);manifest=json.loads((folder/'manifest.json').read_text())
|
||||
for entry in manifest.values():assert hashlib.sha256(pathlib.Path(entry['path']).read_bytes()).hexdigest()==entry['before'],entry['path']+' changed since preparation'
|
||||
for key,p in [('main','/home/ps/huobao-model-bridge/state/jobs.sqlite3'),('hd','/home/ps/huobao-vdn/bridge-state/jobs.sqlite3')]:
|
||||
with sqlite3.connect(p) as src,sqlite3.connect(folder/(key+'-jobs.sqlite3')) as dest:src.backup(dest)
|
||||
changes=[]
|
||||
try:
|
||||
for short in ['general','vdn']:
|
||||
m=manifest[short];p=pathlib.Path(m['path']);p.write_bytes(pathlib.Path(m['candidate']).read_bytes());changes.append((p,folder/(short+'.before.py'),m['name']))
|
||||
subprocess.run(['docker','restart','--timeout','60',m['name']],check=True,stdout=subprocess.DEVNULL)
|
||||
for short,name,port,config in [('main','huobao-model-bridge',18792,'/home/ps/huobao-model-bridge/config.json'),('hd','huobao-vdn-bridge',18794,'/home/ps/huobao-vdn/bridge/config.json')]:
|
||||
p=pathlib.Path(manifest[short+'-bridge']['path']);p.write_bytes((folder/'bridge.after.py').read_bytes());changes.append((p,folder/(short+'-bridge.before.py'),name))
|
||||
subprocess.run(['docker','restart','--timeout','60',name],check=True,stdout=subprocess.DEVNULL)
|
||||
cfg=json.loads(pathlib.Path(config).read_text())
|
||||
for attempt in range(10):
|
||||
try:
|
||||
req=urllib.request.Request(f'http://192.168.110.111:{port}/v1/models',headers={'Authorization':'Bearer '+cfg['token']})
|
||||
with urllib.request.urlopen(req,timeout=10) as r:assert r.status==200
|
||||
break
|
||||
except Exception:
|
||||
if attempt==9:raise
|
||||
time.sleep(1)
|
||||
print('READY',name)
|
||||
cfg=json.loads(pathlib.Path('/home/ps/huobao-model-bridge/config.json').read_text())
|
||||
jid='Z3B1MzpkOTJkZmMxOS0zNWIxLTRmNDEtYjkzZS1mZTJmZjk2OGY0ODc'
|
||||
req=urllib.request.Request('http://192.168.110.111:18792/v2/query/video_generation/'+jid,headers={'Authorization':'Bearer '+cfg['token']})
|
||||
with urllib.request.urlopen(req,timeout=20) as r:task=json.load(r)['task']
|
||||
print(json.dumps({'task':205,'state':task.get('status'),'worker_state':task.get('worker_state'),'routing':task.get('routing')}))
|
||||
except Exception:
|
||||
for p,before,name in reversed(changes):
|
||||
p.write_bytes(before.read_bytes());subprocess.run(['docker','restart','--timeout','60',name],check=True,stdout=subprocess.DEVNULL)
|
||||
raise
|
||||
print('BACKUP',folder)
|
||||
r=subprocess.run(['docker','logs','--tail','1200','comfyui-manju-before-qwentts-20261109'],capture_output=True,text=True)
|
||||
for line in (r.stdout+r.stderr).splitlines():
|
||||
if any(word in line for word in ['RuntimeError:','OutOfMemoryError:','ValueError:','CUDA error:','H3_SPEECH_MISMATCH','H3_SUBTITLE_']):print('PREVIOUS_GPU1_ERROR',line[:300])
|
||||
Reference in New Issue
Block a user