Show ComfyUI task progress and measured sampling time estimates
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
"""Read-only ComfyUI telemetry, embedded in the existing private normalizers.
|
||||
|
||||
Only the requesting Huobao job's client connection is observed. No queue mutation
|
||||
or image previews are used. Sampling ETA is never presented as whole-video ETA.
|
||||
"""
|
||||
import asyncio as _hp_asyncio
|
||||
import json as _hp_json
|
||||
import time as _hp_time
|
||||
import math as _hp_math
|
||||
import statistics as _hp_stats
|
||||
from urllib.parse import quote as _hp_quote
|
||||
import httpx as _hp_httpx
|
||||
import websockets as _hp_ws
|
||||
|
||||
_hp_watchers = {}
|
||||
_hp_histories = {}
|
||||
|
||||
|
||||
def _hp_phase(kind):
|
||||
kind = str(kind).lower()
|
||||
if 'sampler' in kind: return 'sampling'
|
||||
if 'decode' in kind: return 'decoding'
|
||||
if 'savevideo' in kind or 'combine' in kind or 'createvideo' in kind: return 'encoding'
|
||||
if 'loader' in kind or kind == 'loadimage': return 'loading'
|
||||
if 'caption' in kind: return 'postprocessing'
|
||||
if 'encode' in kind or 'tovideo' in kind: return 'conditioning'
|
||||
return 'running'
|
||||
|
||||
|
||||
def _hp_signature(graph):
|
||||
# Keep graph topology, model weights, image count, dimensions, FPS, duration,
|
||||
# steps and sampler. Exclude content, random seeds and output filenames.
|
||||
omitted = {'text', 'prompt', 'positive', 'negative', 'seed', 'noise_seed',
|
||||
'filename_prefix', 'image', 'audio', 'video', 'file', 'path'}
|
||||
result = {}
|
||||
for node, value in graph.items():
|
||||
inputs = {}
|
||||
for key, item in value.get('inputs', {}).items():
|
||||
if key in omitted and not isinstance(item, list): continue
|
||||
if any(word in key.lower() for word in ['prompt', 'text', 'caption']) and not isinstance(item, list): continue
|
||||
inputs[key] = item
|
||||
result[node] = {'class_type': value.get('class_type'), 'inputs': inputs}
|
||||
return _hp_json.dumps(result, sort_keys=True, separators=(',', ':'))
|
||||
|
||||
|
||||
def _hp_history_estimate(graph, history):
|
||||
signature = _hp_signature(graph)
|
||||
durations = []
|
||||
for entry in history.values():
|
||||
status = entry.get('status', {})
|
||||
prompt = entry.get('prompt', [])
|
||||
if status.get('status_str') != 'success' or not status.get('completed') or len(prompt) < 3: continue
|
||||
previous = prompt[2]
|
||||
if not isinstance(previous, dict) or _hp_signature(previous) != signature: continue
|
||||
times = {}; cached = []
|
||||
for event in status.get('messages', []):
|
||||
if len(event) < 2 or not isinstance(event[1], dict): continue
|
||||
times[event[0]] = event[1].get('timestamp')
|
||||
if event[0] == 'execution_cached': cached = event[1].get('nodes', [])
|
||||
if any('sampler' in str(previous.get(str(n), {}).get('class_type', '')).lower() for n in cached): continue
|
||||
start, end = times.get('execution_start'), times.get('execution_success')
|
||||
if not isinstance(start, (int, float)) or not isinstance(end, (int, float)): continue
|
||||
duration = (end - start) / 1000
|
||||
if 2 <= duration <= 86400: durations.append(duration)
|
||||
result = {'historySamples': len(durations)}
|
||||
if len(durations) >= 2:
|
||||
# A range, not a guarantee; model loading and shared GPU contention vary.
|
||||
result.update(historyMinSeconds=round(min(durations) * .8), historyMaxSeconds=round(max(durations) * 1.2))
|
||||
return result
|
||||
|
||||
|
||||
def _hp_event(state, message, timestamp):
|
||||
data = message.get('data', {})
|
||||
if not isinstance(data, dict): return
|
||||
event = message.get('type')
|
||||
if data.get('prompt_id') not in (None, state['prompt']): return
|
||||
if event == 'executing':
|
||||
node = data.get('node')
|
||||
if node is None: return
|
||||
node = str(node)
|
||||
if node != state.get('node'):
|
||||
state.update(node=node, samples=[], value=None, total=None)
|
||||
state['phase'] = _hp_phase(state['classes'].get(node))
|
||||
state['updated'] = timestamp
|
||||
elif event == 'progress' and data.get('prompt_id') == state['prompt']:
|
||||
value, total = data.get('value'), data.get('max')
|
||||
if not all(type(x) in (int, float) and _hp_math.isfinite(x) for x in (value, total)): return
|
||||
if not 0 <= value <= total <= 1000000 or total <= 0: return
|
||||
node = str(data.get('node', state.get('node', '')))
|
||||
if node != state.get('node') or total != state.get('total') or value < (state.get('value') or 0):
|
||||
state.update(node=node, samples=[])
|
||||
if state.get('value') != value or not state['samples']:
|
||||
state['samples'] = (state['samples'] + [(timestamp, value)])[-8:]
|
||||
state.update(value=value, total=total, updated=timestamp, phase=_hp_phase(state['classes'].get(node)))
|
||||
|
||||
|
||||
def _hp_snapshot(state, timestamp):
|
||||
result = {'phase': state.get('phase', 'running')}
|
||||
if not state.get('connected') or timestamp - state.get('updated', 0) > 600:
|
||||
return {'phase': 'running'}
|
||||
if state.get('value') is None: return result
|
||||
result.update(step=state['value'], totalSteps=state['total'], stepUpdatedAt=round(state['updated'] * 1000))
|
||||
samples = state['samples']
|
||||
if state.get('phase') == 'sampling' and len(samples) >= 3:
|
||||
seconds = samples[-1][0] - samples[0][0]; steps = samples[-1][1] - samples[0][1]
|
||||
if seconds >= 5 and steps > 0 and state['value'] < state['total']:
|
||||
speed = seconds / steps
|
||||
if timestamp - state['updated'] <= max(30, speed * 2.5):
|
||||
remaining = max(1, speed * (state['total'] - state['value']) - (timestamp - state['updated']))
|
||||
result.update(samplingMinSeconds=round(remaining * .8), samplingMaxSeconds=round(remaining * 1.3))
|
||||
return result
|
||||
|
||||
|
||||
async def _hp_watch(state, base, headers, client_id):
|
||||
uri = base.replace('https://', 'wss://').replace('http://', 'ws://') + '/ws?clientId=' + _hp_quote(client_id, safe='')
|
||||
while _hp_time.time() - state['requested'] < 120:
|
||||
try:
|
||||
async with _hp_ws.connect(uri, additional_headers=headers, max_size=32 * 1024 * 1024, open_timeout=5) as ws:
|
||||
state['connected'] = True
|
||||
# This observer needs text progress only, never preview images.
|
||||
await ws.send(_hp_json.dumps({'type':'feature_flags','data':{'supports_preview_metadata':True}}))
|
||||
while _hp_time.time() - state['requested'] < 120:
|
||||
try: message = await _hp_asyncio.wait_for(ws.recv(), 15)
|
||||
except _hp_asyncio.TimeoutError: continue
|
||||
if isinstance(message, str):
|
||||
try: _hp_event(state, _hp_json.loads(message), _hp_time.time())
|
||||
except (ValueError, TypeError, AttributeError): pass
|
||||
except _hp_asyncio.CancelledError: raise
|
||||
except Exception:
|
||||
await _hp_asyncio.sleep(5)
|
||||
finally: state['connected'] = False
|
||||
|
||||
|
||||
async def huobao_progress(provider, prompt_id, queue, headers):
|
||||
"""Failure of progress reporting must never fail or resubmit a generation."""
|
||||
timestamp = _hp_time.time()
|
||||
running = queue.get('queue_running', []); pending = queue.get('queue_pending', [])
|
||||
row = next((x for x in running + pending if len(x) > 3 and x[1] == prompt_id), None)
|
||||
if row is None: return None
|
||||
result = {'observedAt': round(timestamp * 1000), 'phase': 'queued' if row in pending else 'running'}
|
||||
if row in pending:
|
||||
result.update(queuePosition=pending.index(row) + 1, queueAhead=len(running) + pending.index(row))
|
||||
try:
|
||||
previous = _hp_watchers.get(provider.id)
|
||||
if previous and (previous['prompt'] != prompt_id and row in running or previous['task'].done()):
|
||||
previous['task'].cancel(); _hp_watchers.pop(provider.id, None); previous = None
|
||||
if row in running:
|
||||
client_id = row[3].get('client_id')
|
||||
# Huobao's normalizer generates unique canvas-* client IDs. Never
|
||||
# replace a human's browser websocket or guess another job's state.
|
||||
if isinstance(client_id, str) and client_id.startswith('canvas-'):
|
||||
if previous is None:
|
||||
previous = {'prompt':prompt_id, 'classes':{str(k):v.get('class_type') for k,v in row[2].items()},
|
||||
'requested':timestamp, 'samples':[], 'connected':False}
|
||||
previous['task'] = _hp_asyncio.create_task(_hp_watch(previous, provider.base_url, headers, client_id))
|
||||
_hp_watchers[provider.id] = previous
|
||||
previous['requested'] = timestamp
|
||||
if previous['prompt'] == prompt_id: result.update(_hp_snapshot(previous, timestamp))
|
||||
cached = _hp_histories.get(provider.id)
|
||||
if cached is None or timestamp - cached[0] > 90:
|
||||
async with _hp_httpx.AsyncClient(timeout=5) as client:
|
||||
response = await client.get(provider.base_url + '/history', params={'max_items':80}, headers=headers)
|
||||
response.raise_for_status(); history = response.json()
|
||||
_hp_histories[provider.id] = (timestamp, history)
|
||||
else: history = cached[1]
|
||||
result.update(_hp_history_estimate(row[2], history))
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
Reference in New Issue
Block a user