Add per-task image and video cancellation with confirmed worker receipts
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
// Inspect the installed app using GET requests only; never cancel live jobs.
|
||||
const {chromium}=require('playwright-core'),fs=require('node:fs'),assert=require('node:assert/strict');
|
||||
(async()=>{
|
||||
const root='D:/web/dj',api=JSON.parse(fs.readFileSync(root+'/huobao-data/last-launch.json')).api;
|
||||
const tasks=(await (await fetch(api+'/episodes/7/generation-tasks')).json()).data.tasks;
|
||||
const browser=await chromium.launch({channel:'chrome',headless:true});
|
||||
try {
|
||||
const page=await browser.newPage({viewport:{width:1600,height:1200}}),errors=[];
|
||||
page.on('pageerror',e=>errors.push(e.message));
|
||||
await page.addInitScript(()=>localStorage.setItem('huobao:tours',JSON.stringify(['episode'])));
|
||||
await page.route('**/api/**',r=>['GET','HEAD'].includes(r.request().method())?r.continue():r.abort());
|
||||
await page.goto(api.replace('/api/v1','')+'/drama/6/episode/1',{waitUntil:'networkidle'});
|
||||
await page.getByRole('button',{name:/^任务/}).click();
|
||||
const drawer=page.locator('.task-drawer-overlay');await drawer.waitFor();
|
||||
const row=id=>drawer.locator('.gen-task-row').filter({has:page.locator('.video-task-meta-line').filter({hasText:'#'+id})});
|
||||
const active=tasks.filter(t=>t.can_cancel);
|
||||
for(const task of active) {
|
||||
assert.equal(await row(task.id).getByRole('button',{name:/^(重试)?取消任务$/,exact:true}).count(),1);
|
||||
}
|
||||
const cancelled=tasks.filter(t=>t.error_code==='user_cancelled');
|
||||
for(const task of cancelled)assert.equal(await row(task.id).locator('.video-task-status.is-cancelled').count(),1);
|
||||
if(active.length)await row(active[active.length-1].id).scrollIntoViewIfNeeded();
|
||||
await page.screenshot({path:root+'/integration-work/cancel-installed.png'});
|
||||
assert.deepEqual(errors,[]);
|
||||
console.log('PASS: installed cancel buttons for tasks',active.map(t=>t.id),'; cancelled history',cancelled.length,'; GET-only verification');
|
||||
}finally{await browser.close()}
|
||||
})().catch(e=>{console.error(e.stack);process.exit(1)});
|
||||
@@ -0,0 +1,49 @@
|
||||
const {chromium}=require('playwright-core'),assert=require('node:assert/strict');
|
||||
(async()=>{
|
||||
const browser=await chromium.launch({channel:'chrome',headless:true});
|
||||
try {
|
||||
const page=await browser.newPage({viewport:{width:1500,height:1100}}),errors=[],calls=[],states={};
|
||||
page.on('pageerror',e=>errors.push(e.message));
|
||||
await page.addInitScript(()=>localStorage.setItem('huobao:tours',JSON.stringify(['episode'])));
|
||||
await page.route('**/api/**',async route=>{
|
||||
const req=route.request(),url=req.url(),match=url.match(/\/tasks\/(\d+)\/cancel$/);
|
||||
if(req.method()==='POST'&&match){
|
||||
const id=+match[1];assert.ok([237,239].includes(id));calls.push(id);
|
||||
states[id]=id===239&&!states[id]?'pending':'cancelled';
|
||||
return route.fulfill({status:200,json:{code:200,data:{outcome:states[id]}}});
|
||||
}
|
||||
if(!['GET','HEAD'].includes(req.method()))throw Error('Unexpected mutation: '+req.method()+' '+url);
|
||||
if(url.includes('/generation-tasks')){
|
||||
const response=await route.fetch(),body=await response.json();
|
||||
for(const task of body.data.tasks){
|
||||
if(![237,239].includes(task.id))continue;
|
||||
task.status=states[task.id]==='cancelled'?'failed':'processing';
|
||||
task.error_code=states[task.id]==='cancelled'?'user_cancelled':null;
|
||||
task.error_msg=states[task.id]==='cancelled'?'任务已取消,历史记录和已有素材保留':null;
|
||||
task.stage=states[task.id]==='pending'?'cancelling':task.id===239?'queued':'running';
|
||||
task.can_cancel=task.status==='processing';
|
||||
}
|
||||
return route.fulfill({response,json:body});
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
await page.goto('http://127.0.0.1:18938/drama/6/episode/1',{waitUntil:'networkidle'});
|
||||
await page.getByRole('button',{name:/^任务/}).click();
|
||||
const drawer=page.locator('.task-drawer-overlay');await drawer.waitFor();
|
||||
const row=id=>drawer.locator('.gen-task-row').filter({has:page.locator('.video-task-meta-line').filter({hasText:'#'+id})});
|
||||
assert.equal(await row(237).getByRole('button',{name:'取消任务',exact:true}).count(),1);
|
||||
await row(239).getByRole('button',{name:'取消任务',exact:true}).click();
|
||||
await row(239).getByRole('button',{name:'重试取消任务',exact:true}).waitFor();
|
||||
assert.match(await row(239).innerText(),/正在等待显卡确认/);
|
||||
assert.equal(await row(237).getByRole('button',{name:'取消任务',exact:true}).count(),1);
|
||||
await row(239).getByRole('button',{name:'重试取消任务',exact:true}).click();
|
||||
await row(239).locator('.video-task-status.is-cancelled').waitFor();
|
||||
assert.match(await row(239).innerText(),/已取消/);
|
||||
assert.equal(await row(239).getByRole('button',{name:/取消任务/}).count(),0);
|
||||
await row(237).getByRole('button',{name:'取消任务',exact:true}).click();
|
||||
await row(237).locator('.video-task-status.is-cancelled').waitFor();
|
||||
assert.deepEqual(calls,[239,239,237]);assert.deepEqual(errors,[]);
|
||||
await page.screenshot({path:'D:/web/dj/integration-work/cancel-ui-fixture.png'});
|
||||
console.log('PASS: per-image/video cancel buttons; pending confirmation/retry; cancelled styling/counts; no real cancellations');
|
||||
}finally{await browser.close()}
|
||||
})().catch(e=>{console.error(e.stack);process.exit(1)});
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Patch only the existing private bridges; no real jobs are cancelled."""
|
||||
from pathlib import Path
|
||||
import datetime,subprocess,json
|
||||
base=Path('/home/ps/huobao-progress-update')
|
||||
backup=base/('cancel-backup-'+datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
|
||||
backup.mkdir(mode=0o700)
|
||||
endpoint=''' @app.post("/v1/images/task/{jid}/cancel", dependencies=[Depends(auth)])
|
||||
async def cancel_image(jid: str):
|
||||
with closing(sqlite3.connect(db_path)) as db:
|
||||
found = db.execute("SELECT 1 FROM image_jobs WHERE id=?", (jid,)).fetchone()
|
||||
if not found:
|
||||
raise HTTPException(404, "Unknown image task")
|
||||
result = await upstream("POST", "/v1/jobs/" + quote(jid, safe="") + "/cancel", {})
|
||||
return {"task": {"id": jid, "status": result.get("status", "running"),
|
||||
"cancelRequested": result.get("cancelRequested", False)}}
|
||||
|
||||
'''
|
||||
for name,location in [('huobao-model-bridge','/home/ps/huobao-model-bridge/bridge.py'),('huobao-vdn-bridge','/home/ps/huobao-vdn/bridge/bridge.py')]:
|
||||
p=Path(location);original=p.read_text();s=original
|
||||
if 'async def cancel_image(' not in s:
|
||||
target=' @app.get("/v1/images/task/{jid}", dependencies=[Depends(auth)])'
|
||||
assert s.count(target)==1;s=s.replace(target,endpoint+target)
|
||||
for old,new in [
|
||||
('return {"status": "failed", "error": {"message": str(result.get("error") or "Image generation failed")[:600]}}', 'return {"task_id": jid, "status": "failed", "cancelled": state == "cancelled", "error": {"message": str(result.get("error") or "Image generation failed")[:600]}}'),
|
||||
('return {"status": "processing", "provider_id": result.get("providerId"), "worker_state": result.get("worker_state"), "queue_position": result.get("queue_position")}', 'return {"task_id": jid, "status": "processing", "provider_id": result.get("providerId"), "worker_state": result.get("worker_state"), "queue_position": result.get("queue_position")}'),
|
||||
('return {"status": "failed", "error": {"message": "Task completed without an image output"}}', 'return {"task_id": jid, "status": "failed", "error": {"message": "Task completed without an image output"}}'),
|
||||
('return {"status": "completed", "image_url": url, "data": [{"url": url}], "provider_id": result.get("providerId")}', 'return {"task_id": jid, "status": "completed", "image_url": url, "data": [{"url": url}], "provider_id": result.get("providerId")}')]:
|
||||
if new not in s:assert s.count(old)==1;s=s.replace(old,new)
|
||||
compile(s,location,'exec');(backup/(name+'.py')).write_text(original);p.write_text(s)
|
||||
subprocess.run(['docker','restart','--time','15',name],check=True,capture_output=True)
|
||||
assert json.loads(subprocess.check_output(['docker','inspect',name]))[0]['State']['Running']
|
||||
print(name,'updated')
|
||||
print('backup',backup)
|
||||
@@ -21,7 +21,7 @@ assert count == 1
|
||||
source, count = re.subn(r'&& !!taskRouting\(\{ provider: config\d*\.provider, taskId \}\)', '&& !!taskId', source)
|
||||
assert count == 1
|
||||
(data / 'backend.mjs').write_text(source, 'utf-8')
|
||||
events = []; reject = False; process = None
|
||||
events = []; reject = False; process = None; cancel_modes = {}; job_states = {}
|
||||
|
||||
class Mock(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args): pass
|
||||
@@ -37,10 +37,19 @@ class Mock(BaseHTTPRequestHandler):
|
||||
self.reply(400, {'error': {'message': 'Local video requires inline image data'}})
|
||||
else: self.reply(200, {'valid': True})
|
||||
elif self.path.endswith('/cancel'):
|
||||
self.reply(200, {'task': {'id': self.path.split('/')[-2], 'status': 'cancelled'}})
|
||||
jid=self.path.split('/')[-2];mode=cancel_modes.get(jid,'cancelled')
|
||||
if mode=='wrong-id':self.reply(200,{'task':{'id':'different-job','status':'cancelled'}})
|
||||
elif mode=='transport':self.reply(503,{'error':'transient'})
|
||||
else:
|
||||
job_states[jid]=mode
|
||||
self.reply(200, {'task': {'id':jid, 'status':mode}})
|
||||
else:
|
||||
self.reply(200, {'task_id': 'new-' + str(len(events)), 'routing': {'gpu': body['gpu'], 'port': 8191, 'providerId': 'gpu3', 'legacy': False}})
|
||||
def do_GET(self):
|
||||
if self.path.startswith('/v1/images/task/'):
|
||||
jid=self.path.split('/')[-1];state=job_states.get(jid,'running')
|
||||
self.reply(200,{'task_id':jid,'status':'failed' if state=='cancelled' else 'processing',
|
||||
'cancelled':state=='cancelled','worker_state':'running','error':{'message':'cancelled'}});return
|
||||
self.reply(200, {'task': {'id': self.path.split('/')[-1], 'status': 'running', 'worker_state': 'running',
|
||||
'progress': {'phase':'sampling','step':4,'totalSteps':20,'observedAt':round(time.time()*1000),'samplingMinSeconds':120,'samplingMaxSeconds':180}}})
|
||||
|
||||
@@ -119,8 +128,49 @@ try:
|
||||
time.sleep(.1)
|
||||
assert progress['step']==4 and progress['totalSteps']==20
|
||||
assert progress['samplingMaxSeconds']==180
|
||||
# Standalone cancellation never creates another generation or touches neighbors.
|
||||
before=len(events);assert api('POST','/tasks/2/cancel',{})['outcome']=='cancelled'
|
||||
assert events[-1][0]=='/v2/video_generation/old-2/cancel' and len(events)==before+1
|
||||
assert sql('SELECT status FROM sys_task WHERE id=3')[0]['status']=='processing'
|
||||
before=len(events);assert api('POST','/tasks/2/cancel',{})['outcome']=='cancelled';assert len(events)==before
|
||||
sql('INSERT INTO ai_service_configs(id,service_type,provider,name,base_url,api_key,model,is_active,created_at,updated_at) VALUES(2,?,?,?,?,?,?,1,?,?)',
|
||||
('image','openai','fixture-image','http://127.0.0.1:18935/v1','test-key',json.dumps(['qwen_image_2_1']),ts,ts))
|
||||
for i in [20,21,22,23]:
|
||||
sql('INSERT INTO sys_task(id,type,provider,model,status,stage,task_id,config_id,config_base_url,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)',
|
||||
(i,'image','openai','qwen_image_2_1','processing','submitting' if i==23 else 'queued',None if i==23 else 'image-'+str(i),2,'http://127.0.0.1:18935/v1',ts,ts))
|
||||
sql('INSERT INTO sys_task(id,type,status,stage,local_path,created_at,updated_at) VALUES(19,?,?,?,?,?,?)',('image','completed','completed','static/images/reference.png',ts,ts))
|
||||
assert api('GET','/tasks/20')['canCancel'] is True
|
||||
before=len(events);assert api('POST','/tasks/20/cancel',{})['outcome']=='cancelled'
|
||||
assert events[-1][0]=='/v1/images/task/image-20/cancel' and len(events)==before+1
|
||||
assert sql('SELECT local_path FROM sys_task WHERE id=19')[0]['local_path']=='static/images/reference.png'
|
||||
assert (data/'static/images/reference.png').read_bytes()==png
|
||||
# Wrong receipt / transport failures are unconfirmed, never falsely cancelled.
|
||||
cancel_modes['image-21']='wrong-id'
|
||||
assert api('POST','/tasks/21/cancel',{})['outcome']=='pending'
|
||||
assert sql('SELECT status,stage FROM sys_task WHERE id=21')[0]=={'status':'processing','stage':'cancelling'}
|
||||
cancel_modes['image-21']='transport'
|
||||
assert api('POST','/tasks/21/cancel',{})['outcome']=='pending'
|
||||
# A completion racing with cancellation is preserved for normal download.
|
||||
cancel_modes['image-22']='succeeded'
|
||||
assert api('POST','/tasks/22/cancel',{})['outcome']=='completed'
|
||||
assert sql('SELECT error_code,status FROM sys_task WHERE id=22')[0]=={'error_code':None,'status':'processing'}
|
||||
before=len(events);api('POST','/tasks/23/cancel',{},expected=400);api('POST','/tasks/19/cancel',{},expected=400)
|
||||
assert len(events)==before and api('GET','/tasks/23')['canCancel'] is False
|
||||
sql('INSERT INTO sys_task(id,type,status,stage,created_at,updated_at) VALUES(24,?,?,?,?,?)',('video','processing','waiting_predecessor',ts,ts))
|
||||
assert api('POST','/tasks/24/cancel',{})['outcome']=='cancelled' and len(events)==before
|
||||
# The unconfirmed request survives restart; polling confirms the same ID,
|
||||
# without another cancel or generation request.
|
||||
job_states['image-21']='cancelled'
|
||||
process.terminate();process.wait(timeout=10)
|
||||
process=subprocess.Popen([str(stage/'HuobaoDrama.exe'),str(data/'backend.mjs')],cwd=data,env=env,stdout=log,stderr=log,creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
for _ in range(180):
|
||||
row=sql('SELECT status,error_code FROM sys_task WHERE id=21')[0]
|
||||
if row['error_code']=='user_cancelled':break
|
||||
time.sleep(.1)
|
||||
assert row=={'status':'failed','error_code':'user_cancelled'} and len(events)==before
|
||||
print(json.dumps({'passed':True,'inline_reference_preflight':True,'first_frame':True,'cancel_only_selected_shot':True,
|
||||
'preserve_old_on_missing_image':True,'preserve_old_on_validation_failure':True,'fixture':str(data)}))
|
||||
'preserve_old_on_missing_image':True,'preserve_old_on_validation_failure':True,'image_cancel':True,'cancel_idempotent':True,
|
||||
'completion_race_preserved':True,'bad_receipt_not_cancelled':True,'cancel_survives_restart':True,'fixture':str(data)}))
|
||||
finally:
|
||||
if process and process.poll() is None: process.terminate(); process.wait(timeout=10)
|
||||
server.shutdown(); log.close()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Serve built frontend with a read-only proxy to verify progress; no DB writes."""
|
||||
import http.server,json,pathlib,subprocess,threading,urllib.request,urllib.error
|
||||
import http.server,json,pathlib,subprocess,threading,urllib.request,urllib.error,sys
|
||||
root=pathlib.Path(__file__).resolve().parent.parent
|
||||
public=root/'huobao-source/frontend/.output/public'
|
||||
upstream=json.loads((root/'huobao-data/last-launch.json').read_text())['api'].removesuffix('/api/v1')
|
||||
@@ -18,5 +18,7 @@ class ReadOnly(http.server.SimpleHTTPRequestHandler):
|
||||
super().do_GET()
|
||||
server=http.server.ThreadingHTTPServer(('127.0.0.1',18938),ReadOnly)
|
||||
threading.Thread(target=server.serve_forever,daemon=True).start()
|
||||
try:subprocess.run(['node',str(root/'integration-work/browser-test/verify_progress_ui.cjs')],cwd=root,check=True)
|
||||
script = sys.argv[1] if len(sys.argv)>1 else 'verify_progress_ui.cjs'
|
||||
if script not in {'verify_progress_ui.cjs','verify_cancel_ui.cjs'}:raise ValueError('Unknown UI test')
|
||||
try:subprocess.run(['node',str(root/'integration-work/browser-test'/script)],cwd=root,check=True)
|
||||
finally:server.shutdown()
|
||||
|
||||
Reference in New Issue
Block a user