Add per-task image and video cancellation with confirmed worker receipts

integration-work/test_gpu_replacement_references.py

@@ -21,7 +21,7 @@ assert count == 1
 source, count = re.subn(r'&& !!taskRouting\(\{ provider: config\d*\.provider, taskId \}\)', '&& !!taskId', source)
 assert count == 1
 (data / 'backend.mjs').write_text(source, 'utf-8')
-events = []; reject = False; process = None
+events = []; reject = False; process = None; cancel_modes = {}; job_states = {}
 
 class Mock(BaseHTTPRequestHandler):
     def log_message(self, *args): pass
@@ -37,10 +37,19 @@ class Mock(BaseHTTPRequestHandler):
                 self.reply(400, {'error': {'message': 'Local video requires inline image data'}})
             else: self.reply(200, {'valid': True})
         elif self.path.endswith('/cancel'):
-            self.reply(200, {'task': {'id': self.path.split('/')[-2], 'status': 'cancelled'}})
+            jid=self.path.split('/')[-2];mode=cancel_modes.get(jid,'cancelled')
+            if mode=='wrong-id':self.reply(200,{'task':{'id':'different-job','status':'cancelled'}})
+            elif mode=='transport':self.reply(503,{'error':'transient'})
+            else:
+                job_states[jid]=mode
+                self.reply(200, {'task': {'id':jid, 'status':mode}})
         else:
             self.reply(200, {'task_id': 'new-' + str(len(events)), 'routing': {'gpu': body['gpu'], 'port': 8191, 'providerId': 'gpu3', 'legacy': False}})
     def do_GET(self):
+        if self.path.startswith('/v1/images/task/'):
+            jid=self.path.split('/')[-1];state=job_states.get(jid,'running')
+            self.reply(200,{'task_id':jid,'status':'failed' if state=='cancelled' else 'processing',
+                'cancelled':state=='cancelled','worker_state':'running','error':{'message':'cancelled'}});return
         self.reply(200, {'task': {'id': self.path.split('/')[-1], 'status': 'running', 'worker_state': 'running',
             'progress': {'phase':'sampling','step':4,'totalSteps':20,'observedAt':round(time.time()*1000),'samplingMinSeconds':120,'samplingMaxSeconds':180}}})
 
@@ -119,8 +128,49 @@ try:
         time.sleep(.1)
     assert progress['step']==4 and progress['totalSteps']==20
     assert progress['samplingMaxSeconds']==180
+    # Standalone cancellation never creates another generation or touches neighbors.
+    before=len(events);assert api('POST','/tasks/2/cancel',{})['outcome']=='cancelled'
+    assert events[-1][0]=='/v2/video_generation/old-2/cancel' and len(events)==before+1
+    assert sql('SELECT status FROM sys_task WHERE id=3')[0]['status']=='processing'
+    before=len(events);assert api('POST','/tasks/2/cancel',{})['outcome']=='cancelled';assert len(events)==before
+    sql('INSERT INTO ai_service_configs(id,service_type,provider,name,base_url,api_key,model,is_active,created_at,updated_at) VALUES(2,?,?,?,?,?,?,1,?,?)',
+        ('image','openai','fixture-image','http://127.0.0.1:18935/v1','test-key',json.dumps(['qwen_image_2_1']),ts,ts))
+    for i in [20,21,22,23]:
+        sql('INSERT INTO sys_task(id,type,provider,model,status,stage,task_id,config_id,config_base_url,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)',
+            (i,'image','openai','qwen_image_2_1','processing','submitting' if i==23 else 'queued',None if i==23 else 'image-'+str(i),2,'http://127.0.0.1:18935/v1',ts,ts))
+    sql('INSERT INTO sys_task(id,type,status,stage,local_path,created_at,updated_at) VALUES(19,?,?,?,?,?,?)',('image','completed','completed','static/images/reference.png',ts,ts))
+    assert api('GET','/tasks/20')['canCancel'] is True
+    before=len(events);assert api('POST','/tasks/20/cancel',{})['outcome']=='cancelled'
+    assert events[-1][0]=='/v1/images/task/image-20/cancel' and len(events)==before+1
+    assert sql('SELECT local_path FROM sys_task WHERE id=19')[0]['local_path']=='static/images/reference.png'
+    assert (data/'static/images/reference.png').read_bytes()==png
+    # Wrong receipt / transport failures are unconfirmed, never falsely cancelled.
+    cancel_modes['image-21']='wrong-id'
+    assert api('POST','/tasks/21/cancel',{})['outcome']=='pending'
+    assert sql('SELECT status,stage FROM sys_task WHERE id=21')[0]=={'status':'processing','stage':'cancelling'}
+    cancel_modes['image-21']='transport'
+    assert api('POST','/tasks/21/cancel',{})['outcome']=='pending'
+    # A completion racing with cancellation is preserved for normal download.
+    cancel_modes['image-22']='succeeded'
+    assert api('POST','/tasks/22/cancel',{})['outcome']=='completed'
+    assert sql('SELECT error_code,status FROM sys_task WHERE id=22')[0]=={'error_code':None,'status':'processing'}
+    before=len(events);api('POST','/tasks/23/cancel',{},expected=400);api('POST','/tasks/19/cancel',{},expected=400)
+    assert len(events)==before and api('GET','/tasks/23')['canCancel'] is False
+    sql('INSERT INTO sys_task(id,type,status,stage,created_at,updated_at) VALUES(24,?,?,?,?,?)',('video','processing','waiting_predecessor',ts,ts))
+    assert api('POST','/tasks/24/cancel',{})['outcome']=='cancelled' and len(events)==before
+    # The unconfirmed request survives restart; polling confirms the same ID,
+    # without another cancel or generation request.
+    job_states['image-21']='cancelled'
+    process.terminate();process.wait(timeout=10)
+    process=subprocess.Popen([str(stage/'HuobaoDrama.exe'),str(data/'backend.mjs')],cwd=data,env=env,stdout=log,stderr=log,creationflags=subprocess.CREATE_NO_WINDOW)
+    for _ in range(180):
+        row=sql('SELECT status,error_code FROM sys_task WHERE id=21')[0]
+        if row['error_code']=='user_cancelled':break
+        time.sleep(.1)
+    assert row=={'status':'failed','error_code':'user_cancelled'} and len(events)==before
     print(json.dumps({'passed':True,'inline_reference_preflight':True,'first_frame':True,'cancel_only_selected_shot':True,
-        'preserve_old_on_missing_image':True,'preserve_old_on_validation_failure':True,'fixture':str(data)}))
+        'preserve_old_on_missing_image':True,'preserve_old_on_validation_failure':True,'image_cancel':True,'cancel_idempotent':True,
+        'completion_race_preserved':True,'bad_receipt_not_cancelled':True,'cancel_survives_restart':True,'fixture':str(data)}))
 finally:
     if process and process.poll() is None: process.terminate(); process.wait(timeout=10)
     server.shutdown(); log.close()