60 lines
3.9 KiB
Python
60 lines
3.9 KiB
Python
"""Final read-only verification of exact cleanup scope and preserved reference images/services."""
|
|
import importlib.util,json,pathlib,sqlite3,subprocess,time
|
|
ROOT=pathlib.Path('/home/ps/huobao-video-cleanup');OUT=ROOT/'historical-20261112'
|
|
spec=importlib.util.spec_from_file_location('cleaner',ROOT/'comfy_video_cleanup.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
|
bridges={b['name']:b for b in json.loads((ROOT/'config.json').read_text())['bridges']}
|
|
approved=json.loads((ROOT/'historical-approved.json').read_text());results=[]
|
|
for a in approved:
|
|
b=bridges[a['bridge']]
|
|
deadline=time.monotonic()+900
|
|
while True:
|
|
with sqlite3.connect('file:'+b['database']+'?mode=ro',uri=True) as db:
|
|
db.row_factory=sqlite3.Row;r=db.execute('select * from video_cleanup_acks where id=?',(a['id'],)).fetchone()
|
|
if r and r['status']=='cleaned':break
|
|
if time.monotonic()>deadline:raise RuntimeError('Cleanup still pending for task '+str(a['task']))
|
|
time.sleep(5)
|
|
manifest=json.loads(r['manifest'])
|
|
guard=m.docker_python(b['normalizer'],m.NORMALIZER_CODE,{'mode':'guard','id':a['id'],'manifest':manifest})
|
|
assert guard['historyMissing'],'History still present'
|
|
try:m.docker_python(a['worker'],m.WORKER_CODE,{'mode':'inspect','locator':manifest['locator']})
|
|
except RuntimeError as e:
|
|
if str(e)!='FileNotFoundError':raise
|
|
else:raise RuntimeError('Source video still present')
|
|
results.append({'task':a['task'],'historyAbsent':True,'sourceAbsent':True,'sourceBytes':a['fileReceipt']['bytes'],'method':a['method']})
|
|
(OUT/'verification-progress.json').write_text(json.dumps(results,indent=2))
|
|
images=json.loads((OUT/'image-snapshot-before.json').read_text());changed=[];counts={}
|
|
SNAPSHOT=r'''
|
|
import os,json,pathlib
|
|
files={}
|
|
for folder in ('/app/input','/app/output'):
|
|
for root,dirs,names in os.walk(folder,followlinks=False):
|
|
for name in names:
|
|
p=pathlib.Path(root)/name
|
|
if p.suffix.lower() not in ('.png','.jpg','.jpeg','.webp','.bmp','.gif') or p.is_symlink():continue
|
|
s=p.stat();files[str(p)]={'bytes':s.st_size,'mtime':s.st_mtime_ns}
|
|
print('HB_CLEANUP_RESULT:'+json.dumps({'ok':True,'result':files}))
|
|
'''
|
|
for worker,before in images.items():
|
|
after=m.docker_python(worker,SNAPSHOT,{})
|
|
different=[p for p,v in before.items() if after.get(p)!=v]
|
|
changed.extend({'worker':worker,'path':p} for p in different);counts[worker]={'preserved':len(before)-len(different),'new':len(set(after)-set(before))}
|
|
assert not changed,'Reference image snapshot changed'
|
|
identities=json.loads((OUT/'services-before.json').read_text())
|
|
inspection=json.loads(subprocess.check_output(['docker','inspect',*[name.lstrip('/') for name in identities]],text=True))
|
|
after={i['Name']:{'id':i['Id'],'started':i['State']['StartedAt']} for i in inspection}
|
|
assert identities==after,'Model service changed'
|
|
# Read only operator-configured restore snapshots: identify possible stale history resurrection.
|
|
snapshots=[]
|
|
targetprompts={a['manifest']['prompt'] for a in approved}
|
|
for i in inspection:
|
|
env=dict(x.split('=',1) for x in i['Config'].get('Env',[]) if '=' in x)
|
|
destination=env.get('H3_HISTORY_SNAPSHOT')
|
|
if not destination:continue
|
|
mount=next((v for v in i['Mounts'] if v['Destination']==destination),None)
|
|
if not mount:continue
|
|
value=json.loads(pathlib.Path(mount['Source']).read_text());overlap=sorted(set(value.get('history',{}))&targetprompts)
|
|
snapshots.append({'worker':i['Name'],'matchingCleanedPrompts':overlap})
|
|
report={'verified':len(results),'removedBytes':sum(r['sourceBytes'] for r in results),'liveHistoryDeleted':sum(r['method']=='live-history-and-download' for r in results),'alreadyAbsentHistory':sum(r['method']=='unique-byte-identical-orphan' for r in results),'referenceImages':counts,'referenceImagesChanged':changed,'modelServicesUnchanged':True,'restoreSnapshots':snapshots,'tasks':results}
|
|
(OUT/'final-server-report.json').write_text(json.dumps(report,indent=2))
|
|
print(json.dumps({k:v for k,v in report.items() if k!='tasks'}),flush=True)
|