121 lines
8.1 KiB
Python
121 lines
8.1 KiB
Python
from pathlib import Path
|
|
p=Path('C:/kefu/wechat_rpa/knowledge_worker.py')
|
|
s=p.read_text(encoding='utf-8')
|
|
s=s.replace('def enhance(candidate, database):','def enhance(candidate, database, options=None):')
|
|
start=s.index(' from model_gateway import Catalog, call_outlet')
|
|
end=s.index(' prompt = ',start)
|
|
s=s[:start]+''' from model_gateway import call_outlet
|
|
from knowledge_models import resolve_model, safe_model_error
|
|
import httpx
|
|
options = dict(options or {})
|
|
outlet = resolve_model(database, options)
|
|
deadline = options['model_timeout_seconds']
|
|
# A separate catalog instance: task timeouts do not change interactive chat settings.
|
|
outlet.config = {**outlet.config, 'timeout_ms': deadline * 1000}
|
|
'''+s[end:]
|
|
s=s.replace('return await call_outlet(client, outlets[0], [{"role": "user", "content": prompt}], deadline=12)', '''return await call_outlet(client, outlet, [{"role": "user", "content": prompt}],
|
|
deadline=deadline, max_tokens=4096, temperature=0.1)''')
|
|
s=s.replace(' result = asyncio.run(run())\n if result.get("error") or not result.get("text"):\n raise ValueError("模型整理失败，可重试或使用规则整理")', ''' try:
|
|
result = asyncio.run(run())
|
|
except (httpx.HTTPError, TimeoutError) as exc:
|
|
raise ValueError(safe_model_error(type(exc).__name__)) from None
|
|
if result.get("error") or not result.get("text"):
|
|
raise ValueError(safe_model_error(result.get('error', '')))''')
|
|
s=s.replace(' data = json.loads(raw)', ''' try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
raise ValueError('模型输出不是完整 JSON，可能被截断或未遵循格式；请更换模型或检查上游输出限制') from None
|
|
if not isinstance(data, dict):
|
|
raise ValueError('模型输出应为 JSON 对象，请更换模型或检查模型提示词')''')
|
|
start=s.index(' def step(self):')
|
|
end=s.index('\n\ndef reindex',start)
|
|
s=s[:start]+''' def step(self):
|
|
job = self.claim()
|
|
if job is None:
|
|
return self.enqueue_incremental()
|
|
try:
|
|
options = json.loads(job['options_json'])
|
|
if options['engine'] == 'model' and not options.get('model_provider_id'):
|
|
from knowledge_models import resolve_model
|
|
resolve_model(self.store.database, options)
|
|
state = json.loads(job['state_json'])
|
|
pending = state.pop('_pending', [])
|
|
done = state.pop('_scan_done', False)
|
|
if not pending and not done:
|
|
remaining = options['max_messages'] - job['processed']
|
|
size = min(20 if options['engine'] == 'model' else 200, remaining)
|
|
fetched = self.rows(job, max(1, size) + 1) if remaining > 0 else []
|
|
rows = fetched[:size] if size > 0 else []
|
|
for row in rows:
|
|
state, emitted = feed(state, row)
|
|
pending.extend(emitted)
|
|
natural_end = remaining > 0 and len(fetched) <= size
|
|
if natural_end:
|
|
last = finish(state)
|
|
if last:
|
|
pending.append(last)
|
|
state = {}
|
|
done = natural_end or len(rows) >= remaining
|
|
if done:
|
|
options['completion_reason'] = 'source_exhausted' if natural_end else 'message_limit'
|
|
state = {}
|
|
cursor = {k: rows[-1][k] for k in ('conversation_id', 'sent_at', 'id')} if rows else json.loads(job['cursor_json'])
|
|
# Persist the scan and redacted candidates before any paid request.
|
|
# Pending candidates survive failure, pause and process restarts.
|
|
with self.store.database.connect() as db:
|
|
updated = db.execute("""UPDATE knowledge_job SET cursor_json=?,state_json=?,
|
|
processed=processed+?,skipped=skipped+?,options_json=?,lease_until=?,updated_at=?
|
|
WHERE id=? AND status='running' AND lease_owner=?""", (
|
|
json_text(cursor), json_text({**state, '_pending': pending, '_scan_done': done}), len(rows),
|
|
sum(r['direction'] == 'unknown' or r['message_type'].lower() not in
|
|
{'text', '文本', '文字', '1'} or r['status'] != 'normal' or not r['content'].strip() for r in rows),
|
|
json_text(options), time.time() + 300, utc_now(), job['id'], self.owner))
|
|
if not updated.rowcount:
|
|
return True
|
|
while pending:
|
|
candidate = pending[0]
|
|
input_chars = output_chars = 0
|
|
with self.store.database.connect() as db:
|
|
db.execute('BEGIN IMMEDIATE')
|
|
current = db.execute("SELECT * FROM knowledge_job WHERE id=? AND lease_owner=? AND status='running'",
|
|
(job['id'], self.owner)).fetchone()
|
|
if current is None:
|
|
return True
|
|
if options['engine'] == 'model':
|
|
if current['model_calls'] >= options['max_model_calls']:
|
|
raise ValueError('模型调用预算已用完；已保存进度，请提高调用总上限或改用规则整理后继续')
|
|
db.execute('UPDATE knowledge_job SET model_calls=model_calls+1 WHERE id=?', (job['id'],))
|
|
# Every call is bounded at <=181s, below the renewed 300s lease.
|
|
db.execute('UPDATE knowledge_job SET lease_until=? WHERE id=?', (time.time() + 300, job['id']))
|
|
if options['engine'] == 'model':
|
|
candidate, input_chars, output_chars = enhance(candidate, self.store.database, options)
|
|
with self.store.database.connect() as db:
|
|
db.execute('BEGIN IMMEDIATE')
|
|
if not db.execute("SELECT 1 FROM knowledge_job WHERE id=? AND status='running' AND lease_owner=?",
|
|
(job['id'], self.owner)).fetchone():
|
|
return True
|
|
created = self.store.add_draft(db, job['tenant_id'], job['id'], candidate)
|
|
pending = pending[1:]
|
|
db.execute("""UPDATE knowledge_job SET state_json=?,created_items=created_items+?,
|
|
duplicates=duplicates+?,input_chars=input_chars+?,output_chars=output_chars+?,updated_at=?
|
|
WHERE id=?""", (json_text({**state, '_pending': pending, '_scan_done': done}),
|
|
int(created), int(not created), input_chars, output_chars, utc_now(), job['id']))
|
|
with self.store.database.connect() as db:
|
|
db.execute("""UPDATE knowledge_job SET status=?,state_json=?,lease_owner='',lease_until=0,error='',
|
|
options_json=?,updated_at=? WHERE id=? AND status='running' AND lease_owner=?""",
|
|
('completed' if done else 'queued', json_text({} if done else state),
|
|
json_text(options), utc_now(), job['id'], self.owner))
|
|
except Exception as exc:
|
|
# Only local validation messages may be persisted; provider bodies stay private.
|
|
error = str(exc)[:240] if isinstance(exc, ValueError) and not isinstance(exc, json.JSONDecodeError) else type(exc).__name__
|
|
with self.store.database.connect() as db:
|
|
db.execute("UPDATE knowledge_job SET status='failed',error=?,lease_owner='',lease_until=0,updated_at=? "
|
|
"WHERE id=? AND lease_owner=? AND status='running'", (error, utc_now(), job['id'], self.owner))
|
|
return True
|
|
'''+s[end:]
|
|
p.write_text(s,encoding='utf-8',newline='\n')
|
|
# Public progress includes waiting candidates without exposing their content.
|
|
p=Path('C:/kefu/wechat_rpa/knowledge_store.py'); s=p.read_text(encoding='utf-8')
|
|
s=s.replace(' result.pop("state", None)', ' result["pending_items"] = len(result.get("state", {}).get("_pending", []))\n result.pop("state", None)')
|
|
p.write_text(s,encoding='utf-8',newline='\n')
|