"""Restartable knowledge processing worker. Run separately from the HTTP server.""" from __future__ import annotations import argparse import asyncio import json import time import threading from collections import Counter from concurrent.futures import ThreadPoolExecutor from pathlib import Path from datetime import datetime, timedelta, timezone import admin_backend from archive_store import json_text, new_id, utc_now from knowledge_processing import classify, feed, finish from knowledge_model_output import ModelDiscard, ModelOutputError, normalize_output, original_draft from knowledge_store import AUTO_RUN, KnowledgeStore, job_scope, load_config # 每步读多少条消息。规则整理很快;模型整理一步读 200 条,攒出足够的问答给并发调用。 SCAN_BATCH = 200 LEASE_SECONDS = 300 def enhance(candidate, database, options=None, *, tenant_id="default", task_id=""): """Optional extraction via the chosen model, using only redacted evidence. 每次调用自己解析出口、自己开事件循环:并发时各线程互不共享连接和信号量。 模型判定没有可复用知识时抛 ModelDiscard;格式不对抛 ModelOutputError;其余失败抛 ValueError。 """ from model_usage import UsageRecorder from knowledge_models import extract, resolve_model, safe_model_error import httpx options = dict(options or {}) outlet = resolve_model(database, options) deadline = options['model_timeout_seconds'] # A separate catalog instance: task timeouts do not change interactive chat settings. outlet.config = {**outlet.config, 'timeout_ms': deadline * 1000} allow_discard = bool(options.get('model_filter', True)) async def run(): async with UsageRecorder(database, tenant_id=tenant_id, task_id=task_id, purpose="knowledge") as recorder: async with httpx.AsyncClient() as client: return await extract(client, outlet, candidate, deadline=deadline, allow_discard=allow_discard, usage_recorder=recorder) try: result, prompt_chars = asyncio.run(run()) except (httpx.HTTPError, TimeoutError) as exc: raise ValueError(safe_model_error(type(exc).__name__)) from None if result.get("error"): raise ValueError(safe_model_error(result['error'])) return normalize_output(candidate, result.get('text'), prompt_chars, allow_discard=allow_discard) class KnowledgeWorker: def __init__(self, store): self.store = store self.owner = new_id() def claim(self): now = time.time() with self.store.database.connect() as db: db.execute("BEGIN IMMEDIATE") job = db.execute("SELECT * FROM knowledge_job WHERE status='queued' OR " "(status='running' AND lease_until(?,?,?,?)") args.extend([cursor["source_account_id"], cursor["conversation_id"], cursor["sent_at"], cursor["id"]]) else: where.append("(m.conversation_id,m.sent_at,m.id)>(?,?,?)") args.extend([cursor["conversation_id"], cursor["sent_at"], cursor["id"]]) with self.store.database.connect() as db: rows = db.execute("""SELECT m.id,m.source_account_id,m.conversation_id,m.sender_person_id,m.message_type, m.direction,m.sent_at,m.sent_at_epoch,v.content,v.status,v.version_no,p.display_name sender_name FROM archive_message m JOIN archive_conversation c ON c.id=m.conversation_id LEFT JOIN archive_person p ON p.id=m.sender_person_id JOIN archive_message_version v ON v.message_id=m.id AND v.version_no=( SELECT MAX(v2.version_no) FROM archive_message_version v2 WHERE v2.message_id=m.id AND v2.created_at<=?) WHERE """ + " AND ".join(where) + " ORDER BY m.source_account_id,m.conversation_id,m.sent_at,m.id LIMIT ?", [job["cutoff_at"], *args, batch_size]).fetchall() # 单聊里收发方向以归档为准:桌面端导入时已按"发送者是不是本账号"判定过。以前再要求 # 发送者身份能对上来源账号,身份没关联的账号几乎所有客服回复都被当成"方向不明"丢掉了。 return [dict(raw) for raw in rows] def enqueue_incremental(self): # Wait for a quiet period so a live, still-incomplete answer is not extracted. cutoff = (datetime.now(timezone.utc) - timedelta(minutes=30)).isoformat(timespec="milliseconds") config = load_config(self.store.database) with self.store.database.connect() as db: db.execute("BEGIN IMMEDIATE") dirty = db.execute("""SELECT d.*,w.created_by FROM knowledge_dirty d JOIN knowledge_watch w ON w.tenant_id=d.tenant_id AND w.source_account_id=d.source_account_id WHERE w.enabled=1 AND d.changed_at 0 else [] rows = fetched[:size] skipped = Counter() for row in rows: use, reason = classify(row) if use == 'skip': skipped[reason] += 1 state, emitted = feed(state, row) pending.extend(emitted) natural_end = size > 0 and len(fetched) <= size if natural_end: last = finish(state) if last: pending.append(last) state = {} done = natural_end or (remaining is not None and len(rows) >= remaining) if done: options['completion_reason'] = 'source_exhausted' if natural_end else 'message_limit' state = {} reasons = options.setdefault('skip_reasons', {}) for reason, count in skipped.items(): reasons[reason] = reasons.get(reason, 0) + count cursor = ({k: rows[-1][k] for k in ('source_account_id', 'conversation_id', 'sent_at', 'id')} if rows else json.loads(job['cursor_json'])) # Persist the scan and redacted candidates before any paid request. # Pending candidates survive failure, pause and process restarts. with self.store.database.connect() as db: updated = db.execute("""UPDATE knowledge_job SET cursor_json=?,state_json=?, processed=processed+?,skipped=skipped+?,options_json=?,lease_until=?,updated_at=? WHERE id=? AND status='running' AND lease_owner=?""", ( json_text(cursor), json_text({**state, '_pending': pending, '_scan_done': done}), len(rows), sum(skipped.values()), json_text(options), time.time() + LEASE_SECONDS, utc_now(), job['id'], self.owner)) return (state, pending, done) if updated.rowcount else None def save_rules(self, job, options, state, pending, done): """规则草稿不花钱:一批一个事务存完。""" with self.store.database.connect() as db: db.execute('BEGIN IMMEDIATE') if not self._owned(db, job): return False counts = Counter(self.store.add_draft(db, job['tenant_id'], job['id'], candidate) for candidate in pending) options['updated_items'] = options.get('updated_items', 0) + counts['updated'] db.execute("""UPDATE knowledge_job SET state_json=?,created_items=created_items+?,duplicates=duplicates+?, options_json=?,updated_at=? WHERE id=?""", (json_text({**state, '_pending': [], '_scan_done': done}), counts['created'], counts[''], json_text(options), utc_now(), job['id'])) return True def _budget(self, db, job, options, wanted): """这一轮还能发几次模型请求。批次有总上限时按整个批次算,旧任务按单个任务算;0 = 不限。""" run_id = job.get('run_id') or '' if run_id and run_id != AUTO_RUN: run = db.execute("SELECT options_json FROM knowledge_run WHERE id=?", (run_id,)).fetchone() cap = int(json.loads(run[0]).get('max_model_calls') or 0) if run else 0 used = db.execute("SELECT COALESCE(SUM(model_calls),0) FROM knowledge_job WHERE run_id=?", (run_id,)).fetchone()[0] else: cap = int(options.get('max_model_calls') or 0) used = db.execute("SELECT model_calls FROM knowledge_job WHERE id=?", (job['id'],)).fetchone()[0] if not cap: return wanted if used >= cap: raise ValueError('模型调用预算已用完;已保存进度,请提高调用总上限(或填 0 不限)或改用规则整理后继续') return min(wanted, cap - used) def _call(self, job, options, candidate): try: return 'ok', enhance(candidate, self.store.database, options, tenant_id=job["tenant_id"], task_id=job["id"]) except ModelDiscard as exc: return 'discard', exc except ModelOutputError as exc: return 'format', exc except ValueError as exc: return 'error', exc except Exception as exc: # noqa: BLE001 - surfaced as the job error, never the provider body return 'error', ValueError(type(exc).__name__) def save_model(self, job, options, state, pending, done): """并发调用模型整理候选问答(默认 4 路);每一轮成功的结果先落库,再处理失败。""" concurrency = max(1, min(16, int(options.get('model_concurrency') or 1))) with ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix='knowledge-model') as pool: while pending: with self.store.database.connect() as db: db.execute('BEGIN IMMEDIATE') if not self._owned(db, job): return False take = self._budget(db, job, options, min(concurrency, len(pending))) # Every call is bounded at <=181s, below the renewed lease. db.execute('UPDATE knowledge_job SET model_calls=model_calls+?,lease_until=? WHERE id=?', (take, time.time() + LEASE_SECONDS, job['id'])) chunk = pending[:take] results = list(pool.map(lambda candidate: self._call(job, options, candidate), chunk)) error = None with self.store.database.connect() as db: db.execute('BEGIN IMMEDIATE') if not self._owned(db, job): return False counts, chars, kept = Counter(), [0, 0], [] for candidate, (kind, value) in zip(chunk, results): if kind == 'error': # Transport/auth/budget errors stop the job; the candidate stays pending. error = error or value kept.append(candidate) continue if kind == 'ok': draft, input_chars, output_chars = value elif kind == 'discard': draft, input_chars, output_chars = None, value.input_chars, value.output_chars counts['filtered'] += 1 options['model_last_filtered'] = value.reason else: # A bad response affects this draft only. draft, input_chars, output_chars = original_draft(candidate, value), value.input_chars, value.output_chars options['model_format_fallbacks'] = options.get('model_format_fallbacks', 0) + 1 options['model_last_warning'] = str(value) chars[0] += input_chars chars[1] += output_chars if draft is not None: counts[self.store.add_draft(db, job['tenant_id'], job['id'], draft, replace=True)] += 1 pending = kept + pending[take:] options['updated_items'] = options.get('updated_items', 0) + counts['updated'] options['model_filtered'] = options.get('model_filtered', 0) + counts['filtered'] db.execute("""UPDATE knowledge_job SET state_json=?,created_items=created_items+?,duplicates=duplicates+?, input_chars=input_chars+?,output_chars=output_chars+?,options_json=?,updated_at=? WHERE id=?""", (json_text({**state, '_pending': pending, '_scan_done': done}), counts['created'], counts[''], chars[0], chars[1], json_text(options), utc_now(), job['id'])) if error is not None: raise error return True def step(self): job = self.claim() if job is None: return self.enqueue_incremental() try: options = json.loads(job['options_json']) if options['engine'] == 'model' and not options.get('model_provider_id'): from knowledge_models import resolve_model resolve_model(self.store.database, options) state = json.loads(job['state_json']) pending = state.pop('_pending', []) done = state.pop('_scan_done', False) if not pending and not done: scanned = self.scan(job, options, state, pending) if scanned is None: return True state, pending, done = scanned if pending: saved = (self.save_model(job, options, state, pending, done) if options['engine'] == 'model' else self.save_rules(job, options, state, pending, done)) if not saved: return True with self.store.database.connect() as db: db.execute("""UPDATE knowledge_job SET status=?,state_json=?,lease_owner='',lease_until=0,error='', options_json=?,updated_at=? WHERE id=? AND status='running' AND lease_owner=?""", ('completed' if done else 'queued', json_text({} if done else state), json_text(options), utc_now(), job['id'], self.owner)) except Exception as exc: # Only local validation messages may be persisted; provider bodies stay private. error = str(exc)[:240] if isinstance(exc, ValueError) and not isinstance(exc, json.JSONDecodeError) else type(exc).__name__ with self.store.database.connect() as db: db.execute("UPDATE knowledge_job SET status='failed',error=?,lease_owner='',lease_until=0,updated_at=? " "WHERE id=? AND lease_owner=? AND status='running'", (error, utc_now(), job['id'], self.owner)) return True def reindex(store, tenant): from archive_store import safe_scope from knowledge_retriever import VectorIndex tenant = safe_scope(tenant) index = VectorIndex() if not index.configured: raise ValueError("请先配置语义检索服务") cursor, count = '', 0 while True: with store.database.connect() as db: rows = [dict(r) for r in db.execute("SELECT * FROM knowledge_item WHERE tenant_id=? AND status='published' " "AND id>? ORDER BY id LIMIT 100", (tenant, cursor))] if not rows: return count cursor = rows[-1]['id'] for item in rows: profile = index.publish(item) with store.database.connect() as db: db.execute("BEGIN IMMEDIATE") if store._fresh(db, item['id'], tenant): result = db.execute("UPDATE knowledge_item SET vector_status='ready',vector_profile=? " "WHERE id=? AND tenant_id=? AND revision=? AND status='published'", (profile, item['id'], tenant, item['revision'])) count += result.rowcount print(f"已重建 {count} 条向量索引", flush=True) def main(): parser = argparse.ArgumentParser(description="聊天知识加工 Worker") parser.add_argument("--db", default="backend.db") parser.add_argument("--once", action="store_true", help="处理一个批次后退出") parser.add_argument("--reindex", action="store_true", help="为已发布知识重建向量索引") parser.add_argument("--tenant", default="", help="重建索引时指定租户") args = parser.parse_args() if args.reindex and not args.tenant: parser.error("--reindex 必须指定 --tenant") database = admin_backend.Database(Path(args.db).resolve()) database.migrate() store = KnowledgeStore(database) store.initialize() if args.reindex: reindex(store, args.tenant) return from knowledge_review import KnowledgeReviewWorker review_worker = KnowledgeReviewWorker(store) def review_loop(): while True: try: worked = review_worker.step() except Exception as exc: # Keep processing isolated from model requests; do not log content. print(f'批量审核暂不可用: {type(exc).__name__}', flush=True) worked = False time.sleep(0.05 if worked else 2) if args.once: review_worker.step() else: threading.Thread(target=review_loop, name='knowledge-review', daemon=True).start() worker = KnowledgeWorker(store) print("知识加工 Worker 已启动", flush=True) while True: worked = worker.step() if args.once: break if not worked: time.sleep(2) if __name__ == "__main__": main()