378 lines
21 KiB
Python
378 lines
21 KiB
Python
"""Restartable knowledge processing worker. Run separately from the HTTP server."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import threading
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import admin_backend
|
|
from archive_store import json_text, new_id, utc_now
|
|
from knowledge_processing import classify, feed, finish
|
|
from knowledge_model_output import ModelDiscard, ModelOutputError, normalize_output, original_draft
|
|
from knowledge_store import AUTO_RUN, KnowledgeStore, job_scope, load_config
|
|
|
|
# 每步读多少条消息。规则整理很快;模型整理一步读 200 条,攒出足够的问答给并发调用。
|
|
SCAN_BATCH = 200
|
|
LEASE_SECONDS = 300
|
|
|
|
|
|
def enhance(candidate, database, options=None, *, tenant_id="default", task_id=""):
|
|
"""Optional extraction via the chosen model, using only redacted evidence.
|
|
|
|
每次调用自己解析出口、自己开事件循环:并发时各线程互不共享连接和信号量。
|
|
模型判定没有可复用知识时抛 ModelDiscard;格式不对抛 ModelOutputError;其余失败抛 ValueError。
|
|
"""
|
|
from model_usage import UsageRecorder
|
|
from knowledge_models import extract, resolve_model, safe_model_error
|
|
import httpx
|
|
options = dict(options or {})
|
|
outlet = resolve_model(database, options)
|
|
deadline = options['model_timeout_seconds']
|
|
# A separate catalog instance: task timeouts do not change interactive chat settings.
|
|
outlet.config = {**outlet.config, 'timeout_ms': deadline * 1000}
|
|
allow_discard = bool(options.get('model_filter', True))
|
|
|
|
async def run():
|
|
async with UsageRecorder(database, tenant_id=tenant_id, task_id=task_id,
|
|
purpose="knowledge") as recorder:
|
|
async with httpx.AsyncClient() as client:
|
|
return await extract(client, outlet, candidate, deadline=deadline,
|
|
allow_discard=allow_discard, usage_recorder=recorder)
|
|
try:
|
|
result, prompt_chars = asyncio.run(run())
|
|
except (httpx.HTTPError, TimeoutError) as exc:
|
|
raise ValueError(safe_model_error(type(exc).__name__)) from None
|
|
if result.get("error"):
|
|
raise ValueError(safe_model_error(result['error']))
|
|
return normalize_output(candidate, result.get('text'), prompt_chars, allow_discard=allow_discard)
|
|
|
|
|
|
class KnowledgeWorker:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
self.owner = new_id()
|
|
|
|
def claim(self):
|
|
now = time.time()
|
|
with self.store.database.connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
job = db.execute("SELECT * FROM knowledge_job WHERE status='queued' OR "
|
|
"(status='running' AND lease_until<?) ORDER BY created_at LIMIT 1", (now,)).fetchone()
|
|
if job is None:
|
|
return None
|
|
db.execute("UPDATE knowledge_job SET status='running',lease_owner=?,lease_until=?,updated_at=? WHERE id=?",
|
|
(self.owner, now + LEASE_SECONDS, utc_now(), job["id"]))
|
|
return dict(job)
|
|
|
|
def rows(self, job, batch_size):
|
|
options = json.loads(job["options_json"])
|
|
cursor = json.loads(job["cursor_json"])
|
|
where, args = job_scope(job["tenant_id"], options, job["cutoff_at"])
|
|
if cursor:
|
|
# 按扫描索引 (tenant, source, conversation, sent_at, id) 的顺序翻页;不指定来源的任务
|
|
# 若按会话排序,SQLite 每一批都要把整个账号的消息重排一遍。
|
|
if "source_account_id" in cursor:
|
|
where.append("(m.source_account_id,m.conversation_id,m.sent_at,m.id)>(?,?,?,?)")
|
|
args.extend([cursor["source_account_id"], cursor["conversation_id"], cursor["sent_at"], cursor["id"]])
|
|
else:
|
|
where.append("(m.conversation_id,m.sent_at,m.id)>(?,?,?)")
|
|
args.extend([cursor["conversation_id"], cursor["sent_at"], cursor["id"]])
|
|
with self.store.database.connect() as db:
|
|
rows = db.execute("""SELECT m.id,m.source_account_id,m.conversation_id,m.sender_person_id,m.message_type,
|
|
m.direction,m.sent_at,m.sent_at_epoch,v.content,v.status,v.version_no,p.display_name sender_name
|
|
FROM archive_message m JOIN archive_conversation c ON c.id=m.conversation_id
|
|
LEFT JOIN archive_person p ON p.id=m.sender_person_id
|
|
JOIN archive_message_version v ON v.message_id=m.id AND v.version_no=(
|
|
SELECT MAX(v2.version_no) FROM archive_message_version v2
|
|
WHERE v2.message_id=m.id AND v2.created_at<=?)
|
|
WHERE """ + " AND ".join(where) +
|
|
" ORDER BY m.source_account_id,m.conversation_id,m.sent_at,m.id LIMIT ?",
|
|
[job["cutoff_at"], *args, batch_size]).fetchall()
|
|
# 单聊里收发方向以归档为准:桌面端导入时已按"发送者是不是本账号"判定过。以前再要求
|
|
# 发送者身份能对上来源账号,身份没关联的账号几乎所有客服回复都被当成"方向不明"丢掉了。
|
|
return [dict(raw) for raw in rows]
|
|
|
|
def enqueue_incremental(self):
|
|
# Wait for a quiet period so a live, still-incomplete answer is not extracted.
|
|
cutoff = (datetime.now(timezone.utc) - timedelta(minutes=30)).isoformat(timespec="milliseconds")
|
|
config = load_config(self.store.database)
|
|
with self.store.database.connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
dirty = db.execute("""SELECT d.*,w.created_by FROM knowledge_dirty d JOIN knowledge_watch w
|
|
ON w.tenant_id=d.tenant_id AND w.source_account_id=d.source_account_id
|
|
WHERE w.enabled=1 AND d.changed_at<? ORDER BY d.changed_at LIMIT 1""", (cutoff,)).fetchone()
|
|
if dirty is None:
|
|
# 持续整理每个会话一个小任务,留一个月够排查,再久就清掉
|
|
old = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat(timespec="milliseconds")
|
|
db.execute("DELETE FROM knowledge_job WHERE id IN (SELECT id FROM knowledge_job WHERE status='completed' "
|
|
"AND (run_id=? OR json_extract(options_json,'$.incremental') IS 1) AND updated_at<? LIMIT 200)",
|
|
(AUTO_RUN, old))
|
|
return False
|
|
options = {"source_account_id": dirty["source_account_id"], "conversation_id": dirty["conversation_id"],
|
|
"engine": "rules", "max_messages": 0, "max_model_calls": 0,
|
|
"staff_confirmed": True, "incremental": True, "run_id": AUTO_RUN}
|
|
if config.get("incremental_engine") == "model":
|
|
options.update(engine="model", model_provider_id=config.get("default_model_provider_id", ""),
|
|
model_concurrency=config.get("model_concurrency", 4),
|
|
model_filter=config.get("model_filter", True), model_timeout_seconds=90)
|
|
now = utc_now()
|
|
estimate = self.store._scope_counts(db, dirty["tenant_id"], options, now, quality=False)
|
|
options.update(total_messages=estimate["total_messages"], available_messages=estimate["available_messages"],
|
|
source_name=estimate["source_name"])
|
|
db.execute("INSERT INTO knowledge_job(id,tenant_id,options_json,cutoff_at,created_by,created_at,updated_at,run_id) "
|
|
"VALUES (?,?,?,?,?,?,?,?)", (new_id(), dirty["tenant_id"], json_text(options), now,
|
|
dirty["created_by"], now, now, AUTO_RUN))
|
|
db.execute("DELETE FROM knowledge_dirty WHERE tenant_id=? AND source_account_id=? AND conversation_id=?",
|
|
(dirty["tenant_id"], dirty["source_account_id"], dirty["conversation_id"]))
|
|
return True
|
|
|
|
def _owned(self, db, job):
|
|
return db.execute("SELECT * FROM knowledge_job WHERE id=? AND status='running' AND lease_owner=?",
|
|
(job['id'], self.owner)).fetchone()
|
|
|
|
def scan(self, job, options, state, pending):
|
|
"""读一批消息,切成问答候选,连同游标一起落库(付费调用之前先保存进度)。"""
|
|
limit = options.get('max_messages') or 0
|
|
remaining = limit - job['processed'] if limit else None
|
|
size = SCAN_BATCH if remaining is None else min(SCAN_BATCH, remaining)
|
|
fetched = self.rows(job, size + 1) if size > 0 else []
|
|
rows = fetched[:size]
|
|
skipped = Counter()
|
|
for row in rows:
|
|
use, reason = classify(row)
|
|
if use == 'skip':
|
|
skipped[reason] += 1
|
|
state, emitted = feed(state, row)
|
|
pending.extend(emitted)
|
|
natural_end = size > 0 and len(fetched) <= size
|
|
if natural_end:
|
|
last = finish(state)
|
|
if last:
|
|
pending.append(last)
|
|
state = {}
|
|
done = natural_end or (remaining is not None and len(rows) >= remaining)
|
|
if done:
|
|
options['completion_reason'] = 'source_exhausted' if natural_end else 'message_limit'
|
|
state = {}
|
|
reasons = options.setdefault('skip_reasons', {})
|
|
for reason, count in skipped.items():
|
|
reasons[reason] = reasons.get(reason, 0) + count
|
|
cursor = ({k: rows[-1][k] for k in ('source_account_id', 'conversation_id', 'sent_at', 'id')}
|
|
if rows else json.loads(job['cursor_json']))
|
|
# Persist the scan and redacted candidates before any paid request.
|
|
# Pending candidates survive failure, pause and process restarts.
|
|
with self.store.database.connect() as db:
|
|
updated = db.execute("""UPDATE knowledge_job SET cursor_json=?,state_json=?,
|
|
processed=processed+?,skipped=skipped+?,options_json=?,lease_until=?,updated_at=?
|
|
WHERE id=? AND status='running' AND lease_owner=?""", (
|
|
json_text(cursor), json_text({**state, '_pending': pending, '_scan_done': done}), len(rows),
|
|
sum(skipped.values()), json_text(options), time.time() + LEASE_SECONDS, utc_now(), job['id'], self.owner))
|
|
return (state, pending, done) if updated.rowcount else None
|
|
|
|
def save_rules(self, job, options, state, pending, done):
|
|
"""规则草稿不花钱:一批一个事务存完。"""
|
|
with self.store.database.connect() as db:
|
|
db.execute('BEGIN IMMEDIATE')
|
|
if not self._owned(db, job):
|
|
return False
|
|
counts = Counter(self.store.add_draft(db, job['tenant_id'], job['id'], candidate) for candidate in pending)
|
|
options['updated_items'] = options.get('updated_items', 0) + counts['updated']
|
|
db.execute("""UPDATE knowledge_job SET state_json=?,created_items=created_items+?,duplicates=duplicates+?,
|
|
options_json=?,updated_at=? WHERE id=?""", (json_text({**state, '_pending': [], '_scan_done': done}),
|
|
counts['created'], counts[''], json_text(options), utc_now(), job['id']))
|
|
return True
|
|
|
|
def _budget(self, db, job, options, wanted):
|
|
"""这一轮还能发几次模型请求。批次有总上限时按整个批次算,旧任务按单个任务算;0 = 不限。"""
|
|
run_id = job.get('run_id') or ''
|
|
if run_id and run_id != AUTO_RUN:
|
|
run = db.execute("SELECT options_json FROM knowledge_run WHERE id=?", (run_id,)).fetchone()
|
|
cap = int(json.loads(run[0]).get('max_model_calls') or 0) if run else 0
|
|
used = db.execute("SELECT COALESCE(SUM(model_calls),0) FROM knowledge_job WHERE run_id=?", (run_id,)).fetchone()[0]
|
|
else:
|
|
cap = int(options.get('max_model_calls') or 0)
|
|
used = db.execute("SELECT model_calls FROM knowledge_job WHERE id=?", (job['id'],)).fetchone()[0]
|
|
if not cap:
|
|
return wanted
|
|
if used >= cap:
|
|
raise ValueError('模型调用预算已用完;已保存进度,请提高调用总上限(或填 0 不限)或改用规则整理后继续')
|
|
return min(wanted, cap - used)
|
|
|
|
def _call(self, job, options, candidate):
|
|
try:
|
|
return 'ok', enhance(candidate, self.store.database, options, tenant_id=job["tenant_id"], task_id=job["id"])
|
|
except ModelDiscard as exc:
|
|
return 'discard', exc
|
|
except ModelOutputError as exc:
|
|
return 'format', exc
|
|
except ValueError as exc:
|
|
return 'error', exc
|
|
except Exception as exc: # noqa: BLE001 - surfaced as the job error, never the provider body
|
|
return 'error', ValueError(type(exc).__name__)
|
|
|
|
def save_model(self, job, options, state, pending, done):
|
|
"""并发调用模型整理候选问答(默认 4 路);每一轮成功的结果先落库,再处理失败。"""
|
|
concurrency = max(1, min(16, int(options.get('model_concurrency') or 1)))
|
|
with ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix='knowledge-model') as pool:
|
|
while pending:
|
|
with self.store.database.connect() as db:
|
|
db.execute('BEGIN IMMEDIATE')
|
|
if not self._owned(db, job):
|
|
return False
|
|
take = self._budget(db, job, options, min(concurrency, len(pending)))
|
|
# Every call is bounded at <=181s, below the renewed lease.
|
|
db.execute('UPDATE knowledge_job SET model_calls=model_calls+?,lease_until=? WHERE id=?',
|
|
(take, time.time() + LEASE_SECONDS, job['id']))
|
|
chunk = pending[:take]
|
|
results = list(pool.map(lambda candidate: self._call(job, options, candidate), chunk))
|
|
error = None
|
|
with self.store.database.connect() as db:
|
|
db.execute('BEGIN IMMEDIATE')
|
|
if not self._owned(db, job):
|
|
return False
|
|
counts, chars, kept = Counter(), [0, 0], []
|
|
for candidate, (kind, value) in zip(chunk, results):
|
|
if kind == 'error':
|
|
# Transport/auth/budget errors stop the job; the candidate stays pending.
|
|
error = error or value
|
|
kept.append(candidate)
|
|
continue
|
|
if kind == 'ok':
|
|
draft, input_chars, output_chars = value
|
|
elif kind == 'discard':
|
|
draft, input_chars, output_chars = None, value.input_chars, value.output_chars
|
|
counts['filtered'] += 1
|
|
options['model_last_filtered'] = value.reason
|
|
else:
|
|
# A bad response affects this draft only.
|
|
draft, input_chars, output_chars = original_draft(candidate, value), value.input_chars, value.output_chars
|
|
options['model_format_fallbacks'] = options.get('model_format_fallbacks', 0) + 1
|
|
options['model_last_warning'] = str(value)
|
|
chars[0] += input_chars
|
|
chars[1] += output_chars
|
|
if draft is not None:
|
|
counts[self.store.add_draft(db, job['tenant_id'], job['id'], draft, replace=True)] += 1
|
|
pending = kept + pending[take:]
|
|
options['updated_items'] = options.get('updated_items', 0) + counts['updated']
|
|
options['model_filtered'] = options.get('model_filtered', 0) + counts['filtered']
|
|
db.execute("""UPDATE knowledge_job SET state_json=?,created_items=created_items+?,duplicates=duplicates+?,
|
|
input_chars=input_chars+?,output_chars=output_chars+?,options_json=?,updated_at=? WHERE id=?""",
|
|
(json_text({**state, '_pending': pending, '_scan_done': done}), counts['created'], counts[''],
|
|
chars[0], chars[1], json_text(options), utc_now(), job['id']))
|
|
if error is not None:
|
|
raise error
|
|
return True
|
|
|
|
def step(self):
|
|
job = self.claim()
|
|
if job is None:
|
|
return self.enqueue_incremental()
|
|
try:
|
|
options = json.loads(job['options_json'])
|
|
if options['engine'] == 'model' and not options.get('model_provider_id'):
|
|
from knowledge_models import resolve_model
|
|
resolve_model(self.store.database, options)
|
|
state = json.loads(job['state_json'])
|
|
pending = state.pop('_pending', [])
|
|
done = state.pop('_scan_done', False)
|
|
if not pending and not done:
|
|
scanned = self.scan(job, options, state, pending)
|
|
if scanned is None:
|
|
return True
|
|
state, pending, done = scanned
|
|
if pending:
|
|
saved = (self.save_model(job, options, state, pending, done) if options['engine'] == 'model'
|
|
else self.save_rules(job, options, state, pending, done))
|
|
if not saved:
|
|
return True
|
|
with self.store.database.connect() as db:
|
|
db.execute("""UPDATE knowledge_job SET status=?,state_json=?,lease_owner='',lease_until=0,error='',
|
|
options_json=?,updated_at=? WHERE id=? AND status='running' AND lease_owner=?""",
|
|
('completed' if done else 'queued', json_text({} if done else state),
|
|
json_text(options), utc_now(), job['id'], self.owner))
|
|
except Exception as exc:
|
|
# Only local validation messages may be persisted; provider bodies stay private.
|
|
error = str(exc)[:240] if isinstance(exc, ValueError) and not isinstance(exc, json.JSONDecodeError) else type(exc).__name__
|
|
with self.store.database.connect() as db:
|
|
db.execute("UPDATE knowledge_job SET status='failed',error=?,lease_owner='',lease_until=0,updated_at=? "
|
|
"WHERE id=? AND lease_owner=? AND status='running'", (error, utc_now(), job['id'], self.owner))
|
|
return True
|
|
|
|
|
|
def reindex(store, tenant):
|
|
from archive_store import safe_scope
|
|
from knowledge_retriever import VectorIndex
|
|
tenant = safe_scope(tenant)
|
|
index = VectorIndex()
|
|
if not index.configured:
|
|
raise ValueError("请先配置语义检索服务")
|
|
cursor, count = '', 0
|
|
while True:
|
|
with store.database.connect() as db:
|
|
rows = [dict(r) for r in db.execute("SELECT * FROM knowledge_item WHERE tenant_id=? AND status='published' "
|
|
"AND id>? ORDER BY id LIMIT 100", (tenant, cursor))]
|
|
if not rows:
|
|
return count
|
|
cursor = rows[-1]['id']
|
|
for item in rows:
|
|
profile = index.publish(item)
|
|
with store.database.connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
if store._fresh(db, item['id'], tenant):
|
|
result = db.execute("UPDATE knowledge_item SET vector_status='ready',vector_profile=? "
|
|
"WHERE id=? AND tenant_id=? AND revision=? AND status='published'",
|
|
(profile, item['id'], tenant, item['revision']))
|
|
count += result.rowcount
|
|
print(f"已重建 {count} 条向量索引", flush=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="聊天知识加工 Worker")
|
|
parser.add_argument("--db", default="backend.db")
|
|
parser.add_argument("--once", action="store_true", help="处理一个批次后退出")
|
|
parser.add_argument("--reindex", action="store_true", help="为已发布知识重建向量索引")
|
|
parser.add_argument("--tenant", default="", help="重建索引时指定租户")
|
|
args = parser.parse_args()
|
|
if args.reindex and not args.tenant:
|
|
parser.error("--reindex 必须指定 --tenant")
|
|
database = admin_backend.Database(Path(args.db).resolve())
|
|
database.migrate()
|
|
store = KnowledgeStore(database)
|
|
store.initialize()
|
|
if args.reindex:
|
|
reindex(store, args.tenant)
|
|
return
|
|
from knowledge_review import KnowledgeReviewWorker
|
|
review_worker = KnowledgeReviewWorker(store)
|
|
def review_loop():
|
|
while True:
|
|
try:
|
|
worked = review_worker.step()
|
|
except Exception as exc:
|
|
# Keep processing isolated from model requests; do not log content.
|
|
print(f'批量审核暂不可用: {type(exc).__name__}', flush=True)
|
|
worked = False
|
|
time.sleep(0.05 if worked else 2)
|
|
if args.once:
|
|
review_worker.step()
|
|
else:
|
|
threading.Thread(target=review_loop, name='knowledge-review', daemon=True).start()
|
|
worker = KnowledgeWorker(store)
|
|
print("知识加工 Worker 已启动", flush=True)
|
|
while True:
|
|
worked = worker.step()
|
|
if args.once:
|
|
break
|
|
if not worked:
|
|
time.sleep(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|