Files
kefu/deploy/knowledge-batch-20261007/server-bundle/payload/wechat_rpa/knowledge_api.py
T
2026-11-16 09:39:42 +08:00

407 lines
20 KiB
Python

"""Tenant-scoped knowledge management routes."""
from __future__ import annotations
import asyncio
import json
import time
from contextlib import contextmanager
from typing import Any, Literal
import httpx
from fastapi import Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field, field_validator
from archive_store import tenant_scope
from knowledge_retriever import KnowledgeRetriever, VectorIndex
from knowledge_store import KnowledgeStore
class JobBody(BaseModel):
source_account_id: str = Field(min_length=1, max_length=64)
conversation_id: str = Field(default="", max_length=64)
date_from: str = ""
date_to: str = ""
engine: Literal["rules", "model"] = "rules"
model_provider_id: str = Field(default="", max_length=128)
model_timeout_seconds: int = Field(default=90, ge=30, le=180)
model_concurrency: int = Field(default=4, ge=1, le=16)
model_filter: bool = True
# 0 = 不限
max_messages: int = Field(default=10000, ge=0, le=100000000)
max_model_calls: int = Field(default=100, ge=0, le=10000000)
staff_confirmed: bool = False
auto_watch: bool = False
class JobConfigBody(BaseModel):
engine: Literal['rules', 'model']
model_provider_id: str = Field(default='', max_length=128)
model_timeout_seconds: int = Field(default=90, ge=30, le=180)
model_concurrency: int = Field(default=4, ge=1, le=16)
model_batch: int | None = Field(default=None, ge=1, le=20)
model_filter: bool = True
max_model_calls: int = Field(default=100, ge=0, le=10000000)
class RunScopeBody(BaseModel):
"""整理范围:不选账号 = 全部授权账号。"""
account_ids: list[int] = Field(default_factory=list, max_length=500)
date_from: str = ""
date_to: str = ""
class RunBody(RunScopeBody):
engine: Literal["rules", "model"] = "model"
model_provider_id: str = Field(default="", max_length=128)
model_timeout_seconds: int = Field(default=90, ge=30, le=180)
model_concurrency: int = Field(default=4, ge=1, le=16)
model_batch: int = Field(default=8, ge=1, le=20)
model_filter: bool = True
max_model_calls: int = Field(default=0, ge=0, le=10000000)
staff_confirmed: bool = False
auto_watch: bool = False
class RunConfigBody(BaseModel):
engine: Literal["rules", "model"]
model_provider_id: str = Field(default="", max_length=128)
model_timeout_seconds: int = Field(default=90, ge=30, le=180)
model_concurrency: int = Field(default=4, ge=1, le=16)
model_batch: int = Field(default=8, ge=1, le=20)
model_filter: bool = True
max_model_calls: int = Field(default=0, ge=0, le=10000000)
class KnowledgeConfigBody(BaseModel):
default_engine: Literal["rules", "model"] = "model"
default_model_provider_id: str = Field(default="", max_length=128)
model_concurrency: int = Field(default=4, ge=1, le=16)
model_batch: int = Field(default=8, ge=1, le=20)
model_filter: bool = True
incremental_engine: Literal["rules", "model"] = "rules"
class ModelTestBody(BaseModel):
model_provider_id: str = Field(default="", max_length=128)
model_timeout_seconds: int = Field(default=90, ge=30, le=180)
class EditBody(BaseModel):
revision: int = Field(ge=1)
title: str = Field(min_length=1, max_length=160)
question: str = Field(min_length=1, max_length=4000)
answer: str = Field(min_length=1, max_length=8000)
conditions: str = Field(min_length=1, max_length=2000)
category: str = Field(default="", max_length=100)
kind: Literal["qa", "procedure", "case"] = "qa"
valid_until: str = ""
class ActionBody(BaseModel):
revision: int = Field(ge=1)
confirmed: bool = False
restore_revision: int = Field(default=0, ge=0)
class SearchBody(BaseModel):
query: str = Field(min_length=2, max_length=2000)
@field_validator('query', mode='before')
@classmethod
def trim_query(cls, value):
return value.strip() if isinstance(value, str) else value
class ReviewConfirmBody(BaseModel):
confirmed: bool = False
class SettingsBody(BaseModel):
enabled: bool
@contextmanager
def errors():
try:
yield
except KeyError as exc:
raise HTTPException(404, str(exc).strip("'")) from exc
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except httpx.HTTPError as exc:
raise HTTPException(502, "知识索引服务连接失败,当前知识未发布,请检查服务配置后重试") from exc
def register_knowledge_routes(app, database, require, client_ip, resolve_admin_tenant, current_desktop):
store = KnowledgeStore(database)
store.initialize()
app.state.knowledge_store = store
def single(principal, account_id):
scope = resolve_admin_tenant(principal, account_id)
if not isinstance(scope, str) or scope == "*":
raise HTTPException(400, "请先选择具体的客户端账号")
return scope
from knowledge_review import KnowledgeReviewStore
reviews = KnowledgeReviewStore(store)
def run_tenants(principal, account_ids):
"""整理范围 → 租户列表。不选账号 = 授权范围内全部有聊天归档的账号。"""
if not account_ids:
return store.scope_tenants(resolve_admin_tenant(principal, -1)), "全部账号"
tenants = []
for account_id in dict.fromkeys(account_ids):
tenant = resolve_admin_tenant(principal, int(account_id))
if isinstance(tenant, str) and tenant != "*":
tenants.append(tenant)
return tenants, f"{len(tenants)} 个账号"
# ── 整理批次:一次整理多个账号、全部聊天 ─────────────────────────────────
@app.post('/api/v2/knowledge/runs/preview')
def preview_run(body: RunScopeBody, principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
tenants, label = run_tenants(principal, body.account_ids)
return {**store.preview_run(tenants, body.model_dump()), 'scope_label': label}
@app.post('/api/v2/knowledge/runs')
def create_run(body: RunBody, request: Request, principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
tenants, label = run_tenants(principal, body.account_ids)
values = body.model_dump(exclude={'account_ids'})
return store.create_run(tenants, values, principal.id, client_ip(request), scope_label=label)
@app.get('/api/v2/knowledge/runs')
def runs(principal=Depends(require('knowledge:read'))):
scope = resolve_admin_tenant(principal, -1)
return {**store.runs(scope), 'auto': store.auto_summary(scope)}
@app.put('/api/v2/knowledge/runs/{run_id}/config')
def configure_run(run_id: str, body: RunConfigBody, request: Request,
principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
return store.configure_run(resolve_admin_tenant(principal, -1), run_id, body.model_dump(),
principal.id, client_ip(request))
@app.post('/api/v2/knowledge/runs/{run_id}/{action}')
def run_action(run_id: str, action: str, request: Request,
principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
return store.run_action(resolve_admin_tenant(principal, -1), run_id, action, principal.id, client_ip(request))
@app.get('/api/v2/knowledge/config')
def get_config(principal=Depends(require('knowledge:read'))):
return store.config()
@app.put('/api/v2/knowledge/config')
def put_config(body: KnowledgeConfigBody, request: Request,
principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
return store.save_config(body.model_dump(), principal.id, client_ip(request))
@app.post('/api/v2/knowledge/models/test')
async def test_model(body: ModelTestBody, principal=Depends(require('knowledge:write'))):
"""用一条虚构问答试一次整理,确认模型能连通、能按格式输出。不读任何真实聊天。"""
from knowledge_model_output import ModelOutputError, normalize_output
from knowledge_models import SAMPLE, extract, resolve_model, safe_model_error
from model_usage import UsageRecorder
options = {'model_provider_id': body.model_provider_id, 'model_timeout_seconds': body.model_timeout_seconds}
try:
outlet = await asyncio.to_thread(resolve_model, database, options)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
outlet.config = {**outlet.config, 'timeout_ms': body.model_timeout_seconds * 1000}
candidate = {**SAMPLE, 'title': SAMPLE['question'], 'conditions': '', 'category': '待分类',
'kind': 'qa', 'flags': [], 'sources': []}
started = time.monotonic()
async with UsageRecorder(database, tenant_id='default', task_id='knowledge-model-test',
purpose='knowledge') as recorder:
async with httpx.AsyncClient() as client:
result, prompt_chars = await extract(client, outlet, candidate, deadline=body.model_timeout_seconds,
allow_discard=False, usage_recorder=recorder)
reply = {'model_provider_id': outlet.id, 'model_name': outlet.name,
'latency_ms': int((time.monotonic() - started) * 1000)}
if result.get('error'):
return {**reply, 'ok': False, 'error': safe_model_error(result['error'])}
try:
draft, _, _ = normalize_output(candidate, result.get('text'), prompt_chars)
except ModelOutputError as exc:
return {**reply, 'ok': False, 'error': f'模型能连通,但输出不符合整理格式:{exc}'}
return {**reply, 'ok': True, 'sample': SAMPLE,
'draft': {key: draft[key] for key in ('title', 'question', 'answer', 'conditions', 'category', 'kind')}}
@app.post('/api/v2/knowledge/review-tasks/preview')
def preview_reviews(request: Request, account_id: int = -1,
principal=Depends(require('knowledge:read', 'knowledge:review', 'im:content:read'))):
with errors():
return reviews.preview(resolve_admin_tenant(principal, account_id), account_id, principal.id, client_ip(request))
@app.get('/api/v2/knowledge/review-tasks')
def review_tasks(principal=Depends(require('knowledge:read', 'knowledge:review', 'im:content:read'))):
return {'items': reviews.jobs(principal.id)}
@app.get('/api/v2/knowledge/review-tasks/{job_id}/results')
def review_results(job_id: str, result: str = 'skipped', page: int = Query(default=1, ge=1),
page_size: int = Query(default=30, ge=1, le=100),
principal=Depends(require('knowledge:read', 'knowledge:review', 'im:content:read'))):
with errors():
return reviews.results(job_id, principal.id, resolve_admin_tenant(principal, -1), result, page, page_size)
@app.get('/api/v2/knowledge/review-tasks/{job_id}')
def review_task(job_id: str, principal=Depends(require('knowledge:read', 'knowledge:review', 'im:content:read'))):
with errors():
return reviews.get(job_id, principal.id)
@app.post('/api/v2/knowledge/review-tasks/{job_id}/{action}')
def review_action(job_id: str, action: str, body: ReviewConfirmBody, request: Request,
principal=Depends(require('knowledge:read', 'knowledge:review', 'im:content:read'))):
with errors():
return reviews.action(job_id, action, principal.id, client_ip(request), body.confirmed)
@app.get("/api/v2/knowledge/overview")
def overview(account_id: int | None = None, principal=Depends(require("knowledge:read"))):
scope = resolve_admin_tenant(principal, account_id)
ts = tenant_scope(scope)
with database.connect() as db:
accounts = [dict(r) for r in db.execute(
f"SELECT id,tenant_id,display_name,external_account_id FROM archive_source_account WHERE {ts.clause()}", ts.params)]
return {"counts": store.stats(scope), "source_accounts": accounts, "watches": store.watches(scope),
"settings": store.settings(scope) if isinstance(scope, str) and scope != "*" else None,
"semantic_configured": VectorIndex().configured}
@app.put("/api/v2/knowledge/settings")
def settings(body: SettingsBody, request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:publish"))):
with errors():
return store.set_enabled(single(principal, account_id), body.enabled, principal.id, client_ip(request))
@app.put("/api/v2/knowledge/watch/{source_id}")
def watch(source_id: str, body: SettingsBody, request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:write", "im:content:read"))):
with errors():
return store.set_watch(single(principal, account_id), source_id, body.enabled, principal.id, client_ip(request))
@app.get('/api/v2/knowledge/models')
def models(account_id: int | None = None, principal=Depends(require('knowledge:write', 'im:content:read'))):
# 模型清单是全局的,「全部账号」下也能选
from knowledge_models import model_choices
return {'items': model_choices(database)}
@app.put('/api/v2/knowledge/jobs/{job_id}/config')
def job_config(job_id: str, body: JobConfigBody, request: Request, account_id: int | None = None,
principal=Depends(require('knowledge:write', 'im:content:read'))):
with errors():
return store.configure_job(single(principal, account_id), job_id, body.model_dump(), principal.id, client_ip(request))
@app.get("/api/v2/knowledge/jobs")
def jobs(account_id: int | None = None, include_auto: bool = False, principal=Depends(require("knowledge:read"))):
return {"items": store.jobs(resolve_admin_tenant(principal, account_id), include_auto=include_auto)}
@app.post("/api/v2/knowledge/jobs/preview")
def preview_job(body: JobBody, account_id: int | None = None,
principal=Depends(require("knowledge:write", "im:content:read"))):
with errors():
return store.preview_job(single(principal, account_id), body.model_dump())
@app.post("/api/v2/knowledge/jobs")
def create_job(body: JobBody, request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:write", "im:content:read"))):
with errors():
return store.create_job(single(principal, account_id), body.model_dump(), principal.id, client_ip(request))
@app.post("/api/v2/knowledge/jobs/{job_id}/{action}")
def job_action(job_id: str, action: str, request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:write", "im:content:read"))):
with errors():
return store.job_action(single(principal, account_id), job_id, action, principal.id, client_ip(request))
@app.get("/api/v2/knowledge/items")
def items(account_id: int | None = None, status: str = "", query: str = "",
page: int = Query(default=1, ge=1), page_size: int = Query(default=30, ge=1, le=100),
principal=Depends(require("knowledge:read"))):
return store.list_items(resolve_admin_tenant(principal, account_id), status, query, page, page_size)
@app.get("/api/v2/knowledge/items/{item_id}")
def detail(item_id: str, account_id: int | None = None, principal=Depends(require("knowledge:read"))):
with errors():
return store.detail(single(principal, account_id), item_id, sources="im:content:read" in principal.permissions)
@app.put("/api/v2/knowledge/items/{item_id}")
def edit(item_id: str, body: EditBody, request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:write", "im:content:read"))):
with errors():
return store.save(single(principal, account_id), item_id, body.model_dump(), principal.id, client_ip(request))
@app.post("/api/v2/knowledge/items/{item_id}/{action}")
def transition(item_id: str, action: str, body: ActionBody, request: Request,
account_id: int | None = None, principal=Depends(require("knowledge:read"))):
permission = "knowledge:publish" if action in {"publish", "disable"} else (
"knowledge:write" if action in {"restore", "delete"} else "knowledge:review")
if permission not in principal.permissions or "im:content:read" not in principal.permissions:
raise HTTPException(403, "缺少知识操作或来源正文查看权限")
tenant = single(principal, account_id)
with errors():
if action == 'delete':
if not body.confirmed:
raise ValueError('请确认删除已驳回知识,原始聊天归档会保留')
return store.delete_rejected(tenant, item_id, body.revision, principal.id, client_ip(request))
if action == "publish":
return store.publish(tenant, item_id, body.revision, principal.id, client_ip(request))
if action == "restore":
with database.connect() as db:
store._item(db, tenant, item_id)
previous = db.execute("SELECT snapshot_json FROM knowledge_revision WHERE item_id=? AND revision=? "
"ORDER BY created_at DESC LIMIT 1", (item_id, body.restore_revision)).fetchone()
if previous is None:
raise KeyError("历史版本不存在")
values = json.loads(previous[0])
values["revision"] = body.revision
return store.save(tenant, item_id, values, principal.id, client_ip(request))
return store.transition(tenant, item_id, body.revision, action, principal.id, client_ip(request), body.confirmed)
@app.post("/api/v2/knowledge/search")
def search(body: SearchBody, account_id: int | None = None, principal=Depends(require("knowledge:read"))):
return KnowledgeRetriever(store).search(single(principal, account_id), body.query)
@app.get("/api/v2/knowledge/retrievals")
def retrievals(account_id: int | None = None, principal=Depends(require("knowledge:read", "stats:read"))):
scope = tenant_scope(resolve_admin_tenant(principal, account_id))
with database.connect() as db:
rows = db.execute(f"SELECT * FROM knowledge_retrieval_log WHERE {scope.clause()} "
"ORDER BY created_at DESC LIMIT 100", scope.params).fetchall()
return {"items": [{**dict(r), "hits": json.loads(r["hits_json"])} for r in rows]}
@app.get("/api/v2/knowledge/export")
def export(request: Request, account_id: int | None = None,
principal=Depends(require("knowledge:read", "knowledge:export"))):
tenant = single(principal, account_id)
with database.connect() as db:
database._audit(db, principal.id, "knowledge.export", f"tenant={tenant}", client_ip(request))
def stream():
cursor = ""
while True:
with database.connect() as db:
rows = db.execute("SELECT * FROM knowledge_item WHERE tenant_id=? AND status='published' AND id>? "
"AND (valid_until='' OR valid_until>=?) ORDER BY id LIMIT 200",
(tenant, cursor, KnowledgeRetriever._today())).fetchall()
if not rows:
return
cursor = rows[-1]["id"]
rows = [r for r in rows if store._fresh(db, r["id"], tenant)]
for row in rows:
yield json.dumps({"schema_version": "knowledge-v1", **{k: row[k] for k in
("id", "revision", "title", "question", "answer", "conditions", "category", "kind", "valid_until")}},
ensure_ascii=False) + "\n"
return StreamingResponse(stream(), media_type="application/x-ndjson",
headers={"Content-Disposition": 'attachment; filename="knowledge.jsonl"'})
@app.get("/api/v2/knowledge/desktop/summary")
def desktop_summary(account: Any = Depends(current_desktop)):
tenant = str(account["tenant_id"])
counts = store.stats(tenant)
return {"published": counts.get("published", 0), "enabled": bool(store.settings(tenant)["enabled"])}
return store