Files
kefu/wechat_rpa/knowledge_review.py
T
2026-11-16 09:28:25 +08:00

432 lines
25 KiB
Python

"""Durable cross-page approval of a fixed, authorized set of knowledge drafts."""
from __future__ import annotations
import json
import time
from datetime import date
from archive_store import json_text, new_id, utc_now
from knowledge_assignments import ASSIGNOR, KnowledgeAssignments, _access_if_allowed, _covers
from knowledge_retriever import KnowledgeRetriever
from knowledge_access import REVIEWER_PERMISSIONS, current_access
from knowledge_review_policy import SELF_REVIEW_MESSAGE, blocked_self_review_tenants
REVIEW_SCHEMA = """
CREATE TABLE IF NOT EXISTS knowledge_review_job (
id TEXT PRIMARY KEY, created_by INTEGER NOT NULL, account_id INTEGER NOT NULL,
scope_label TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'preview',
total INTEGER NOT NULL DEFAULT 0, processed INTEGER NOT NULL DEFAULT 0,
approved INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0,
preview_json TEXT NOT NULL DEFAULT '{}', expires_at REAL NOT NULL,
error TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_knowledge_review_queue ON knowledge_review_job(status,created_at,id);
CREATE INDEX IF NOT EXISTS idx_knowledge_review_owner ON knowledge_review_job(created_by,created_at DESC);
CREATE TABLE IF NOT EXISTS knowledge_review_target (
job_id TEXT NOT NULL REFERENCES knowledge_review_job(id) ON DELETE CASCADE,
item_id TEXT NOT NULL, tenant_id TEXT NOT NULL, revision INTEGER NOT NULL,
item_updated_at TEXT NOT NULL, preview_reason TEXT NOT NULL DEFAULT '',
result TEXT NOT NULL DEFAULT 'pending', reason TEXT NOT NULL DEFAULT '',
PRIMARY KEY(job_id,item_id)
);
CREATE INDEX IF NOT EXISTS idx_knowledge_review_pending ON knowledge_review_target(job_id,result,item_id);
CREATE TABLE IF NOT EXISTS knowledge_review_claim (
job_id TEXT PRIMARY KEY REFERENCES knowledge_review_job(id) ON DELETE CASCADE,
token TEXT NOT NULL, item_id TEXT NOT NULL DEFAULT '', lease_until REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS knowledge_review_assigned_job (
job_id TEXT PRIMARY KEY REFERENCES knowledge_review_job(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS knowledge_review_target_assignment (
job_id TEXT NOT NULL REFERENCES knowledge_review_assigned_job(job_id) ON DELETE CASCADE,
item_id TEXT NOT NULL, assignment_id TEXT NOT NULL,
PRIMARY KEY(job_id,item_id),
FOREIGN KEY(job_id,item_id) REFERENCES knowledge_review_target(job_id,item_id) ON DELETE CASCADE
);
"""
REASONS = {
'': '符合校验条件', 'missing_conditions': '缺少适用条件,需编辑补充',
'invalid_fields': '知识字段不完整、过长或类型无效', 'expired': '已超过有效期',
'missing_source': '缺少来源记录', 'changed_source': '来源已变化,请重新整理',
'changed_item': '预览后知识版本或状态已变化', 'deleted': '知识已删除',
'unauthorized': '审核分配已失效',
'self_review': SELF_REVIEW_MESSAGE,
'index_failed': '搜索索引写入失败;草稿保留,请排查后重新预览',
'publish_failed': '发布校验失败;草稿保留,请编辑后重新预览',
}
# 「一键审核」按钮的权限。后台每跑一步都重新核对:中途收回权限,任务就停。
REVIEW_PERMISSIONS = (*REVIEWER_PERMISSIONS, 'knowledge:review:auto')
PUBLISH_PERMISSIONS = (*REVIEW_PERMISSIONS, 'knowledge:publish')
CLAIM_SECONDS = 300
ASSIGNED_SCOPE_LABEL = '分配给我的待审核知识'
def review_mode(row):
return json.loads(row['preview_json']).get('mode', 'approve')
def reason_sql(alias='k'):
"""The same eligibility predicate is used by preview and execution."""
k = alias
return f"""CASE
WHEN trim({k}.conditions)='' THEN 'missing_conditions'
WHEN trim({k}.title)='' OR length({k}.title)>160 OR trim({k}.question)=''
OR length({k}.question)>4000 OR trim({k}.answer)='' OR length({k}.answer)>8000
OR length({k}.conditions)>2000 OR length({k}.category)>100
OR {k}.kind NOT IN ('qa','procedure','case') THEN 'invalid_fields'
WHEN {k}.valid_until!='' AND (length({k}.valid_until)!=10
OR {k}.valid_until NOT GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'
OR date({k}.valid_until,'+0 days') IS NULL OR date({k}.valid_until,'+0 days')!={k}.valid_until)
THEN 'invalid_fields'
WHEN {k}.valid_until!='' AND {k}.valid_until<? THEN 'expired'
WHEN NOT EXISTS(SELECT 1 FROM knowledge_source s WHERE s.item_id={k}.id) THEN 'missing_source'
WHEN EXISTS(SELECT 1 FROM knowledge_source s LEFT JOIN archive_message m ON m.id=s.message_id
WHERE s.item_id={k}.id AND (m.id IS NULL OR m.tenant_id!={k}.tenant_id OR m.status!='normal'
OR s.version_no!=COALESCE((SELECT MAX(version_no) FROM archive_message_version WHERE message_id=s.message_id),-1)))
THEN 'changed_source'
ELSE '' END"""
def reviewer_access(db, actor, mode='approve'):
try:
return current_access(db, actor, PUBLISH_PERMISSIONS if mode == 'approve_publish' else REVIEW_PERMISSIONS)
except PermissionError as error:
raise ValueError('审核权限已变化或账号已停用,请恢复权限后继续') from error
class KnowledgeReviewStore:
def __init__(self, store):
self.store = store
self.database = store.database
self.assignments = KnowledgeAssignments(store)
@staticmethod
def owned(db, job_id, actor):
row = db.execute('SELECT j.* FROM knowledge_review_job j JOIN knowledge_review_assigned_job a '
'ON a.job_id=j.id WHERE j.id=? AND j.created_by=?', (job_id, actor)).fetchone()
if row is None:
raise KeyError('审核任务不存在或不属于当前用户')
return row
def _visible_targets(self, db, job, result=''):
# A successful transition ends inbox eligibility. Retain that result only
# while the exact original assignment, revision, issuer and sources still match.
assignors = {}
rows = db.execute("""SELECT t.item_id,t.tenant_id,t.revision,t.result,t.reason,t.preview_reason,k.title,a.assignor_id
FROM knowledge_review_target t
JOIN knowledge_review_target_assignment b ON b.job_id=t.job_id AND b.item_id=t.item_id
JOIN knowledge_review_assignment a ON a.item_id=t.item_id AND a.assignment_id=b.assignment_id
AND a.revision=t.revision AND a.assignee_id=?
JOIN knowledge_item k ON k.id=t.item_id AND k.tenant_id=t.tenant_id AND k.revision=t.revision
WHERE t.job_id=? AND (?='' OR t.result=?) AND (
(t.result IN ('pending','skipped') AND k.status='draft' AND k.updated_at=t.item_updated_at)
OR (t.result='approved' AND k.status IN ('approved','published'))
OR (t.result='published' AND k.status='published'))
ORDER BY t.item_id""", (job['created_by'], job['id'], result, result))
for target in rows:
issuer = target['assignor_id']
if issuer not in assignors:
assignors[issuer] = _access_if_allowed(db, issuer, ASSIGNOR)
if _covers(assignors[issuer], target['tenant_id']) and self.store._fresh(db, target['item_id'], target['tenant_id']):
yield target
def visible_public(self, db, row):
reasons, outcomes = {}, {}
for target in self._visible_targets(db, row):
reason = target['preview_reason']
reasons[reason] = reasons.get(reason, 0) + 1
outcome = target['result']
outcomes[outcome] = outcomes.get(outcome, 0) + 1
total, eligible = sum(reasons.values()), reasons.get('', 0)
published = outcomes.get('published', 0)
result = dict(row)
result.pop('preview_json')
result.update(mode=review_mode(row), total=total, processed=total - outcomes.get('pending', 0),
approved=outcomes.get('approved', 0) + published, published=published,
skipped=outcomes.get('skipped', 0))
result['preview'] = {'mode': result['mode'], 'total': total, 'eligible': eligible, 'blocked': total - eligible,
'accounts': [], 'snapshot_at': row['created_at'],
'reasons': [{'code': key, 'count': count, 'reason': REASONS[key]} for key, count in sorted(reasons.items()) if key]}
result['progress_percent'] = (100 if result['status'] == 'completed' else
min(99, int(result['processed'] * 100 / result['total'])) if result['total'] else 0)
return result
def get(self, job_id, actor):
with self.database.connect() as db:
db.execute('BEGIN')
current_access(db, actor, REVIEW_PERMISSIONS)
return self.visible_public(db, self.owned(db, job_id, actor))
def jobs(self, actor):
with self.database.connect() as db:
db.execute('BEGIN')
current_access(db, actor, REVIEW_PERMISSIONS)
return [self.visible_public(db, row) for row in db.execute(
"SELECT j.* FROM knowledge_review_job j JOIN knowledge_review_assigned_job a ON a.job_id=j.id "
"WHERE j.created_by=? AND j.status NOT IN ('preview','expired') ORDER BY j.created_at DESC,j.id LIMIT 30", (actor,))]
def preview(self, scope, account_id, actor, ip, mode='approve'):
if mode not in {'approve', 'approve_publish'}:
raise ValueError('未知审核任务模式')
if account_id != -1:
raise ValueError('请选择分配给我的全部待审核知识')
job_id, now = new_id(), utc_now()
with self.database.connect() as db:
db.execute('BEGIN IMMEDIATE')
reviewer_access(db, actor, mode)
db.execute("UPDATE knowledge_review_job SET status='expired' WHERE created_by=? AND status='preview' "
"AND id IN (SELECT job_id FROM knowledge_review_assigned_job)", (actor,))
db.execute('INSERT INTO knowledge_review_job(id,created_by,account_id,scope_label,expires_at,created_at,updated_at) '
'VALUES (?,?,?,?,?,?,?)', (job_id, actor, -1, ASSIGNED_SCOPE_LABEL, time.time() + 1800, now, now))
db.execute('INSERT INTO knowledge_review_assigned_job VALUES (?)', (job_id,))
blocked_tenants = blocked_self_review_tenants(db, actor)
today = KnowledgeRetriever._today()
total = 0
for total, item in enumerate(self.assignments._pending(db, actor), 1):
db.execute(f"""INSERT INTO knowledge_review_target(job_id,item_id,tenant_id,revision,item_updated_at,preview_reason)
SELECT ?,k.id,k.tenant_id,k.revision,k.updated_at,
CASE WHEN ? THEN 'self_review' ELSE {reason_sql()} END FROM knowledge_item k WHERE k.id=?""",
(job_id, item['tenant_id'] in blocked_tenants, today, item['id']))
db.execute('INSERT INTO knowledge_review_target_assignment VALUES (?,?,?)',
(job_id, item['id'], item['assignment_id']))
db.execute('UPDATE knowledge_review_job SET total=?,preview_json=? WHERE id=?', (total, json_text({'mode': mode}), job_id))
self.database._audit(db, actor, 'knowledge.review.preview', f'job={job_id} total={total}', ip)
return self.get(job_id, actor)
def action(self, job_id, action, actor, ip, confirmed=False, *, raw_confirmed=None):
if action not in {'start', 'pause', 'resume', 'cancel'}:
raise ValueError('未知审核任务操作')
with self.database.connect() as db:
db.execute('BEGIN IMMEDIATE')
row = self.owned(db, job_id, actor)
mode = review_mode(row)
reviewer_access(db, actor, mode if action in {'start', 'resume'} else 'approve')
if action == 'start':
if mode == 'approve_publish' and (confirmed if raw_confirmed is None else raw_confirmed) is not True:
raise ValueError('请明确确认当前范围知识已完成审核并同意发布')
if not confirmed:
raise ValueError('请确认当前范围内的知识已完成必要审核,再批量通过')
# A repeated HTTP submission cannot create or enqueue a second job.
if row['status'] in {'queued', 'running', 'completed'}:
return self.visible_public(db, row)
if row['status'] != 'preview' or row['expires_at'] < time.time():
raise ValueError('预览已失效,请重新预览审核范围')
if next(self._visible_targets(db, row, 'pending'), None) is None:
raise ValueError('当前范围没有待审核草稿')
status = 'queued'
else:
allowed = {'pause': {'queued', 'running'}, 'resume': {'paused', 'failed'},
'cancel': {'preview', 'queued', 'running', 'paused', 'failed'}}[action]
if row['status'] not in allowed:
raise ValueError('当前状态不能执行此操作')
status = {'pause': 'paused', 'resume': 'queued', 'cancel': 'cancelled'}[action]
db.execute("UPDATE knowledge_review_job SET status=?,error='',updated_at=? WHERE id=?", (status, utc_now(), job_id))
# Revoking a running lease also fences an upload that outlives pause/resume.
db.execute('DELETE FROM knowledge_review_claim WHERE job_id=?', (job_id,))
self.database._audit(db, actor, 'knowledge.review.' + action, f'job={job_id}', ip)
return self.get(job_id, actor)
def results(self, job_id, actor, scope, result='skipped', page=1, page_size=30):
if result not in {'', 'pending', 'approved', 'published', 'skipped'}:
raise ValueError('未知审核结果筛选')
start, total, items = (page - 1) * page_size, 0, []
with self.database.connect() as db:
db.execute('BEGIN')
current_access(db, actor, REVIEW_PERMISSIONS)
job = self.owned(db, job_id, actor)
for total, row in enumerate(self._visible_targets(db, job, result), 1):
if start < total <= start + page_size:
item = {key: row[key] for key in ('item_id', 'title', 'revision', 'result', 'reason', 'preview_reason')}
item['reason_text'] = REASONS.get(row['reason'] or row['preview_reason'], '')
items.append(item)
return {'items': items, 'total': total}
class KnowledgeReviewWorker:
"""Short transactions with fenced, expiring leases around external index writes."""
def __init__(self, store):
self.store = store
self.database = store.database
self.assignments = KnowledgeAssignments(store)
def step(self, batch_size=100):
batch_size = max(1, min(100, batch_size))
token = new_id()
with self.database.connect() as db:
db.execute('BEGIN IMMEDIATE')
job = db.execute("SELECT j.* FROM knowledge_review_job j JOIN knowledge_review_assigned_job a "
"ON a.job_id=j.id WHERE j.status IN ('queued','running') "
"AND NOT EXISTS(SELECT 1 FROM knowledge_review_claim c WHERE c.job_id=j.id AND c.lease_until>?) "
'ORDER BY j.created_at,j.id LIMIT 1', (time.time(),)).fetchone()
if job is None:
return self.cleanup(db)
if review_mode(job) == 'approve':
return self._approve_step(db, job, batch_size)
db.execute('INSERT INTO knowledge_review_claim(job_id,token,lease_until) VALUES (?,?,?) '
'ON CONFLICT(job_id) DO UPDATE SET token=excluded.token,item_id=\'\',lease_until=excluded.lease_until',
(job['id'], token, time.time() + CLAIM_SECONDS))
return self._publish_batch(job['id'], token, batch_size)
def _approve_step(self, db, job, batch_size):
if not self._access_or_stop(db, job):
return True
blocked_tenants = blocked_self_review_tenants(db, job['created_by'])
rows = self._targets(db, job['id'], limit=batch_size).fetchall()
approved = 0
for row in rows:
item, reason = self._validate_target(db, job, row, blocked_tenants)
if not reason:
self.store._transition_item(db, item, row['revision'], 'approve', job['created_by'], '', True)
approved += 1
db.execute('UPDATE knowledge_review_target SET result=?,reason=? WHERE job_id=? AND item_id=?',
('skipped' if reason else 'approved', reason, job['id'], row['item_id']))
done = job['processed'] + len(rows) >= job['total']
db.execute('UPDATE knowledge_review_job SET status=?,processed=processed+?,approved=approved+?,skipped=skipped+?,updated_at=? WHERE id=?',
('completed' if done else 'running', len(rows), approved, len(rows) - approved, utc_now(), job['id']))
if done:
self.database._audit(db, job['created_by'], 'knowledge.review.complete', f"job={job['id']} approved={job['approved'] + approved}", '')
return True
def _active_claim(self, db, job_id, token, item_id=None):
claim = db.execute('SELECT * FROM knowledge_review_claim WHERE job_id=? AND token=? AND lease_until>?',
(job_id, token, time.time())).fetchone()
job = db.execute('SELECT j.* FROM knowledge_review_job j JOIN knowledge_review_assigned_job a ON a.job_id=j.id '
'WHERE j.id=?', (job_id,)).fetchone()
if (claim is None or job is None or job['status'] not in {'queued', 'running'}
or review_mode(job) != 'approve_publish' or (item_id is not None and claim['item_id'] != item_id)):
return None
return job
@staticmethod
def _targets(db, job_id, item_id=None, *, limit=1):
return db.execute(f"""SELECT t.*,b.assignment_id,{reason_sql()} eligibility
FROM knowledge_review_target t LEFT JOIN knowledge_item k ON k.id=t.item_id AND k.tenant_id=t.tenant_id
LEFT JOIN knowledge_review_target_assignment b ON b.job_id=t.job_id AND b.item_id=t.item_id
WHERE t.job_id=? AND t.result='pending' {'AND t.item_id=?' if item_id is not None else ''}
ORDER BY t.item_id LIMIT ?""",
[KnowledgeRetriever._today(), job_id, *([item_id] if item_id is not None else []), limit])
def _validate_target(self, db, job, row, blocked_tenants):
if not row['assignment_id']:
return None, 'unauthorized'
try:
item = self.assignments._assigned_item(db, job['created_by'], row['item_id'],
assignment_id=row['assignment_id'])
except KeyError:
return None, 'unauthorized'
if (item['tenant_id'] != row['tenant_id'] or item['revision'] != row['revision']
or item['updated_at'] != row['item_updated_at']):
return None, 'changed_item'
if row['preview_reason'] == 'self_review' or row['tenant_id'] in blocked_tenants:
return None, 'self_review'
if item['valid_until']:
try:
if date.fromisoformat(item['valid_until']).isoformat() != item['valid_until']:
return None, 'invalid_fields'
except ValueError:
return None, 'invalid_fields'
return item, row['eligibility']
def _finish_target(self, db, job, target, reason=''):
# Publication and this update share the caller's single transaction.
changed = db.execute("UPDATE knowledge_review_target SET result=?,reason=? WHERE job_id=? AND item_id=? AND result='pending'",
('skipped' if reason else 'published', reason, job['id'], target['item_id'])).rowcount
if changed != 1:
raise RuntimeError('审核目标已被其他执行器处理')
done = job['processed'] + 1 >= job['total']
db.execute('UPDATE knowledge_review_job SET status=?,processed=processed+1,approved=approved+?,skipped=skipped+?,updated_at=? WHERE id=?',
('completed' if done else 'running', int(not reason), int(bool(reason)), utc_now(), job['id']))
if done:
self.database._audit(db, job['created_by'], 'knowledge.review.complete',
f"job={job['id']} published={job['approved'] + int(not reason)}", '')
db.execute('DELETE FROM knowledge_review_claim WHERE job_id=?', (job['id'],))
def _access_or_stop(self, db, job):
try:
reviewer_access(db, job['created_by'], review_mode(job))
except ValueError as error:
db.execute("UPDATE knowledge_review_job SET status='failed',error=?,updated_at=? WHERE id=?",
(str(error), utc_now(), job['id']))
db.execute('DELETE FROM knowledge_review_claim WHERE job_id=?', (job['id'],))
return False
return True
def _publish_batch(self, job_id, token, batch_size):
from knowledge_retriever import VectorIndex
for _ in range(batch_size):
with self.database.connect() as db:
db.execute('BEGIN IMMEDIATE')
job = self._active_claim(db, job_id, token)
if job is None:
break
if not self._access_or_stop(db, job):
break
target = self._targets(db, job_id).fetchone()
if target is None:
db.execute("UPDATE knowledge_review_job SET status='completed',updated_at=? WHERE id=?", (utc_now(), job_id))
db.execute('DELETE FROM knowledge_review_claim WHERE job_id=?', (job_id,))
break
blocked_tenants = blocked_self_review_tenants(db, job['created_by'])
_, reason = self._validate_target(db, job, target, blocked_tenants)
if not reason:
try:
draft = self.store._publication_draft(db, target['tenant_id'], target['item_id'], target['revision'], True)
except ValueError:
reason = 'publish_failed'
if reason:
self._finish_target(db, job, target, reason)
continue
db.execute("UPDATE knowledge_review_job SET status='running',updated_at=? WHERE id=?", (utc_now(), job_id))
db.execute('UPDATE knowledge_review_claim SET item_id=?,lease_until=? WHERE job_id=? AND token=?',
(target['item_id'], time.time() + CLAIM_SECONDS, job_id, token))
# No SQLite connection or transaction is held while indexing.
reason, profile = '', ''
try:
index = VectorIndex()
profile = index.publish(draft) if index.configured else ''
except Exception:
# Do not retry a mutation automatically or expose remote error bodies.
reason = 'index_failed'
with self.database.connect() as db:
db.execute('BEGIN IMMEDIATE')
job = self._active_claim(db, job_id, token, target['item_id'])
if job is None:
break
if not self._access_or_stop(db, job):
break
current_target = self._targets(db, job_id, target['item_id']).fetchone()
if current_target is None:
break
blocked_tenants = blocked_self_review_tenants(db, job['created_by'])
_, state_reason = self._validate_target(db, job, current_target, blocked_tenants)
reason = state_reason or reason
if not reason:
try:
current = self.store._publication_draft(db, target['tenant_id'], target['item_id'], target['revision'], True)
if current != draft:
reason = 'changed_item'
except ValueError:
reason = 'publish_failed'
if not reason:
self.store._commit_approve_publish(db, draft, profile, job['created_by'], '')
self._finish_target(db, job, current_target, reason)
with self.database.connect() as db:
db.execute('DELETE FROM knowledge_review_claim WHERE job_id=? AND token=?', (job_id, token))
return True
@staticmethod
def cleanup(db):
# Abandoned previews expire. Delete a bounded number of IDs per idle tick.
row = db.execute("SELECT j.id FROM knowledge_review_job j JOIN knowledge_review_assigned_job a ON a.job_id=j.id "
"WHERE (j.status='preview' AND j.expires_at<?) OR j.status='expired' "
'ORDER BY j.created_at LIMIT 1', (time.time(),)).fetchone()
if row is None:
return False
db.execute("UPDATE knowledge_review_job SET status='expired' WHERE id=?", (row['id'],))
db.execute('DELETE FROM knowledge_review_target WHERE rowid IN (SELECT rowid FROM knowledge_review_target WHERE job_id=? LIMIT 1000)', (row['id'],))
if not db.execute('SELECT 1 FROM knowledge_review_target WHERE job_id=? LIMIT 1', (row['id'],)).fetchone():
db.execute('DELETE FROM knowledge_review_job WHERE id=?', (row['id'],))
return True