179 lines
12 KiB
Python
179 lines
12 KiB
Python
from pathlib import Path
|
|
|
|
root = Path(r'C:\kefu')
|
|
path = root / 'wechat_rpa/knowledge_store.py'
|
|
s = path.read_text(encoding='utf-8')
|
|
insert = '''
|
|
def job_scope(tenant, options, cutoff):
|
|
"""Use identical scope and cutoff predicates for preview, progress and worker."""
|
|
where = ["m.tenant_id=?", "m.source_account_id=?", "m.created_at<=?",
|
|
"c.conversation_type IN ('direct_wechat','direct_wecom')"]
|
|
args = [tenant, options["source_account_id"], cutoff]
|
|
for key, condition in (("date_start", "m.sent_at>=?"), ("date_end", "m.sent_at<?")):
|
|
if options.get(key):
|
|
where.append(condition)
|
|
args.append(options[key])
|
|
if options.get("conversation_id"):
|
|
where.append("m.conversation_id=?")
|
|
args.append(options["conversation_id"])
|
|
return where, args
|
|
|
|
|
|
def job_options(values):
|
|
options = dict(values)
|
|
start = day_boundary(options.get("date_from", ""))
|
|
end = day_boundary(options.get("date_to", ""), end=True)
|
|
if start and end and start >= end:
|
|
raise ValueError("结束日期不能早于开始日期")
|
|
if options.get("engine", "rules") not in {"rules", "model"}:
|
|
raise ValueError("未知整理方式")
|
|
for key, default, maximum in (("max_messages", 10000, 1000000), ("max_model_calls", 100, 10000)):
|
|
value = options.get(key, default)
|
|
if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= maximum:
|
|
raise ValueError("处理上限超出允许范围")
|
|
options[key] = value
|
|
options.update(date_start=start, date_end=end, pipeline_version=PIPELINE_VERSION)
|
|
return options
|
|
|
|
'''
|
|
s = s.replace('\nclass KnowledgeStore:', '\n' + insert + '\nclass KnowledgeStore:')
|
|
s = s.replace(' result.pop("cursor", None)\n return result', ''' result.pop("cursor", None)
|
|
if "options" in result:
|
|
total = result["options"].get("total_messages")
|
|
result["total_messages"] = total
|
|
result["completion_reason"] = result["options"].get("completion_reason", "")
|
|
result["progress_percent"] = (100 if result["status"] == "completed" else
|
|
min(99, int(result["processed"] * 100 / total)) if total else None)
|
|
return result''')
|
|
s = s.replace(''' with self.database.connect() as db:
|
|
db.execute("INSERT INTO knowledge_settings''', ''' with self.database.connect() as db:
|
|
if enabled:
|
|
from knowledge_retriever import KnowledgeRetriever
|
|
available = db.execute("SELECT id FROM knowledge_item WHERE tenant_id=? AND status='published' "
|
|
"AND (valid_until='' OR valid_until>=?)", (tenant, KnowledgeRetriever._today()))
|
|
if not any(self._fresh(db, row["id"], tenant) for row in available):
|
|
raise ValueError("请先审核并发布至少一条有效知识，再开启客服使用知识")
|
|
db.execute("INSERT INTO knowledge_settings''')
|
|
s = s.replace(''' if action == "approve":
|
|
''', ''' if action == "disable" and row["status"] != "published":
|
|
raise ValueError("仅已发布知识可停用")
|
|
if action == "approve":
|
|
self._check_expiry(row)
|
|
''')
|
|
s = s.replace(''' def publish(self, tenant, item_id, revision, actor, ip):''', ''' @staticmethod
|
|
def _check_expiry(row):
|
|
from knowledge_retriever import KnowledgeRetriever
|
|
if row["valid_until"] and row["valid_until"] < KnowledgeRetriever._today():
|
|
raise ValueError("知识已过有效期，请更新内容和有效期后重新审核")
|
|
|
|
def publish(self, tenant, item_id, revision, actor, ip):''')
|
|
s = s.replace(''' index = VectorIndex()
|
|
profile = index.publish(row)''', ''' self._check_expiry(row)
|
|
index = VectorIndex()
|
|
profile = index.publish(row)''')
|
|
start = s.index(' def create_job(')
|
|
end = s.index(' def job(self,', start)
|
|
s = s[:start] + ''' def _scope_counts(self, db, tenant, options, cutoff):
|
|
source = db.execute("SELECT id,display_name,external_account_id FROM archive_source_account "
|
|
"WHERE id=? AND tenant_id=?", (options["source_account_id"], tenant)).fetchone()
|
|
if source is None:
|
|
raise ValueError("请选择当前租户的客服来源账号")
|
|
where, args = job_scope(tenant, options, cutoff)
|
|
row = db.execute("""SELECT COUNT(*) available_messages,COUNT(DISTINCT m.conversation_id) conversations,
|
|
COALESCE(SUM(v.status='normal' AND lower(m.message_type) IN ('text','文本','文字','1')
|
|
AND trim(v.content)!='' AND m.sender_person_id IS NOT NULL AND m.direction IN ('inbound','outbound')
|
|
AND ((m.direction='outbound')=EXISTS(SELECT 1 FROM archive_person_identity pi
|
|
WHERE pi.person_id=m.sender_person_id AND pi.tenant_id=m.tenant_id
|
|
AND pi.external_id=a.external_account_id))),0) eligible_text_messages
|
|
FROM archive_message m JOIN archive_conversation c ON c.id=m.conversation_id
|
|
JOIN archive_source_account a ON a.id=m.source_account_id
|
|
JOIN archive_message_version v ON v.message_id=m.id AND v.version_no=(
|
|
SELECT MAX(v2.version_no) FROM archive_message_version v2
|
|
WHERE v2.message_id=m.id AND v2.created_at<=?) WHERE """ + " AND ".join(where),
|
|
[cutoff, *args]).fetchone()
|
|
result = dict(row)
|
|
result.update(total_messages=min(row["available_messages"], options["max_messages"]),
|
|
limited=row["available_messages"] > options["max_messages"], cutoff_at=cutoff,
|
|
source_name=source["display_name"] or source["external_account_id"])
|
|
return result
|
|
|
|
def preview_job(self, tenant, values):
|
|
tenant = safe_scope(tenant)
|
|
options = job_options(values)
|
|
with self.database.connect() as db:
|
|
return self._scope_counts(db, tenant, options, utc_now())
|
|
|
|
def create_job(self, tenant, options, actor, ip):
|
|
tenant = safe_scope(tenant)
|
|
options = job_options(options)
|
|
if not options.get("staff_confirmed"):
|
|
raise ValueError("请确认所选来源账号是客服账号，且这些聊天允许用于知识整理")
|
|
scope_key = fingerprint(json_text({k: options.get(k, "") for k in (
|
|
"source_account_id", "conversation_id", "date_start", "date_end", "engine", "max_messages", "max_model_calls")}))
|
|
with self.database.connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
active = db.execute("SELECT options_json FROM knowledge_job WHERE tenant_id=? "
|
|
"AND status IN ('queued','running','paused')", (tenant,))
|
|
if any(json.loads(row[0]).get("scope_key") == scope_key for row in active):
|
|
raise ValueError("相同范围已有未完成任务，请在加工任务中继续或取消原任务")
|
|
job_id, now = new_id(), utc_now()
|
|
estimate = self._scope_counts(db, tenant, options, now)
|
|
options.update(total_messages=estimate["total_messages"], available_messages=estimate["available_messages"],
|
|
source_name=estimate["source_name"], scope_key=scope_key)
|
|
db.execute("INSERT INTO knowledge_job(id,tenant_id,options_json,cutoff_at,created_by,created_at,updated_at) "
|
|
"VALUES (?,?,?,?,?,?,?)", (job_id, tenant, json_text(options), now, actor, now, now))
|
|
if options.get("auto_watch"):
|
|
db.execute("INSERT INTO knowledge_watch VALUES (?,?,1,?) ON CONFLICT(tenant_id,source_account_id) "
|
|
"DO UPDATE SET enabled=1,created_by=excluded.created_by", (tenant, options["source_account_id"], actor))
|
|
self.database._audit(db, actor, "knowledge.job.create", f"job={job_id} tenant={tenant}", ip)
|
|
return self.job(tenant, job_id)
|
|
|
|
''' + s[end:]
|
|
s = s.replace('if action not in {"pause", "resume"}:','if action not in {"pause", "resume", "cancel"}:')
|
|
s = s.replace(''' allowed = {"queued", "running"} if action == "pause" else {"paused", "failed"}''',''' allowed = ({"queued", "running"} if action == "pause" else {"paused", "failed"}
|
|
if action == "resume" else {"queued", "running", "paused", "failed"})''')
|
|
s = s.replace('''("paused" if action == "pause" else "queued", utc_now(), job_id))''','''({"pause": "paused", "resume": "queued", "cancel": "cancelled"}[action], utc_now(), job_id))
|
|
if action == "cancel":
|
|
db.execute("UPDATE knowledge_job SET state_json='{}' WHERE id=?", (job_id,))''')
|
|
path.write_text(s, encoding='utf-8', newline='\n')
|
|
|
|
path = root / 'wechat_rpa/knowledge_worker.py'
|
|
s = path.read_text(encoding='utf-8').replace('from knowledge_store import KnowledgeStore','from knowledge_store import KnowledgeStore, job_scope')
|
|
start = s.index(' where = [', s.index(' def rows('))
|
|
end = s.index(' if cursor:', start)
|
|
s = s[:start] + ' where, args = job_scope(job["tenant_id"], options, job["cutoff_at"])\n' + s[end:]
|
|
s = s.replace(''' now = utc_now()
|
|
db.execute("INSERT INTO knowledge_job''',''' now = utc_now()
|
|
estimate = self.store._scope_counts(db, dirty["tenant_id"], options, now)
|
|
options.update(total_messages=estimate["total_messages"], available_messages=estimate["available_messages"],
|
|
source_name=estimate["source_name"])
|
|
db.execute("INSERT INTO knowledge_job''')
|
|
s = s.replace(''' rows = self.rows(job, max(1, size)) if remaining > 0 else []''',''' fetched = self.rows(job, max(1, size) + 1) if remaining > 0 else []
|
|
rows = fetched[:size] if size > 0 else []''')
|
|
s = s.replace('natural_end = len(rows) < size','natural_end = remaining > 0 and len(fetched) <= size')
|
|
s = s.replace(''' if done and not natural_end:
|
|
state = {}''',''' if done:
|
|
options["completion_reason"] = "source_exhausted" if natural_end else "message_limit"
|
|
if done and not natural_end:
|
|
state = {}''')
|
|
s = s.replace('''created_items=created_items+?,duplicates=duplicates+?,skipped=skipped+?,lease_owner='',
|
|
lease_until=0,error='',updated_at=? WHERE id=?''','''created_items=created_items+?,duplicates=duplicates+?,skipped=skipped+?,lease_owner='',
|
|
lease_until=0,error='',options_json=?,updated_at=? WHERE id=?''')
|
|
s = s.replace('''{"text", "文本", "文字", "1"} or r["status"] != "normal" for r in rows), utc_now(), job["id"]))''','''{"text", "文本", "文字", "1"} or r["status"] != "normal" or not r["content"].strip() for r in rows),
|
|
json_text(options), utc_now(), job["id"]))''')
|
|
path.write_text(s, encoding='utf-8', newline='\n')
|
|
|
|
path = root / 'wechat_rpa/knowledge_api.py'
|
|
s = path.read_text(encoding='utf-8')
|
|
s = s.replace(''' return store.set_enabled(single(principal, account_id), body.enabled, principal.id, client_ip(request))''',''' with errors():
|
|
return store.set_enabled(single(principal, account_id), body.enabled, principal.id, client_ip(request))''')
|
|
s = s.replace(''' @app.post("/api/v2/knowledge/jobs")''',''' @app.post("/api/v2/knowledge/jobs/preview")
|
|
def preview_job(body: JobBody, account_id: int | None = None,
|
|
principal=Depends(require("knowledge:write", "im:content:read"))):
|
|
with errors():
|
|
return store.preview_job(single(principal, account_id), body.model_dump())
|
|
|
|
@app.post("/api/v2/knowledge/jobs")''')
|
|
path.write_text(s, encoding='utf-8', newline='\n')
|
|
print('Updated knowledge store, worker and preview API.')
|