gr/kefu
Template
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新bug

Browse Source
This commit is contained in:
Your Name
2026-07-31 11:48:16 +08:00
parent f913a57529
commit f22cc1a70d
109 changed files with 37586 additions and 927 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wechat_rpa/ai_chat.py
+764 -38
View File
@@ -16,6 +16,16 @@ import ai_config
# 这样在 GUI「AI 高级配置」中修改保存后，下一次请求立即生效，无需重启。
def _provider_type() -> str:
value = str(getattr(ai_config, "AI_PROVIDER_TYPE", "auto") or "auto").lower()
if value in {"openai", "dify", "comfyui"}:
return value
path = (urlparse((ai_config.AI_API_BASE or "").rstrip("/")).path or "").lower()
if "chat-messages" in path or "completion-messages" in path:
return "dify"
return "openai"
def _completions_url() -> str:
"""
组装实际请求地址：
@@ -24,6 +34,15 @@ def _completions_url() -> str:
"""
base = (ai_config.AI_API_BASE or "").rstrip("/")
path = urlparse(base).path or ""
if _provider_type() == "dify":
lower_path = path.lower().rstrip("/")
if lower_path.endswith(("/chat-messages", "/completion-messages")):
return base
if lower_path.endswith("/v1"):
return f"{base}/chat-messages"
return f"{base}/v1/chat-messages"
if path.lower().rstrip("/").endswith("/chat/completions"):
return base
if re.match(r"^/v1/.+", path):
return base
return f"{base}/chat/completions"
@@ -31,10 +50,60 @@ def _completions_url() -> str:
def _is_dify_endpoint() -> bool:
"""AI_API_BASE 指向 Dify 的 chat-messages / completion-messages 时走 Dify 协议。"""
if _provider_type() == "dify":
return True
if _provider_type() in {"openai", "comfyui"}:
return False
path = (urlparse((ai_config.AI_API_BASE or "").rstrip("/")).path or "").lower()
return "chat-messages" in path or "completion-messages" in path
def _development_mode_enabled() -> bool:
value = getattr(ai_config, "AI_DEVELOPMENT_MODE", False)
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
def _log_request_diagnostics(url: str, protocol: str) -> None:
"""Log request routing/configuration without exposing prompts or credentials."""
if not _development_mode_enabled():
return
try:
from backend_client import diagnostic_url
safe_url = diagnostic_url(url)
safe_base = diagnostic_url(getattr(ai_config, "AI_API_BASE", ""))
except Exception:
safe_url = str(url or "")
safe_base = str(getattr(ai_config, "AI_API_BASE", "") or "")
details = {
"服务类型": _provider_type(),
"调用协议": protocol,
"API 基础地址": safe_base,
"实际请求地址": safe_url,
"模型名称": str(getattr(ai_config, "AI_MODEL", "") or ""),
"API Key": (
"[已配置，值已隐藏]"
if str(getattr(ai_config, "AI_API_KEY", "") or "").strip()
else "[未配置]"
),
"请求超时（秒）": getattr(ai_config, "AI_TIMEOUT", None),
"最大回复 tokens": getattr(ai_config, "AI_MAX_TOKENS", None),
"温度": getattr(ai_config, "AI_TEMPERATURE", None),
"始终视觉模式": bool(getattr(ai_config, "AI_USE_VISION", False)),
"媒体消息自动视觉": True,
"AI 页面守护": bool(getattr(ai_config, "AI_UI_GUARD_ENABLED", True)),
"上下文": bool(getattr(ai_config, "AI_CONTEXT_ENABLED", False)),
"MCP 工具": bool(getattr(ai_config, "AI_MCP_ENABLED", False)),
}
print(f"[开发模式] 模型请求地址: {safe_url}")
print(
"[开发模式] 本次模型配置（聊天内容与敏感值未输出）: "
+ json.dumps(details, ensure_ascii=False, separators=(",", ":"))
)
def _system_prompt() -> str:
"""
动态构建系统提示词：基础人设（用当前昵称/医院名实时渲染）
@@ -48,10 +117,12 @@ def _system_prompt() -> str:
return prompt
_CHAT_HEADER_RE = re.compile(
r"^.+?\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?"
_CHAT_HEADER_PARTS_RE = re.compile(
r"^(?P<speaker>.+?)\s+"
r"(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?"
r"\d{1,2}:\d{2}(?::\d{2})?$"
)
_CHAT_HEADER_RE = _CHAT_HEADER_PARTS_RE
_CASUAL_RE = re.compile(
r"(?:好困|困死|想睡|好累|累死|无聊|好烦|烦死|好饿|饿死|"
r"在干嘛|干什么呢|多大了|几岁|哪里人|叫什么|吃饭了吗|"
@@ -66,6 +137,84 @@ _MEDICAL_DRIFT_RE = re.compile(
r"调药|调整方案|治疗方案)"
)
_CLARIFY_RE = re.compile(r"^(?:什么|啥|什么意思|没懂|没看懂|没明白)[？?。！!]*$")
_LOW_INFORMATION_RE = re.compile(
r"^[\s!！?？。,.，…~～啊呀哦噢嗯唔诶欸哎]+$"
)
_LOW_INFORMATION_GUESS_RE = re.compile(
r"(?:不小心|误触|碰到手机|按到手机|是不是.{0,12}(?:生气|不舒服|出事|发生什么))"
)
# 企业微信会把部分非文字气泡复制成占位符。标准 Unicode emoji 仍是普通
# 文字，不在这里识别；只有图片、贴纸、语音等真正需要额外能力的气泡才触发。
_MEDIA_PATTERNS = {
"image": re.compile(
r"[\[【]\s*(?:图片|照片|相片|图像|image|photo)\s*[\]】]",
re.IGNORECASE,
),
"sticker": re.compile(
r"[\[【]\s*(?:动画表情|表情包|表情|贴纸|sticker|emoticon)\s*[\]】]",
re.IGNORECASE,
),
"voice": re.compile(
r"[\[【]\s*(?:语音|语音消息|音频|voice|audio)"
r"(?:\s*[:：]?\s*(?:\d+(?:\.\d+)?\s*(?:秒|s|″|”|′|＇|'|\")|\d{1,2}:\d{2}))?\s*[\]】]"
r"|(?m:^[ \t]*(?:语音消息|语音|音频|voice|audio)[ \t:：]+(?:\d{1,2}:\d{2}|"
r"\d+(?:\.\d+)?[ \t]*(?:秒|s|″|”|′|＇|'|\"))[ \t]*$)",
re.IGNORECASE,
),
"video": re.compile(
r"[\[【]\s*(?:视频|小视频|video)\s*[\]】]",
re.IGNORECASE,
),
"file": re.compile(
r"[\[【]\s*(?:文件|文档|file)\s*[\]】]",
re.IGNORECASE,
),
}
_MEDIA_PLACEHOLDER_RE = re.compile(
"|".join(f"(?:{pattern.pattern})" for pattern in _MEDIA_PATTERNS.values()),
re.IGNORECASE,
)
_VOICE_DURATION_ONLY_RE = re.compile(
r"^[\s，,。:：-]*(?:\d{1,2}:\d{2}|\d+(?:\.\d+)?\s*(?:秒|s|″|”|′|＇|'|\")+)"
r"[\s，,。]*$",
re.IGNORECASE,
)
_VOICE_CONTROL_ONLY_RE = re.compile(
r"^(?:转(?:成|为)?文字|语音转文字|重新转文字|播放|暂停|"
r"(?:正在)?(?:识别|转写|转文字|转换(?:成|为)?文字)中?(?:…|\.\.\.)?|"
r"(?:暂时)?无法识别(?:该)?语音|未识别出文字|(?:语音识别|转文字)失败)[。！!…\s]*$"
)
VISION_NO_INCOMING = "__NO_INCOMING_MESSAGE__"
VISION_VOICE_NEEDS_TEXT = "__VOICE_NOT_TRANSCRIBED__"
class VisionReply(str):
"""String-compatible reply carrying media facts proved by the vision response."""
def __new__(cls, value: str, media_types=None, voice_transcribed: bool = False):
obj = str.__new__(cls, value or "")
obj.media_types = frozenset(media_types or ())
obj.voice_transcribed = bool(voice_transcribed)
return obj
def detect_media_types(chat_text: str) -> set[str]:
"""Return media bubble types explicitly present in copied WeCom text."""
text = str(chat_text or "")
return {
media_type
for media_type, pattern in _MEDIA_PATTERNS.items()
if pattern.search(text)
}
def _chat_header_match(line: str):
"""Do not mistake media durations such as '[语音] 00:05' for speaker headers."""
if detect_media_types(line):
return None
return _CHAT_HEADER_PARTS_RE.match(line)
def latest_customer_message(chat_text: str) -> str:
@@ -75,7 +224,7 @@ def latest_customer_message(chat_text: str) -> str:
return ""
last_header = -1
for index, line in enumerate(lines):
if _CHAT_HEADER_RE.match(line):
if _chat_header_match(line):
last_header = index
if 0 <= last_header < len(lines) - 1:
return "\n".join(lines[last_header + 1:]).strip()
@@ -83,8 +232,122 @@ def latest_customer_message(chat_text: str) -> str:
return "\n".join(lines).strip()
def latest_customer_turn(chat_text: str) -> str:
"""合并聊天末尾由同一位客户连续发送的多个消息气泡。"""
lines = [line.strip() for line in str(chat_text or "").splitlines() if line.strip()]
if not lines:
return ""
blocks = []
current = None
for line in lines:
match = _chat_header_match(line)
if match:
if current is not None:
blocks.append(current)
current = {
"speaker": match.group("speaker").strip(),
"content": [],
}
elif current is not None:
current["content"].append(line)
if current is not None:
blocks.append(current)
if not blocks:
return "\n".join(lines).strip()
tail_speaker = blocks[-1]["speaker"]
merged = []
for block in reversed(blocks):
if block["speaker"] != tail_speaker:
break
content = "\n".join(block["content"]).strip()
if content:
merged.append(content)
return "\n".join(reversed(merged)).strip()
def media_text_content(chat_text: str) -> str:
"""Return the current customer turn with media placeholders removed."""
current = latest_customer_turn(chat_text)
if not current:
return ""
cleaned = _MEDIA_PLACEHOLDER_RE.sub("", current).strip()
if "voice" in detect_media_types(current):
# 企业微信有的版本复制成“[语音] 00:05”，也可能把时长单独放一行。
# 时长不是转写，必须删除；真正显示的转写文字仍会保留。
cleaned = "\n".join(
line
for line in cleaned.splitlines()
if line.strip()
and not _VOICE_DURATION_ONLY_RE.fullmatch(line.strip())
and not _VOICE_CONTROL_ONLY_RE.fullmatch(line.strip())
).strip()
if not cleaned:
return ""
return cleaned.strip(" \t\r\n，,。；;：:")
def media_archive_text(
chat_text: str,
media_types=None,
*,
voice_transcribed: bool = False,
) -> str:
"""Build a non-sensitive archive entry without storing screenshot contents."""
kinds = set(
detect_media_types(latest_customer_turn(chat_text))
if media_types is None
else media_types
)
visible_text = media_text_content(chat_text)
labels = {
"image": "（客户发来图片）",
"sticker": "（客户发来表情）",
"voice": (
"（客户发来语音及可见文字/转写）"
if visible_text
else (
"（客户发来语音，视觉确认已有转写）"
if voice_transcribed
else "（客户发来语音，未取得转写）"
)
),
"video": "（客户发来视频）",
"file": "（客户发来文件）",
}
parts = [labels[kind] for kind in ("image", "sticker", "voice", "video", "file") if kind in kinds]
if visible_text:
parts.append(visible_text)
if not parts:
parts.append("（客户发来非文字消息，内容未识别）")
return "\n".join(parts)
def safe_media_reply(media_types=None) -> str:
"""Conservative fallback used when the configured model cannot read media."""
kinds = set(media_types or ())
if "voice" in kinds:
return "这条语音我这边暂时没法准确听清，麻烦您把重点打成文字发我一下。"
if "image" in kinds:
return "图片我收到了，这边暂时没看清具体内容，您想让我重点看哪一处？"
if "sticker" in kinds:
return "看到您发的表情啦，您接着说，我在看。"
if "video" in kinds:
return "视频我收到了，这边暂时没法准确读取内容，麻烦您把重点打字说一下。"
if "file" in kinds:
return "文件我收到了，麻烦您说下需要我重点看什么内容。"
return "收到您刚才的消息了，这边暂时没能完整识别，麻烦您把重点打字说一下。"
def _conversation_mode_instruction(latest: str) -> str:
text = str(latest or "").strip()
if _LOW_INFORMATION_RE.fullmatch(text):
return (
"【本轮信息很少】客户只发了语气词或标点。不要猜测误触手机、情绪、病情或任何原因；"
"自然表示自己在听，再用至多一个问题请对方继续说。"
)
if _CLARIFY_RE.fullmatch(text):
return (
"【本轮是追问澄清】客户是在说没听懂你上一句。"
@@ -154,6 +417,7 @@ def _call_dify(query: str, user: str = "wechat-rpa", conversation_id: str = "")
payload["conversation_id"] = conversation_id
url = _completions_url()
_log_request_diagnostics(url, "Dify chat-messages")
resp = requests.post(
url, headers=_headers(), json=payload, timeout=ai_config.AI_TIMEOUT
)
@@ -191,7 +455,7 @@ def _dify_query_from_chat(chat_text: str, history: list = None) -> str:
except Exception:
hosp = "甄养堂互联网医院"
agent = "客服"
latest = latest_customer_message(chat_text)
latest = latest_customer_turn(chat_text)
mode_instruction = _conversation_mode_instruction(latest)
rules = (
"【事实铁律｜必须遵守】\n"
@@ -206,7 +470,7 @@ def _dify_query_from_chat(chat_text: str, history: list = None) -> str:
"可轻提一句需要可来本院挂号；【禁止】直接说已帮您预约。\n"
f"6. 仅当客户明确说要挂号/预约/面诊/帮我约 时，才说「已帮您预约了，稍后预约上了再联系您」，"
f"医院是{hosp}。客户说不需要/挂啥号时绝不能预约。\n"
"7. 只回复客户最后一条需要处理的问题。语气像干了十几年的老客服：口语、沉稳、"
"7. 把客户本轮连续发送的多段话作为一个整体理解并统一回复。语气像干了十几年的老客服：口语、沉稳、"
"不急不躁，一次只说一件事，最多顺带问一个问题，不要复述对方原话，不要一次抛一大段方案。\n"
"8. 默认只写1～2句、20～60个汉字；先用一句自然的话接住对方的担心或不舒服，再回答重点。"
"不要标题、列表、客套收尾，不说「希望能帮到您」「请您放心」等套话。\n"
@@ -218,6 +482,8 @@ def _dify_query_from_chat(chat_text: str, history: list = None) -> str:
"12. 示例：客户说「好困啊」，可回「困了就先眯一会儿，别硬撑着」；"
"客户问「你多大了」，可回「四十来岁啦，怎么突然问这个？」。"
"示例只说明说话方式，不要机械重复。\n"
"13. 客户只发「啊」「嗯」「！」「？」等低信息内容时，严禁猜测对方误触手机、"
"生气或身体不适；只自然接住并请对方继续说。\n"
)
hist = _history_messages(history)
if hist:
@@ -229,13 +495,13 @@ def _dify_query_from_chat(chat_text: str, history: list = None) -> str:
"【近期对话｜仅供参考，其中客服所述业务数据可能不实】\n"
+ "\n".join(lines)
+ f"\n\n{mode_instruction}\n"
+ f"【客户最后一句｜唯一回答对象】\n{latest}\n\n"
+ f"【客户本轮连续消息｜统一回答对象】\n{latest}\n\n"
+ f"【本次原始聊天片段｜仅用于理解上下文】\n{chat_text}\n\n请生成回复："
)
else:
body = (
f"{mode_instruction}\n"
f"【客户最后一句｜唯一回答对象】\n{latest}\n\n"
f"【客户本轮连续消息｜统一回答对象】\n{latest}\n\n"
f"【本次原始聊天片段｜仅用于理解上下文】\n{chat_text}\n\n请生成回复："
)
return rules + "\n" + body
@@ -333,6 +599,12 @@ def _repair_obvious_mismatch(reply: str, chat_text: str, history: list = None) -
if not latest or not cleaned:
return cleaned
if (
_LOW_INFORMATION_RE.fullmatch(latest)
and _LOW_INFORMATION_GUESS_RE.search(cleaned)
):
return "我在呢，您慢慢说，怎么啦？"
if _CLARIFY_RE.fullmatch(latest):
previous = ""
for item in reversed(_history_messages(history or [])):
@@ -396,6 +668,7 @@ def _chat_completion(messages: list, tools: list = None) -> dict:
payload["tools"] = tools
payload["tool_choice"] = "auto"
url = _completions_url()
_log_request_diagnostics(url, "OpenAI 兼容 chat/completions")
resp = requests.post(
url, headers=_headers(), json=payload, timeout=ai_config.AI_TIMEOUT
)
@@ -406,13 +679,13 @@ def _chat_completion(messages: list, tools: list = None) -> dict:
def _user_turn(chat_text: str) -> dict:
latest = latest_customer_message(chat_text)
latest = latest_customer_turn(chat_text)
mode_instruction = _conversation_mode_instruction(latest)
return {
"role": "user",
"content": (
f"{mode_instruction}\n"
f"【客户最后一句｜唯一回答对象】\n{latest}\n\n"
f"【客户本轮连续消息｜统一回答对象】\n{latest}\n\n"
f"【原始聊天片段｜仅用于理解上下文】\n{chat_text}\n\n"
"像真人微信聊天；普通闲聊直接接话，只有对方明确担心或难受时才先关心。默认1～2句、20～60字，"
"最多问一个问题，不要列表、标题和客套收尾。"
@@ -443,7 +716,7 @@ def call_ai_text(chat_text: str, history: list = None) -> str:
history,
)
except Exception as e:
print(f" [MCP] ⚠ 工具增强失败，回退普通回复: {e}")
print(f" [MCP] [!] 工具增强失败，回退普通回复: {e}")
messages = [{"role": "system", "content": _system_prompt()}]
messages += _history_messages(history)
@@ -513,12 +786,462 @@ async def _call_ai_text_with_mcp(chat_text: str, history: list = None) -> str:
return _strip_thinking(msg.get("content") or "")
def call_ai_vision(image_bytes: bytes, history: list = None) -> str:
"""
视觉模式：将聊天区域截图发给多模态 AI，让 AI 直接阅读并回复。
image_bytes 为 PNG 图片的 bytes。
history 为该会话的历史上下文（多轮记忆），可为 None。
"""
_UI_GUARD_STATES = {
"chat_ready",
"blocking_modal",
"non_message_page",
"security_verification",
"unknown",
}
_UI_GUARD_ACTIONS = {"none", "escape", "close_modal", "open_messages"}
def _dify_api_root() -> str:
"""返回 Dify App API 根地址（通常以 /v1 结尾）。"""
endpoint = _completions_url().rstrip("/")
for suffix in ("/chat-messages", "/completion-messages"):
if endpoint.lower().endswith(suffix):
return endpoint[: -len(suffix)]
return endpoint
def _call_dify_with_image(
query: str,
image_bytes: bytes,
*,
user: str = "wechat-rpa-vision",
timeout: float | None = None,
) -> str:
"""按 Dify App API 的“上传文件 → chat-messages 引用文件”流程调用视觉应用。"""
timeout = timeout or ai_config.AI_TIMEOUT
api_root = _dify_api_root()
upload_url = f"{api_root}/files/upload"
headers = {
"Authorization": f"Bearer {(ai_config.AI_API_KEY or '').strip()}",
"Accept": "application/json",
}
_log_request_diagnostics(upload_url, "Dify 文件上传（AI 页面守护）")
upload = requests.post(
upload_url,
headers=headers,
data={"user": user},
files={"file": ("wecom-ui.png", image_bytes, "image/png")},
timeout=timeout,
)
if not upload.ok:
raise RuntimeError(
f"Dify 截图上传失败 {upload.status_code}: {(upload.text or '')[:300]}"
)
upload_id = str((upload.json() or {}).get("id") or "").strip()
if not upload_id:
raise RuntimeError("Dify 截图上传成功，但响应中没有文件 ID")
payload = {
"inputs": {},
"query": query,
"response_mode": "blocking",
"user": user,
"files": [
{
"type": "image",
"transfer_method": "local_file",
"upload_file_id": upload_id,
}
],
}
chat_url = f"{api_root}/chat-messages"
_log_request_diagnostics(chat_url, "Dify 视觉 chat-messages")
response = requests.post(
chat_url,
headers=_headers(),
json=payload,
timeout=timeout,
)
if not response.ok:
raise RuntimeError(
f"Dify 视觉请求失败 {response.status_code}: {(response.text or '')[:300]}"
)
data = response.json() or {}
answer = data.get("answer") or (data.get("data") or {}).get("answer")
if not answer:
raise RuntimeError("Dify 视觉请求未返回 answer")
return str(answer)
def _parse_ui_guard_decision(raw: str) -> dict:
"""从模型文本中提取并收紧为页面守护允许的结构和动作。"""
text = _strip_thinking(str(raw or "")).strip()
candidates = []
decoder = json.JSONDecoder()
for index, char in enumerate(text):
if char != "{":
continue
try:
value, _ = decoder.raw_decode(text[index:])
except json.JSONDecodeError:
continue
# 模型有时会先输出调试/计量 JSON，随后才给页面守护协议对象。
# 只有同时显式包含 state 与 action 的对象才有执行资格，不能让第一个
# 无关 dict 抢占解析结果。
if isinstance(value, dict) and "state" in value and "action" in value:
candidates.append(value)
if not candidates:
return {
"state": "unknown",
"action": "none",
"confidence": 0.0,
"reason": "模型未返回有效 JSON",
}
# 同一次响应出现两个互相矛盾的协议对象时，无法证明哪个才是最终判断。
# 失败关闭比任意选择一个并执行 Esc/点击更安全；完全重复的对象不算冲突。
decisions = {
(
str(item.get("state") or "unknown").strip().lower(),
str(item.get("action") or "none").strip().lower(),
)
for item in candidates
}
if len(decisions) != 1:
return {
"state": "unknown",
"action": "none",
"confidence": 0.0,
"reason": "模型返回冲突 JSON",
}
data = candidates[-1]
state = str(data.get("state") or "unknown").strip().lower()
action = str(data.get("action") or "none").strip().lower()
if state not in _UI_GUARD_STATES:
state = "unknown"
if action not in _UI_GUARD_ACTIONS:
action = "none"
try:
confidence = float(data.get("confidence", 0.0))
except (TypeError, ValueError):
confidence = 0.0
confidence = max(0.0, min(1.0, confidence))
# 状态与动作必须匹配。即使模型返回了额外动作，也不能越过本地白名单。
allowed_by_state = {
"chat_ready": {"none"},
"blocking_modal": {"none", "escape", "close_modal"},
"non_message_page": {"none", "escape", "open_messages"},
"security_verification": {"none"},
"unknown": {"none"},
}
if action not in allowed_by_state[state]:
action = "none"
return {
"state": state,
"action": action,
"confidence": confidence,
"reason": str(data.get("reason") or "")[:120],
}
def classify_wecom_ui(image_bytes: bytes, trigger: str = "页面异常") -> dict:
"""用视觉模型判断企业微信当前页面；只返回受限动作，不返回点击坐标。"""
if not image_bytes:
return _parse_ui_guard_decision("")
if _provider_type() == "comfyui":
return {
"state": "unknown",
"action": "none",
"confidence": 0.0,
"reason": "ComfyUI 不用于界面判断",
}
prompt = (
"你是企业微信桌面端的页面安全分类器。判断截图是否妨碍客服自动回复。"
f"触发环节：{trigger}。只输出一个 JSON 对象，禁止输出解释、Markdown 或点击坐标。\n"
"state 只能是：chat_ready（聊天页和输入框可用）、blocking_modal（弹窗/浮层遮挡）、"
"non_message_page（文档、微盘、工作台、会议等非消息页面）、"
"security_verification（登录、扫码、安全验证）、unknown。\n"
"action 只能是：none、escape、close_modal、open_messages。"
"安全验证必须 none；不确定必须 unknown+none；普通聊天页必须 chat_ready+none。\n"
"格式：{\"state\":\"...\",\"action\":\"...\","
"\"confidence\":0.0,\"reason\":\"不超过30字\"}"
)
timeout = min(float(getattr(ai_config, "AI_TIMEOUT", 120) or 120), 30.0)
if _is_dify_endpoint():
raw = _call_dify_with_image(
prompt,
image_bytes,
user="wechat-rpa-ui-guard",
timeout=timeout,
)
return _parse_ui_guard_decision(raw)
encoded = base64.b64encode(image_bytes).decode("ascii")
payload = {
"model": ai_config.AI_MODEL,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{encoded}"},
},
],
}
],
"max_tokens": 180,
"temperature": 0,
}
url = _completions_url()
_log_request_diagnostics(url, "OpenAI 兼容 AI 页面守护")
response = requests.post(
url,
headers=_headers(),
json=payload,
timeout=timeout,
)
if not response.ok:
raise RuntimeError(
f"AI 页面判断失败 {response.status_code}: {(response.text or '')[:300]}"
)
raw = response.json()["choices"][0]["message"]["content"]
return _parse_ui_guard_decision(raw)
def _vision_chat_prompt(chat_text: str = "", media_types=None) -> str:
kinds = set(
detect_media_types(latest_customer_turn(chat_text))
if media_types is None
else media_types
)
kind_names = {
"image": "图片",
"sticker": "表情/贴纸",
"voice": "语音",
"video": "视频",
"file": "文件",
}
detected = "、".join(
kind_names[kind]
for kind in ("image", "sticker", "voice", "video", "file")
if kind in kinds
) or "剪贴板未识别具体类型"
current = latest_customer_turn(chat_text) or "（剪贴板未提取到本轮文字，请只依据截图中可见的新内容判断）"
return (
"这是企业微信聊天消息区域的局部截图。通常左侧气泡是客户，右侧气泡是我方；"
"请同时按气泡左右位置、时间顺序和本轮文字判断，不要把我方旧回复当成客户消息。\n"
f"【剪贴板检测到的媒体】{detected}\n"
f"【本轮可提取文字】\n{current}\n\n"
"把客户在本轮约20秒内连续发送的文字、图片和表情作为一个整体理解后统一回复。"
"动画表情内部换帧不代表又来了一条消息；只有最末端新出现的客户侧气泡，"
"或位于最近一条我方气泡之后的客户连续气泡，才算本轮新消息。"
"图片或贴纸只能描述截图里确实可见的信息；看不清就自然追问，不得编造病情、处方、"
"订单、物流或其他业务事实，也不得仅凭图片声称已经修改、提交、预约或执行任何操作。\n"
"重要：截图不含语音声音。绝对不要根据波形或时长猜语音内容。\n"
"只输出一个 JSON 对象，不要 Markdown 或解释，字段固定为："
'{"has_new_customer_message":true,"media_type":"text|image|sticker|voice|video|file|mixed|unknown",'
'"media_types":["image"],"contains_voice":false,"voice_transcribed":false,'
'"reply":"可直接发送的回复"}。media_types 只能包含 image、sticker、voice、video、file；'
"mixed 时必须列出全部可确认类型。\n"
"若最新内容其实是我方发送的或没有新客户消息，has_new_customer_message=false 且 reply 为空；"
"若 mixed 中包含任何语音，contains_voice 必须为 true；不含语音则必须为 false。"
"若是语音且截图/本轮文字里没有可见转写，media_type=voice、contains_voice=true、"
"voice_transcribed=false、reply 为空。"
"其他情况 reply 默认1～2句、20～60字，最多问一个问题，不要标题或列表。"
)
def _finalize_vision_reply(raw: str, chat_text: str = "", media_types=None) -> str:
cleaned = _strip_thinking(str(raw or "")).strip()
# Some otherwise capable models prepend a small analysis JSON object before
# the requested protocol object. Taking the first arbitrary dict turns a
# valid result into ``has_new=False`` because the field is absent. Collect
# only protocol-shaped objects and use the last one (the final answer).
protocol_objects = []
decoder = json.JSONDecoder()
index = 0
while index < len(cleaned):
object_start = cleaned.find("{", index)
if object_start < 0:
break
try:
value, consumed = decoder.raw_decode(cleaned[object_start:])
except json.JSONDecodeError:
index = object_start + 1
continue
if isinstance(value, dict) and "has_new_customer_message" in value:
protocol_objects.append(value)
# Skip the whole decoded object. This prevents a nested diagnostic dict
# from overriding the enclosing protocol result when we choose the last
# top-level answer.
index = object_start + max(1, consumed)
data = protocol_objects[-1] if protocol_objects else None
# Keep compatibility with the two internal sentinels, but do not search for
# them as substrings inside a JSON reply: customer-visible text could contain
# those words and must not silently override a valid structured result.
if data is None:
standalone = cleaned.strip().strip("`").strip().upper()
if standalone == VISION_VOICE_NEEDS_TEXT.upper():
return VISION_VOICE_NEEDS_TEXT
if standalone == VISION_NO_INCOMING.upper():
return VISION_NO_INCOMING
return ""
def protocol_bool(value):
if isinstance(value, bool):
return value
if isinstance(value, int) and value in (0, 1):
return bool(value)
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes"}:
return True
if normalized in {"0", "false", "no"}:
return False
return None
has_new = protocol_bool(data.get("has_new_customer_message"))
if has_new is None:
return ""
if not has_new:
return VISION_NO_INCOMING
allowed_kinds = {"image", "sticker", "voice", "video", "file"}
media_type_value = data.get("media_type", "unknown")
if not isinstance(media_type_value, str):
return ""
media_type = media_type_value.strip().lower() or "unknown"
if media_type not in allowed_kinds | {"text", "mixed", "unknown"}:
return ""
kinds = set(
detect_media_types(latest_customer_turn(chat_text))
if media_types is None
else media_types
)
kinds.intersection_update(allowed_kinds)
transcribed = protocol_bool(data.get("voice_transcribed", False))
contains_voice = protocol_bool(data.get("contains_voice", False))
if transcribed is None or contains_voice is None:
if "voice" in kinds:
return VisionReply(
VISION_VOICE_NEEDS_TEXT,
media_types=kinds,
voice_transcribed=False,
)
return ""
response_kinds = set()
raw_response_kinds = data.get("media_types")
if raw_response_kinds is not None:
if not isinstance(raw_response_kinds, (list, tuple, set)):
if "voice" in kinds:
return VisionReply(
VISION_VOICE_NEEDS_TEXT,
media_types=kinds,
voice_transcribed=bool(
transcribed and media_text_content(chat_text)
),
)
return ""
normalized_response_kinds = [
str(item).strip().lower() for item in raw_response_kinds
]
if any(item not in allowed_kinds for item in normalized_response_kinds):
if "voice" in kinds:
return VisionReply(
VISION_VOICE_NEEDS_TEXT,
media_types=kinds,
voice_transcribed=bool(
transcribed and media_text_content(chat_text)
),
)
return ""
response_kinds.update(normalized_response_kinds)
claimed_kinds = set(response_kinds)
if media_type in allowed_kinds:
claimed_kinds.add(media_type)
if contains_voice:
claimed_kinds.add("voice")
# Clipboard evidence or the model's own declaration of a voice bubble is
# stronger than any remaining schema inconsistency. A screenshot can never
# reveal audio, so always collapse to the non-guessing voice path while still
# preserving other media types for the archive.
if "voice" in kinds | claimed_kinds:
return VisionReply(
VISION_VOICE_NEEDS_TEXT,
media_types=(kinds | claimed_kinds | {"voice"}),
voice_transcribed=bool(transcribed and media_text_content(chat_text)),
)
# media_type and media_types are two views of the same model decision. A
# primitive type cannot disagree with, or omit a different type from, its
# list; multiple media must be declared as mixed.
if media_type in allowed_kinds:
if response_kinds and response_kinds != {media_type}:
return ""
response_kinds.add(media_type)
elif media_type == "mixed":
if "contains_voice" not in data or not response_kinds:
return ""
elif response_kinds:
# text/unknown with a non-empty media list is internally inconsistent.
return ""
if contains_voice:
response_kinds.add("voice")
all_kinds = kinds | response_kinds
if transcribed:
# A transcription flag without a voice is impossible under the contract.
return ""
if kinds:
if media_type in {"text", "unknown"}:
return ""
if media_type in allowed_kinds and kinds != {media_type}:
return ""
if media_type == "mixed" and not kinds.issubset(response_kinds):
return ""
# 对“剪贴板完全为空 + 模型也无法判断媒体类型”的结果不采纳正文，
# 避免把未转写语音误当成图片并生成臆测回复。
if media_type == "unknown" and not str(chat_text or "").strip():
return ""
reply = data.get("reply", "")
if not isinstance(reply, str):
return ""
return VisionReply(
_humanize(reply),
media_types=all_kinds,
voice_transcribed=bool(transcribed),
)
def call_ai_vision(
image_bytes: bytes,
history: list = None,
chat_text: str = "",
media_types=None,
) -> str:
"""Read a chat-area screenshot together with copied text and archive history."""
prompt = _vision_chat_prompt(chat_text, media_types)
if _is_dify_endpoint():
# Dify's image request is a single query, so include the same conversation
# rules and current copied text in that query before attaching the screenshot.
query_text = chat_text or "（客户本轮发来一条无法复制为文字的消息）"
query = _dify_query_from_chat(query_text, history) + "\n\n【媒体识别规则】\n" + prompt
raw = _call_dify_with_image(
query,
image_bytes,
user="wechat-rpa-chat-vision",
)
return _finalize_vision_reply(raw, chat_text, media_types)
b64 = base64.b64encode(image_bytes).decode("utf-8")
simple_headers = {
"Authorization": f"Bearer {ai_config.AI_API_KEY}",
@@ -530,16 +1253,7 @@ def call_ai_vision(image_bytes: bytes, history: list = None) -> str:
"model": ai_config.AI_MODEL,
"messages": messages + [
{"role": "user", "content": [
{
"type": "text",
"text": (
"这是一个聊天对话窗口的截图。"
"左边的灰色气泡是对方（客户）发的消息，右边的蓝色气泡是我方之前的回复。"
"请只关注对方（客户）发的最后一条消息，针对那条消息直接回复。"
"像真人微信聊天，先关心一句，再说重点；默认1～2句、20～60字，最多问一个问题。"
"只输出回复内容，不要描述图片，不要解释，不要加引号、标题或列表。"
),
},
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{b64}"},
@@ -550,22 +1264,34 @@ def call_ai_vision(image_bytes: bytes, history: list = None) -> str:
"temperature": ai_config.AI_TEMPERATURE,
}
url = _completions_url()
_log_request_diagnostics(url, "OpenAI 兼容视觉请求")
resp = requests.post(url, headers=simple_headers, json=payload, timeout=ai_config.AI_TIMEOUT)
resp.raise_for_status()
content = resp.json()["choices"][0]["message"]["content"]
return _humanize(_strip_thinking(content))
return _finalize_vision_reply(content, chat_text, media_types)
def get_ai_reply(chat_text: str = None, image_bytes: bytes = None, history: list = None) -> str:
"""
统一入口：根据 AI_USE_VISION 配置自动选择模式。
history 为该会话的历史上下文（多轮记忆），可为 None。
返回 AI 生成的回复文本。
"""
def get_ai_reply(
chat_text: str = None,
image_bytes: bytes = None,
history: list = None,
*,
force_vision: bool = False,
media_types=None,
) -> str:
"""Unified text/vision entry; media fallback may explicitly force vision."""
try:
if ai_config.AI_USE_VISION and image_bytes:
if _provider_type() == "comfyui":
print(" [AI] [!] 当前配置为 ComfyUI 文生图服务，不能用于企微文本自动回复")
return ""
if image_bytes and (force_vision or ai_config.AI_USE_VISION):
print(" [AI] 使用视觉模式分析聊天截图...")
return call_ai_vision(image_bytes, history=history)
return call_ai_vision(
image_bytes,
history=history,
chat_text=chat_text or "",
media_types=media_types,
)
elif chat_text:
if getattr(ai_config, "AI_MCP_ENABLED", False):
print(" [AI] 文本模式 + MCP 工具增强...")
@@ -575,11 +1301,11 @@ def get_ai_reply(chat_text: str = None, image_bytes: bytes = None, history: list
else:
return ""
except requests.exceptions.Timeout:
print(" [AI] ⚠ API 请求超时")
print(" [AI] [!] API 请求超时")
return ""
except requests.exceptions.RequestException as e:
print(f" [AI] ⚠ API 请求失败: {e}")
print(f" [AI] [!] API 请求失败: {e}")
return ""
except (KeyError, IndexError, json.JSONDecodeError) as e:
print(f" [AI] ⚠ 解析响应失败: {e}")
print(f" [AI] [!] 解析响应失败: {e}")
return ""