Files
kefu/wechat_rpa/test_review_policy_regression.py
T
2026-09-21 10:34:06 +08:00

429 lines
22 KiB
Python

"""Pure-memory review regressions. No application imports, UI, queue or network I/O.
Run once per deployment root:
REVIEW_POLICY_ROOT=C:/wechat_rpa python -B -m unittest <this file>
The actual review helper and selected WeChatBot methods are compiled from that root.
"""
from __future__ import annotations
import ast
import copy
import importlib.util
import os
from pathlib import Path
import sys
import threading
import time
from types import ModuleType
import unittest
from unittest import mock
ROOT = Path(os.environ.get('REVIEW_POLICY_ROOT') or Path(__file__).resolve().parent)
POLICY_PATH = ROOT / 'review_policy.py'
spec = importlib.util.spec_from_file_location('review_policy', POLICY_PATH)
policy = importlib.util.module_from_spec(spec)
sys.modules['review_policy'] = policy
spec.loader.exec_module(policy)
def isolated_visual_class():
path = ROOT / 'wechat_bot.py'
tree = ast.parse(path.read_text(encoding='utf-8-sig'), filename=str(path))
original = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'WeChatBot')
names = {
'_review_mode_enabled', '_reply_needs_review', '_drop_stale_review_flags',
'_record_model_call', '_knowledge_trace_metadata', '_orchestrated_reply',
'_reset_pending_batch', '_reset_pending_batch_unlocked',
}
selected = [node for node in original.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name in names]
missing = names - {node.name for node in selected}
if missing:
raise AssertionError('Production methods missing: ' + repr(missing))
cls = ast.ClassDef(name='WeChatBot', bases=[], keywords=[], body=selected, decorator_list=[])
module = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0), cls], type_ignores=[])
namespace = dict(vars(policy), review_policy=policy, time=time, threading=threading,
SEND_MODE_AUTO='auto', SEND_MODE_REVIEW='review')
exec(compile(ast.fix_missing_locations(module), str(path), 'exec'), namespace)
return namespace['WeChatBot']
WeChatBot = isolated_visual_class()
class MemoryFixtures(unittest.TestCase):
def setUp(self):
self.config = ModuleType('ai_config')
self.config.AI_REVIEW_RULES = []
self.config.AI_JUDGE_ENABLED = False
self.config.AI_JUDGE_MODE = 'shadow'
self.settings = {'gateway': {'enabled': False, 'url': ''}, 'model_plan': {'roles': {}}}
self.backend = ModuleType('backend_client')
self.backend.load_settings = mock.Mock(side_effect=lambda: copy.deepcopy(self.settings))
self.backend.report_model_call = mock.Mock()
self.backend.device_id = mock.Mock(return_value='memory-test')
self.ai_chat = ModuleType('ai_chat')
self.ai_chat.gateway_provider = mock.Mock(return_value=None)
self.ai_chat.get_ai_reply = mock.Mock(return_value='reply')
self.router = ModuleType('model_router')
self.modules = mock.patch.dict(sys.modules, {
'ai_config': self.config, 'backend_client': self.backend,
'ai_chat': self.ai_chat, 'model_router': self.router,
})
self.modules.start()
self.addCleanup(self.modules.stop)
self.fp = b't' * 40
self.key = self.fp.hex()
self.state = dict(awaiting_review=True, review_origin='rule', stage='manual_review',
review_reason='命中审核规则「rule」', staged_user_text='question',
staged_reply_text='reply', reply_text='reply', judge_risk='low')
self.bot = WeChatBot()
self.bot.send_mode = 'auto'
self.bot._pending_lock = threading.RLock()
self.bot._pending_reply_sessions = {self.key: self.state}
self.bot._pending_reply_state = lambda fp: self.bot._pending_reply_sessions.get(bytes(fp).hex())
self.bot._persist_pending_replies = mock.Mock(return_value=True)
self.bot._forget_uncertain_tracking = mock.Mock()
self.bot._queue_log_instance = mock.Mock()
self.bot._backend_model_plan = {}
self.bot._call_model_with_observer = lambda callback, *args, **kwargs: callback(*args, **kwargs)
self.bot._model_plan = mock.Mock(return_value=([object(), object()], object(), 'shadow'))
def gateway_mode(self, mode):
self.settings = {'gateway': {'enabled': True, 'url': 'https://memory.invalid/gateway'},
'model_plan': {'roles': {'judge_mode': mode}}}
def drop(self):
with mock.patch('builtins.print'):
self.bot._drop_stale_review_flags()
def outcome(self, **kwargs):
value = {'reply': 'reply', 'judge_mode': 'shadow',
'judge': {'participated': True, 'risk': 'high', 'score': .2, 'reason': 'observed'},
'chosen': 'test-model', 'candidates': []}
value.update(kwargs)
return value
class ReviewPolicyTests(MemoryFixtures):
def test_high_risk_shadow_observes_and_enforcing_modes_hold(self):
for mode, expected in [('shadow', False), ('score_only', True), ('arbitrate', True), ('', True), ('nonsense', True)]:
with self.subTest(mode=mode):
result = policy.judge_review_reason({'judge_risk': 'high', 'judge_mode': mode}, current_mode='')
self.assertEqual(bool(result), expected)
def test_missing_mode_and_unknown_current_plan_remain_manual(self):
self.assertTrue(policy.judge_review_reason({'judge_risk': 'high'}))
def test_explicit_current_shadow_rechecks_old_high_risk(self):
for old_mode in ('', 'nonsense', None):
with self.subTest(old_mode=old_mode):
self.assertEqual(policy.judge_review_reason({'judge_risk': 'high', 'judge_mode': old_mode}, current_mode='shadow'), '')
def test_recorded_enforcing_mode_cannot_be_overridden_by_current_shadow(self):
for mode in ('score_only', 'arbitrate'):
with self.subTest(mode=mode):
self.assertTrue(policy.judge_review_reason(
{'judge_risk': 'high', 'judge_mode': mode}, current_mode='shadow'))
def test_valid_recorded_mode_never_reads_current_plan(self):
with mock.patch.object(policy, 'current_judge_mode',
side_effect=AssertionError('Recorded mode is authoritative')):
for mode, expected in (('shadow', False), ('score_only', True), ('arbitrate', True)):
with self.subTest(mode=mode):
reason = policy.judge_review_reason({'judge_risk': 'high', 'judge_mode': mode})
self.assertEqual(bool(reason), expected)
def test_current_enforcing_plan_does_not_turn_old_shadow_observation_into_hold(self):
for current in ('score_only', 'arbitrate'):
with self.subTest(current=current):
self.assertEqual(policy.judge_review_reason({'judge_risk': 'high', 'judge_mode': 'shadow'}, current_mode=current), '')
def test_non_high_risks_do_not_trigger_judge_review(self):
for risk in ('low', 'medium', 'unknown', '', None):
with self.subTest(risk=risk):
self.assertEqual(policy.judge_review_reason({'judge_risk': risk, 'judge_mode': 'arbitrate'}, current_mode=''), '')
def test_shadow_still_honors_enabled_keyword_rule(self):
reason = policy.selective_review_reason({'judge_risk': 'high', 'judge_mode': 'shadow'}, 'QUESTION', 'reply',
rules=[{'label': 'active', 'enabled': True, 'keywords': ['question']}], current_mode='')
self.assertIn('active', reason)
def test_disabled_rules_cannot_hold_customer_or_reply_text(self):
for customer, reply in [('trigger', 'reply'), ('question', 'trigger')]:
with self.subTest(customer=customer):
self.assertEqual(policy.selective_review_reason({}, customer, reply,
rules=[{'label': 'disabled', 'enabled': False, 'keywords': ['trigger']}], current_mode=''), '')
def test_gateway_current_mode_uses_only_this_process_settings(self):
self.gateway_mode('arbitrate')
self.config.AI_JUDGE_ENABLED = True
self.config.AI_JUDGE_MODE = 'shadow'
with mock.patch('builtins.open', side_effect=AssertionError('Cross-directory configuration read')), \
mock.patch.object(Path, 'read_text', side_effect=AssertionError('Cross-directory configuration read')):
self.assertEqual(policy.current_judge_mode(), 'arbitrate')
self.backend.load_settings.assert_called()
def test_gateway_shadow_is_effective_for_legacy_record(self):
self.gateway_mode('shadow')
self.assertEqual(policy.current_judge_mode(), 'shadow')
self.assertEqual(policy.judge_review_reason({'judge_risk': 'high'}), '')
def test_missing_or_invalid_gateway_mode_is_unknown(self):
self.config.AI_JUDGE_ENABLED = True
self.config.AI_JUDGE_MODE = 'shadow'
for roles in ({}, {'judge_mode': ''}, {'judge_mode': 'invalid'}, None):
with self.subTest(roles=roles):
self.settings = {'gateway': {'enabled': True, 'url': 'https://memory.invalid/gateway'},
'model_plan': {'roles': roles}}
self.assertEqual(policy.current_judge_mode(), '')
self.assertTrue(policy.judge_review_reason({'judge_risk': 'high'}))
def test_disabled_gateway_cache_cannot_override_local_mode(self):
self.settings['model_plan']['roles']['judge_mode'] = 'shadow'
self.config.AI_JUDGE_ENABLED = True
self.config.AI_JUDGE_MODE = 'arbitrate'
self.assertEqual(policy.current_judge_mode(), 'arbitrate')
def test_local_judge_disabled_means_unknown_even_with_shadow_default(self):
self.assertEqual(policy.current_judge_mode(), '')
self.assertTrue(policy.judge_review_reason({'judge_risk': 'high'}))
def test_configuration_failure_preserves_high_risk_hold(self):
self.backend.load_settings.side_effect = RuntimeError('unavailable')
self.assertEqual(policy.current_judge_mode(), '')
self.assertTrue(policy.judge_review_reason({'judge_risk': 'high'}))
def test_recheck_rejects_any_send_state(self):
for state in ('sending', 'sent', 'sent_uncommitted', 'unknown', 'uncertain', 'failed', 'unexpected'):
with self.subTest(state=state):
self.assertFalse(policy.can_recheck_review(dict(self.state, send_state=state)))
def test_recheck_protects_receipt_and_manual_or_foreign_work(self):
variants = [dict(stage=stage) for stage in ('sending', 'unknown', 'uncertain', 'receipt_check')]
variants.extend([{'manual_required': True}, {'foreign_draft_text': 'human draft'},
{'foreign_draft_at': 123}, {'cancel_requested': True}])
for fields in variants:
with self.subTest(fields=fields):
self.assertFalse(policy.can_recheck_review(dict(self.state, **fields)))
def test_plain_review_with_draft_can_be_rechecked(self):
self.assertTrue(policy.can_recheck_review(self.state))
class VisualReviewIntegrationTests(MemoryFixtures):
def test_visual_shadow_high_risk_does_not_hold(self):
self.state.update(judge_risk='high', judge_mode='shadow')
self.assertEqual(self.bot._reply_needs_review(self.state, 'question', 'reply'), (False, ''))
def test_visual_enforcing_and_legacy_high_risk_still_hold(self):
for mode in ('score_only', 'arbitrate', ''):
with self.subTest(mode=mode):
self.state.update(judge_risk='high', judge_mode=mode)
self.assertTrue(self.bot._reply_needs_review(self.state, 'question', 'reply')[0])
def test_disabled_keyword_review_is_requeued_without_approval(self):
self.config.AI_REVIEW_RULES = [{'label': 'rule', 'keywords': ['question'], 'enabled': False}]
self.drop()
self.assertFalse(self.state.get('awaiting_review'))
self.assertFalse(self.state.get('review_reason'))
self.assertEqual(self.state['stage'], 'queued')
self.assertFalse(self.state.get('review_approved'))
self.assertEqual(self.state['staged_reply_text'], 'reply')
self.bot._persist_pending_replies.assert_called_once()
def test_active_keyword_rule_remains_held(self):
self.config.AI_REVIEW_RULES = [{'label': 'active', 'keywords': ['reply'], 'enabled': True}]
self.drop()
self.assertTrue(self.state['awaiting_review'])
self.assertIn('active', self.state['review_reason'])
def test_current_gateway_shadow_releases_known_old_model_hold(self):
self.gateway_mode('shadow')
self.state.update(judge_risk='high', review_reason='模型裁判评估为高风险')
self.drop()
self.assertFalse(self.state.get('awaiting_review'))
self.assertEqual(self.state['stage'], 'queued')
self.assertFalse(self.state.get('review_approved'))
def test_recorded_strict_visual_verdict_stays_held_with_cached_shadow(self):
self.gateway_mode('shadow')
for mode in ('score_only', 'arbitrate'):
with self.subTest(mode=mode):
self.state.update(judge_risk='high', judge_mode=mode,
awaiting_review=True, review_reason='模型裁判评估为高风险')
self.assertTrue(self.bot._reply_needs_review(self.state, 'question', 'reply')[0])
self.drop()
self.assertTrue(self.state['awaiting_review'])
self.assertEqual(self.state['stage'], 'manual_review')
def test_legacy_high_risk_without_valid_current_mode_stays_held(self):
self.state.update(judge_risk='high', review_reason='模型裁判评估为高风险')
self.drop()
self.assertTrue(self.state['awaiting_review'])
def test_unknown_historical_reason_is_not_automatically_released(self):
self.state.update(review_origin='legacy', review_reason='历史审核原因缺失,请人工确认')
self.drop()
self.assertTrue(self.state['awaiting_review'])
def test_global_review_mode_does_not_clear_any_hold(self):
self.bot.send_mode = 'review'
before = copy.deepcopy(self.state)
self.drop()
self.assertEqual(self.state, before)
self.bot._persist_pending_replies.assert_not_called()
def test_global_review_releases_in_auto_mode_when_unprotected(self):
self.state.update(review_origin='global', review_reason='')
self.drop()
self.assertFalse(self.state.get('awaiting_review'))
self.assertFalse(self.state.get('review_approved'))
def test_disabled_rules_do_not_release_protected_tasks(self):
variants = [dict(send_state=state) for state in ('sending', 'unknown', 'uncertain', 'failed')]
variants.extend([dict(stage=stage) for stage in ('sending', 'receipt_check', 'unknown', 'uncertain')])
variants.extend([{'manual_required': True}, {'foreign_draft_text': 'human draft'},
{'foreign_draft_at': 123}, {'cancel_requested': True}])
for fields in variants:
with self.subTest(fields=fields):
state = dict(self.state, **fields)
before = copy.deepcopy(state)
self.bot._pending_reply_sessions = {self.key: state}
self.drop()
self.assertEqual(state, before)
def test_missing_rule_evidence_stays_conservative(self):
self.state.pop('staged_user_text')
self.state.pop('staged_reply_text')
self.state.pop('reply_text')
self.drop()
self.assertTrue(self.state['awaiting_review'])
def test_direct_model_round_clears_previous_judge_metadata(self):
self.bot._model_plan.return_value = ([object()], None, 'shadow')
self.state.update(judge_risk='high', judge_mode='arbitrate')
self.assertEqual(self.bot._orchestrated_reply(self.fp, chat_text='new question'), 'reply')
self.assertFalse(self.state.get('judge_risk'))
self.assertFalse(self.state.get('judge_mode'))
def test_review_cleanup_rolls_back_when_save_returns_false(self):
before = copy.deepcopy(self.state)
self.bot._persist_pending_replies.return_value = False
self.drop()
self.assertIs(self.bot._pending_reply_sessions[self.key], self.state)
self.assertEqual(self.state, before)
self.assertTrue(self.state['awaiting_review'])
self.assertFalse(self.state.get('review_approved'))
def test_review_cleanup_rolls_back_when_save_raises(self):
other = dict(self.state, review_origin='global', review_reason='')
self.bot._pending_reply_sessions['other'] = other
before = copy.deepcopy(self.state)
other_before = copy.deepcopy(other)
self.bot._persist_pending_replies.side_effect = OSError('synthetic disk failure')
self.drop()
self.assertIs(self.bot._pending_reply_sessions[self.key], self.state)
self.assertIs(self.bot._pending_reply_sessions['other'], other)
self.assertEqual(self.state, before)
self.assertEqual(other, other_before)
self.assertTrue(self.state['awaiting_review'])
self.assertTrue(other['awaiting_review'])
def test_model_record_persists_actual_judge_mode(self):
self.state['model_task_id'] = 'task-current'
self.bot._record_model_call(self.fp, self.outcome(), task_id='task-current',
expected_task_id='task-current', expected_state=self.state)
self.assertEqual(self.state['judge_mode'], 'shadow')
self.assertEqual(self.state['judge_risk'], 'high')
self.bot._persist_pending_replies.assert_called_once()
def test_stale_callback_cannot_write_to_new_task_object(self):
old = self.state
self.state = {'model_task_id': 'new', 'staged_user_text': 'new question'}
self.bot._pending_reply_sessions[self.key] = self.state
before = copy.deepcopy(self.state)
self.bot._record_model_call(self.fp, self.outcome(), task_id='old', expected_task_id='old', expected_state=old)
self.assertEqual(self.state, before)
self.bot._persist_pending_replies.assert_not_called()
def test_stale_callback_cannot_write_after_inplace_reset_or_new_id(self):
for next_id in ('next', None):
with self.subTest(next_id=next_id):
self.state.clear()
if next_id is not None:
self.state['model_task_id'] = next_id
self.state['staged_user_text'] = 'new question'
before = copy.deepcopy(self.state)
self.bot._record_model_call(self.fp, self.outcome(), task_id='old', expected_task_id='old', expected_state=self.state)
self.assertEqual(self.state, before)
self.bot._persist_pending_replies.assert_not_called()
def test_orchestration_binds_callback_to_generation_before_model_starts(self):
captured = {}
def answer(**kwargs):
captured['callback'] = kwargs['on_verdict']
captured['task_id'] = self.state.get('model_task_id')
self.assertTrue(captured['task_id'])
self.assertFalse(self.state.get('judge_risk'))
self.assertFalse(self.state.get('judge_mode'))
return self.outcome(judge={'participated': False, 'risk': 'unknown'})
self.router.answer = answer
self.state.update(judge_risk='high', judge_mode='arbitrate')
self.assertEqual(self.bot._orchestrated_reply(self.fp, chat_text='question'), 'reply')
self.assertEqual(self.state['model_task_id'], captured['task_id'])
captured['callback'](self.outcome())
self.assertEqual(self.state['judge_risk'], 'high')
self.assertEqual(self.state['judge_mode'], 'shadow')
self.bot._reset_pending_batch(self.fp)
self.assertNotIn('model_task_id', self.state)
self.assertNotIn('judge_mode', self.state)
before = copy.deepcopy(self.state)
captured['callback'](self.outcome())
self.assertEqual(self.state, before)
@unittest.skipUnless((ROOT / 'protocol_engine.py').exists(), 'visual distribution has no protocol engine')
class ProtocolGenerationIdentityTests(MemoryFixtures):
def test_old_gateway_or_local_result_cannot_overwrite_new_taskid_on_same_object(self):
path = ROOT / 'protocol_engine.py'
tree = ast.parse(path.read_text(encoding='utf-8-sig'), filename=str(path))
original = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'ProtocolBot')
method = next(n for n in original.body if isinstance(n, ast.FunctionDef) and n.name == '_generate')
cls = ast.ClassDef(name='ProtocolBot', bases=[], keywords=[], body=[method], decorator_list=[])
module = ast.Module(body=[cls], type_ignores=[])
namespace = {'session_key': lambda account, conv: account + '/' + conv,
'model_context_text': lambda context: context['text']}
exec(compile(ast.fix_missing_locations(module), str(path), 'exec'), namespace)
self.config.AI_ENABLED = True
self.config.AI_JUDGE_ENABLED = True
self.config.AI_JUDGE_MODE = 'score_only'
for gateway in (True, False):
with self.subTest(gateway=gateway):
bot = namespace['ProtocolBot']()
bot.account = 'memory-account'
bot._lock = threading.RLock()
state = {'model_task_id': 'previous', 'judge_risk': ''}
bot._pending = {'memory-account/memory-conv': state}
next_state = {'model_task_id': 'new-generation', 'judge_risk': 'high', 'judge_mode': 'arbitrate'}
self.ai_chat.gateway_provider = lambda: object() if gateway else None
self.ai_chat.current_provider = lambda: object()
self.ai_chat.take_last_gateway_trace = mock.Mock(side_effect=[{}, {
'task_id': 'old-trace', 'judge_mode': 'shadow', 'judge': {'risk': 'low'}}])
def finish_old_model(**kwargs):
self.assertNotEqual(state.get('model_task_id'), 'previous')
state.clear()
state.update(next_state)
return 'old reply' if gateway else {
'reply': 'old reply', 'judge_mode': 'shadow', 'judge': {'risk': 'low'}}
self.ai_chat.get_ai_reply = finish_old_model
self.router.answer = finish_old_model
bot._generate({'conv_id': 'memory-conv', 'text': 'old question',
'last_message': {'content': 'old question'}})
self.assertIs(bot._pending['memory-account/memory-conv'], state)
self.assertEqual(state, next_state)
if __name__ == '__main__':
unittest.main(verbosity=2)