155 lines
9.6 KiB
Python
155 lines
9.6 KiB
Python
"""Model output tolerance must preserve evidence and the human review gate."""
|
|
import copy
|
|
import json
|
|
from unittest import TestCase, mock
|
|
|
|
import test_knowledge_tasks as fixtures
|
|
from knowledge_model_output import ModelOutputError, normalize_output, original_draft
|
|
from knowledge_worker import KnowledgeWorker
|
|
|
|
|
|
def candidate():
|
|
return {'title': '原始测试问题', 'question': '虚构门店预约需要提前多久？',
|
|
'answer': '测试门店请提前一天预约，具体以当天安排为准。',
|
|
'conditions': '', 'category': '待分类', 'kind': 'qa', 'flags': ['来源需核对'],
|
|
'sources': [{'message_id': 'synthetic-1', 'content': '完整原始测试证据'}]}
|
|
|
|
|
|
def fields(**overrides):
|
|
return {'title': '测试预约', 'question': '测试门店何时预约？',
|
|
'answer': '测试门店应提前一天预约，以当天安排为准。',
|
|
'conditions': '仅限虚构测试门店', 'category': '预约咨询', 'kind': 'qa', **overrides}
|
|
|
|
|
|
class ModelOutputTest(TestCase):
|
|
def test_missing_auxiliary_fields_are_safe_and_conditions_remain_empty(self):
|
|
source = candidate(); before = copy.deepcopy(source)
|
|
for values in ({'question': '测试问题？', 'answer': '仅供虚构门店验收的测试答案。'},
|
|
fields(title=None, conditions=None, category=None, kind=None),
|
|
fields(title=[], conditions={}, category=2, kind=[])):
|
|
with self.subTest(values=values):
|
|
result, _, _ = normalize_output(source, json.dumps(values), 100)
|
|
self.assertTrue(result['title'])
|
|
self.assertEqual(result['conditions'], '')
|
|
self.assertEqual(result['category'], '待分类')
|
|
self.assertEqual(result['kind'], 'qa')
|
|
self.assertEqual(result['sources'], source['sources'])
|
|
self.assertEqual(source, before, 'Model processing must not corrupt pending evidence')
|
|
|
|
def test_auxiliary_lengths_and_aliases_do_not_truncate_core_facts(self):
|
|
data = fields(title='题' * 200, conditions='条件' * 1100, category='类' * 120, kind='标准问答')
|
|
result, _, _ = normalize_output(candidate(), json.dumps(data), 10)
|
|
self.assertEqual(len(result['title']), 160)
|
|
self.assertEqual(result['question'], '测试门店何时预约?')
|
|
self.assertEqual(result['answer'], '测试门店应提前一天预约,以当天安排为准。')
|
|
self.assertEqual(result['conditions'], '')
|
|
self.assertEqual(result['category'], '待分类')
|
|
self.assertEqual(result['kind'], 'qa')
|
|
self.assertTrue(any('适用条件' in flag for flag in result['flags']))
|
|
|
|
def test_bad_core_fields_raise_specific_safe_errors_and_leave_source_untouched(self):
|
|
source = candidate(); before = copy.deepcopy(source)
|
|
for key, value, word in [('question', '', '问题'), ('question', None, '问题'),
|
|
('answer', ['不应拼接'], '答案'), ('answer', {}, '答案'),
|
|
('answer', '字' * 8001, '8000'), ('question', '字' * 4001, '4000'),
|
|
('answer', '\u337f' * 3000, '8000')]:
|
|
raw = json.dumps(fields(**{key: value}), ensure_ascii=False)
|
|
with self.subTest(key=key, value_type=type(value).__name__):
|
|
with self.assertRaisesRegex(ModelOutputError, word) as raised:
|
|
normalize_output(source, raw, 123)
|
|
self.assertEqual((raised.exception.input_chars, raised.exception.output_chars), (123, len(raw)))
|
|
self.assertEqual(source, before)
|
|
|
|
def test_invalid_json_and_empty_results_do_not_retain_raw_response(self):
|
|
for raw in ('{"private-response":', '["private-response"]', '', None):
|
|
with self.assertRaises(ModelOutputError) as raised:
|
|
normalize_output(candidate(), raw, 75)
|
|
self.assertNotIn('private-response', str(raised.exception))
|
|
self.assertEqual(raised.exception.input_chars, 75)
|
|
raw = '\ufeff```JSON\n' + json.dumps(fields()) + '\n```'
|
|
result, _, count = normalize_output(candidate(), raw, 60)
|
|
self.assertEqual(result['answer'], '测试门店应提前一天预约,以当天安排为准。')
|
|
self.assertEqual(count, len(raw))
|
|
|
|
def test_fallback_retains_complete_original_even_when_long(self):
|
|
source = candidate(); source['answer'] = '原始业务限制。' * 1500
|
|
result = original_draft(source, ModelOutputError('模型答案字段为空'))
|
|
self.assertEqual(result['answer'], source['answer'])
|
|
self.assertEqual(result['sources'], source['sources'])
|
|
self.assertEqual(result['conditions'], '')
|
|
self.assertIn('不能直接发布', result['flags'][-1])
|
|
self.assertEqual(len(source['flags']), 1)
|
|
|
|
|
|
class ModelOutputWorkerTest(TestCase):
|
|
setUp = fixtures.KnowledgeTaskTest.setUp
|
|
tearDown = fixtures.KnowledgeTaskTest.tearDown
|
|
payload = fixtures.KnowledgeTaskTest.payload
|
|
start_job = fixtures.KnowledgeTaskTest.start_job
|
|
|
|
def seed(self):
|
|
self.archive.import_messages(self.payload(messages=[
|
|
('inbound', '虚构门店需要提前多久预约？'), ('outbound', '测试门店请提前一天预约，以当天确认结果为准。'),
|
|
('inbound', '测试门店周末的营业时间是几点？'), ('outbound', '测试门店周末上午九点至下午五点营业。')]), self.actor, '')
|
|
return self.start_job(engine='model', model_provider_id='alpha')
|
|
|
|
def test_invalid_response_is_quarantined_as_draft_and_next_pair_continues(self):
|
|
job = self.seed()
|
|
first_reply = json.dumps(fields(answer=None))
|
|
second_reply = json.dumps(fields(title='正常第二条', question='测试门店周末几点营业？'))
|
|
with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call:
|
|
call.side_effect = [{'text': first_reply, 'error': ''}, {'text': second_reply, 'error': ''}]
|
|
KnowledgeWorker(self.store).step()
|
|
result = self.store.job('default', job['id'])
|
|
self.assertEqual((result['status'], result['created_items'], result['pending_items'], result['model_calls']), ('completed', 2, 0, 2))
|
|
self.assertEqual(result['options']['model_format_fallbacks'], 1)
|
|
self.assertIn('答案', result['options']['model_last_warning'])
|
|
self.assertEqual(result['output_chars'], len(first_reply) + len(second_reply))
|
|
items = self.store.list_items('default')['items']
|
|
original = next(item for item in items if any('模型格式异常' in flag for flag in item['flags']))
|
|
self.assertEqual(original['question'], '虚构门店需要提前多久预约?')
|
|
self.assertEqual(original['answer'], '测试门店请提前一天预约,以当天确认结果为准。')
|
|
self.assertEqual(original['conditions'], '')
|
|
with self.assertRaises(ValueError):
|
|
self.store.transition('default', original['id'], original['revision'], 'approve', self.actor, '', True)
|
|
self.assertEqual(len(self.store.detail('default', original['id'])['sources']), 2)
|
|
|
|
def test_fallback_commit_survives_later_transport_failure_and_resume(self):
|
|
job = self.seed()
|
|
with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call:
|
|
call.side_effect = [{'text': 'incomplete JSON', 'error': ''}, {'text': '', 'error': '上游返回 429: private-provider-body'}]
|
|
KnowledgeWorker(self.store).step()
|
|
failed = self.store.job('default', job['id'])
|
|
self.assertEqual((failed['status'], failed['created_items'], failed['pending_items']), ('failed', 1, 1))
|
|
self.assertEqual(failed['options']['model_format_fallbacks'], 1)
|
|
self.assertIn('429', failed['error']); self.assertNotIn('private-provider-body', failed['error'])
|
|
self.store.job_action('default', job['id'], 'resume', self.actor, '')
|
|
with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call:
|
|
call.return_value = {'text': json.dumps(fields()), 'error': ''}
|
|
KnowledgeWorker(self.store).step()
|
|
self.assertEqual(call.call_count, 1)
|
|
result = self.store.job('default', job['id'])
|
|
self.assertEqual((result['status'], result['created_items'], result['model_calls']), ('completed', 2, 3))
|
|
self.assertEqual(result['options']['model_format_fallbacks'], 1)
|
|
|
|
def test_cancel_during_invalid_response_does_not_save_fallback(self):
|
|
job = self.seed()
|
|
async def cancel(*args, **kwargs):
|
|
self.store.job_action('default', job['id'], 'cancel', self.actor, '')
|
|
return {'text': '{}', 'error': ''}
|
|
with mock.patch('model_gateway.call_outlet', side_effect=cancel):
|
|
KnowledgeWorker(self.store).step()
|
|
result = self.store.job('default', job['id'])
|
|
self.assertEqual((result['status'], result['created_items'], result['pending_items']), ('cancelled', 0, 0))
|
|
self.assertEqual(result['options'].get('model_format_fallbacks', 0), 0)
|
|
|
|
def test_normalized_empty_conditions_cannot_be_approved_without_editing(self):
|
|
job = self.seed()
|
|
with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call:
|
|
call.return_value = {'text': json.dumps(fields(conditions=None)), 'error': ''}
|
|
KnowledgeWorker(self.store).step()
|
|
self.assertEqual(self.store.job('default', job['id'])['status'], 'completed')
|
|
for item in self.store.list_items('default')['items']:
|
|
with self.assertRaises(ValueError):
|
|
self.store.transition('default', item['id'], item['revision'], 'approve', self.actor, '', True)
|