"""Model output tolerance must preserve evidence and the human review gate.""" import copy import json from unittest import TestCase, mock import test_knowledge_tasks as fixtures from knowledge_model_output import ModelOutputError, normalize_output, original_draft from knowledge_worker import KnowledgeWorker def candidate(): return {'title': '原始测试问题', 'question': '虚构门店预约需要提前多久？', 'answer': '测试门店请提前一天预约，具体以当天安排为准。', 'conditions': '', 'category': '待分类', 'kind': 'qa', 'flags': ['来源需核对'], 'sources': [{'message_id': 'synthetic-1', 'content': '完整原始测试证据'}]} def fields(**overrides): return {'title': '测试预约', 'question': '测试门店何时预约？', 'answer': '测试门店应提前一天预约，以当天安排为准。', 'conditions': '仅限虚构测试门店', 'category': '预约咨询', 'kind': 'qa', **overrides} class ModelOutputTest(TestCase): def test_missing_auxiliary_fields_are_safe_and_conditions_remain_empty(self): source = candidate(); before = copy.deepcopy(source) for values in ({'question': '测试问题？', 'answer': '仅供虚构门店验收的测试答案。'}, fields(title=None, conditions=None, category=None, kind=None), fields(title=[], conditions={}, category=2, kind=[])): with self.subTest(values=values): result, _, _ = normalize_output(source, json.dumps(values), 100) self.assertTrue(result['title']) self.assertEqual(result['conditions'], '') self.assertEqual(result['category'], '待分类') self.assertEqual(result['kind'], 'qa') self.assertEqual(result['sources'], source['sources']) self.assertEqual(source, before, 'Model processing must not corrupt pending evidence') def test_auxiliary_lengths_and_aliases_do_not_truncate_core_facts(self): data = fields(title='题' * 200, conditions='条件' * 1100, category='类' * 120, kind='标准问答') result, _, _ = normalize_output(candidate(), json.dumps(data), 10) self.assertEqual(len(result['title']), 160) self.assertEqual(result['question'], '测试门店何时预约?') self.assertEqual(result['answer'], '测试门店应提前一天预约,以当天安排为准。') self.assertEqual(result['conditions'], '') self.assertEqual(result['category'], '待分类') self.assertEqual(result['kind'], 'qa') self.assertTrue(any('适用条件' in flag for flag in result['flags'])) def test_bad_core_fields_raise_specific_safe_errors_and_leave_source_untouched(self): source = candidate(); before = copy.deepcopy(source) for key, value, word in [('question', '', '问题'), ('question', None, '问题'), ('answer', ['不应拼接'], '答案'), ('answer', {}, '答案'), ('answer', '字' * 8001, '8000'), ('question', '字' * 4001, '4000'), ('answer', '\u337f' * 3000, '8000')]: raw = json.dumps(fields(**{key: value}), ensure_ascii=False) with self.subTest(key=key, value_type=type(value).__name__): with self.assertRaisesRegex(ModelOutputError, word) as raised: normalize_output(source, raw, 123) self.assertEqual((raised.exception.input_chars, raised.exception.output_chars), (123, len(raw))) self.assertEqual(source, before) def test_invalid_json_and_empty_results_do_not_retain_raw_response(self): for raw in ('{"private-response":', '["private-response"]', '', None): with self.assertRaises(ModelOutputError) as raised: normalize_output(candidate(), raw, 75) self.assertNotIn('private-response', str(raised.exception)) self.assertEqual(raised.exception.input_chars, 75) raw = '\ufeff```JSON\n' + json.dumps(fields()) + '\n```' result, _, count = normalize_output(candidate(), raw, 60) self.assertEqual(result['answer'], '测试门店应提前一天预约,以当天安排为准。') self.assertEqual(count, len(raw)) def test_fallback_retains_complete_original_even_when_long(self): source = candidate(); source['answer'] = '原始业务限制。' * 1500 result = original_draft(source, ModelOutputError('模型答案字段为空')) self.assertEqual(result['answer'], source['answer']) self.assertEqual(result['sources'], source['sources']) self.assertEqual(result['conditions'], '') self.assertIn('不能直接发布', result['flags'][-1]) self.assertEqual(len(source['flags']), 1) class ModelOutputWorkerTest(TestCase): setUp = fixtures.KnowledgeTaskTest.setUp tearDown = fixtures.KnowledgeTaskTest.tearDown payload = fixtures.KnowledgeTaskTest.payload start_job = fixtures.KnowledgeTaskTest.start_job def seed(self): self.archive.import_messages(self.payload(messages=[ ('inbound', '虚构门店需要提前多久预约？'), ('outbound', '测试门店请提前一天预约，以当天确认结果为准。'), ('inbound', '测试门店周末的营业时间是几点？'), ('outbound', '测试门店周末上午九点至下午五点营业。')]), self.actor, '') return self.start_job(engine='model', model_provider_id='alpha') def test_invalid_response_is_quarantined_as_draft_and_next_pair_continues(self): job = self.seed() first_reply = json.dumps(fields(answer=None)) second_reply = json.dumps(fields(title='正常第二条', question='测试门店周末几点营业？')) with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call: call.side_effect = [{'text': first_reply, 'error': ''}, {'text': second_reply, 'error': ''}] KnowledgeWorker(self.store).step() result = self.store.job('default', job['id']) self.assertEqual((result['status'], result['created_items'], result['pending_items'], result['model_calls']), ('completed', 2, 0, 2)) self.assertEqual(result['options']['model_format_fallbacks'], 1) self.assertIn('答案', result['options']['model_last_warning']) self.assertEqual(result['output_chars'], len(first_reply) + len(second_reply)) items = self.store.list_items('default')['items'] original = next(item for item in items if any('模型格式异常' in flag for flag in item['flags'])) self.assertEqual(original['question'], '虚构门店需要提前多久预约?') self.assertEqual(original['answer'], '测试门店请提前一天预约,以当天确认结果为准。') self.assertEqual(original['conditions'], '') with self.assertRaises(ValueError): self.store.transition('default', original['id'], original['revision'], 'approve', self.actor, '', True) self.assertEqual(len(self.store.detail('default', original['id'])['sources']), 2) def test_fallback_commit_survives_later_transport_failure_and_resume(self): job = self.seed() with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call: call.side_effect = [{'text': 'incomplete JSON', 'error': ''}, {'text': '', 'error': '上游返回 429: private-provider-body'}] KnowledgeWorker(self.store).step() failed = self.store.job('default', job['id']) self.assertEqual((failed['status'], failed['created_items'], failed['pending_items']), ('failed', 1, 1)) self.assertEqual(failed['options']['model_format_fallbacks'], 1) self.assertIn('429', failed['error']); self.assertNotIn('private-provider-body', failed['error']) self.store.job_action('default', job['id'], 'resume', self.actor, '') with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call: call.return_value = {'text': json.dumps(fields()), 'error': ''} KnowledgeWorker(self.store).step() self.assertEqual(call.call_count, 1) result = self.store.job('default', job['id']) self.assertEqual((result['status'], result['created_items'], result['model_calls']), ('completed', 2, 3)) self.assertEqual(result['options']['model_format_fallbacks'], 1) def test_cancel_during_invalid_response_does_not_save_fallback(self): job = self.seed() async def cancel(*args, **kwargs): self.store.job_action('default', job['id'], 'cancel', self.actor, '') return {'text': '{}', 'error': ''} with mock.patch('model_gateway.call_outlet', side_effect=cancel): KnowledgeWorker(self.store).step() result = self.store.job('default', job['id']) self.assertEqual((result['status'], result['created_items'], result['pending_items']), ('cancelled', 0, 0)) self.assertEqual(result['options'].get('model_format_fallbacks', 0), 0) def test_normalized_empty_conditions_cannot_be_approved_without_editing(self): job = self.seed() with mock.patch('model_gateway.call_outlet', new_callable=mock.AsyncMock) as call: call.return_value = {'text': json.dumps(fields(conditions=None)), 'error': ''} KnowledgeWorker(self.store).step() self.assertEqual(self.store.job('default', job['id'])['status'], 'completed') for item in self.store.list_items('default')['items']: with self.assertRaises(ValueError): self.store.transition('default', item['id'], item['revision'], 'approve', self.actor, '', True)