331 lines
19 KiB
Python
331 lines
19 KiB
Python
"""Synthetic queue/paid-call/result-boundary authorization and rewash isolation."""
|
|
import asyncio
|
|
import json
|
|
from unittest import TestCase, mock
|
|
|
|
import rbac
|
|
import test_knowledge_tasks as fixtures
|
|
from archive_store import utc_now
|
|
from knowledge_access import KnowledgeProcessingDenied, OWN_CODE, TASKS_READ
|
|
from knowledge_worker import KnowledgeWorker, _processing_client_options
|
|
|
|
|
|
class WorkerScopeTest(TestCase):
|
|
tearDown = fixtures.KnowledgeTaskTest.tearDown
|
|
payload = fixtures.KnowledgeTaskTest.payload
|
|
start_job = fixtures.KnowledgeTaskTest.start_job
|
|
process = fixtures.KnowledgeTaskTest.process
|
|
first = fixtures.KnowledgeTaskTest.first
|
|
|
|
def setUp(self):
|
|
fixtures.KnowledgeTaskTest.setUp(self)
|
|
now = utc_now()
|
|
self.codes = {TASKS_READ, 'knowledge:read', 'knowledge:review:assign', 'knowledge:run',
|
|
'knowledge:run:control', 'knowledge:watch', 'im:content:read'}
|
|
with self.db.connect() as db:
|
|
db.execute('INSERT INTO roles VALUES (?,?,?,?,?)', ('worker-test', '合成清洗员', 0, now, now))
|
|
rbac.grant(db, 'worker-test', sorted(self.codes), now)
|
|
self.db.create_user('worker-test', 'InitialWorker123', 'worker-test', self.actor, '')
|
|
self.cleaner = self.db.authenticate('worker-test', 'InitialWorker123')['id']
|
|
self.accounts = []
|
|
for index in range(2):
|
|
_, account = self.db.create_desktop_session({'user_id': f'admin:{165 + index}', 'sn': f'WORKER{index}',
|
|
'nickname': f'合成账号{index}'}, f'worker-device-{index}', 'Synthetic QA', '1.0', '')
|
|
self.accounts.append(account)
|
|
self.tenant = self.accounts[0]['tenant_id']
|
|
self.bridge_env = mock.patch.dict('os.environ', {
|
|
'ZYT_GROUP_API_URL': 'https://group.synthetic.invalid/adminapi',
|
|
'ZYT_KNOWLEDGE_ACCOUNT_BRIDGE': json.dumps({'group_api_url': 'https://group.synthetic.invalid/adminapi',
|
|
'desktop_api_url': 'https://desktop.synthetic.invalid', 'namespace': 'admin'})})
|
|
self.bridge_env.start(); self.addCleanup(self.bridge_env.stop)
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE model_config SET config_json=? WHERE id=1", (json.dumps({'ZYT_LOGIN_API_URL': 'https://desktop.synthetic.invalid'}),))
|
|
db.execute('INSERT INTO group_sso_bindings VALUES (?,?,?,?,?,?,?,?,?)',
|
|
('zyt_group', 'gid_' + 'a' * 32, self.cleaner, 165, 'synthetic', 'Synthetic', 1, now, now))
|
|
self.assign(0)
|
|
self.archive.import_messages(self.payload(self.tenant), self.actor, '')
|
|
|
|
def assign(self, index):
|
|
with self.db.connect() as db:
|
|
db.execute('DELETE FROM admin_account_scope WHERE admin_user_id=?', (self.cleaner,))
|
|
db.execute('INSERT INTO admin_account_scope(admin_user_id,desktop_account_id,created_at) VALUES (?,?,?)',
|
|
(self.cleaner, self.accounts[index]['id'], utc_now()))
|
|
|
|
def grant(self, *, own=False, run=True, watch=True, active=True):
|
|
codes = self.codes | ({OWN_CODE} if own else set())
|
|
if not run:
|
|
codes -= {'knowledge:run'}
|
|
if not watch:
|
|
codes -= {'knowledge:watch'}
|
|
with self.db.connect() as db:
|
|
rbac.grant(db, 'worker-test', sorted(codes), utc_now())
|
|
db.execute('UPDATE users SET active=? WHERE id=?', (int(active), self.cleaner))
|
|
|
|
def job(self, *, model=True, auto_watch=False):
|
|
with self.db.connect() as db:
|
|
source = db.execute('SELECT id FROM archive_source_account WHERE tenant_id=?', (self.tenant,)).fetchone()[0]
|
|
return self.store.create_job(self.tenant, {'source_account_id': source, 'engine': 'model' if model else 'rules',
|
|
'model_provider_id': 'alpha' if model else '', 'model_concurrency': 1, 'max_model_calls': 100,
|
|
'staff_confirmed': True, 'auto_watch': auto_watch}, self.cleaner, '')
|
|
|
|
def raw_item(self, item_id):
|
|
with self.db.connect() as db:
|
|
return dict(db.execute('SELECT * FROM knowledge_item WHERE id=?', (item_id,)).fetchone())
|
|
|
|
def bob_draft(self):
|
|
self.process(self.tenant)
|
|
return self.first(self.tenant)
|
|
|
|
def test_queued_work_continues_after_task_page_revoke(self):
|
|
job = self.job(model=False, auto_watch=True)
|
|
with self.db.connect() as db:
|
|
rbac.grant(db, 'worker-test', sorted(self.codes - {TASKS_READ}), utc_now())
|
|
worker = KnowledgeWorker(self.store)
|
|
self.assertTrue(worker.step())
|
|
self.assertEqual(self.store.job(self.tenant, job['id'])['status'], 'completed')
|
|
changed = self.payload(self.tenant)
|
|
changed['messages'][1]['content'] = '继续处理自动整理的新消息。'
|
|
self.archive.import_messages(changed, self.actor, '')
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE knowledge_dirty SET changed_at='2000-01-01'")
|
|
self.assertTrue(worker.step())
|
|
self.assertTrue(worker.step())
|
|
with self.db.connect() as db:
|
|
auto = db.execute("SELECT status FROM knowledge_job WHERE run_id='auto'").fetchone()
|
|
self.assertEqual(auto['status'], 'completed')
|
|
|
|
def test_rewash_same_tenant_allows_admin_other_and_unknown_creator(self):
|
|
original = self.bob_draft()
|
|
self.grant(own=True)
|
|
for owner in ('other', 'unknown', 'self'):
|
|
with self.subTest(owner=owner):
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE knowledge_item SET job_id=? WHERE id=?',
|
|
(None if owner == 'unknown' else original['job_id'], original['id']))
|
|
db.execute('UPDATE knowledge_job SET created_by=? WHERE id=?',
|
|
(self.cleaner if owner == 'self' else self.actor, original['job_id']))
|
|
before = self.raw_item(original['id'])
|
|
job = self.job()
|
|
def wash(candidate, *_args, **_kwargs):
|
|
return {**candidate, 'answer': '清洗后的虚构答案-' + owner, 'conditions': '仅适用于合成测试'}, 10, 20
|
|
with mock.patch('knowledge_worker.enhance', side_effect=wash):
|
|
KnowledgeWorker(self.store).step()
|
|
after = self.raw_item(original['id'])
|
|
final = self.store.job(self.tenant, job['id'])
|
|
self.assertEqual(final['status'], 'completed', final)
|
|
self.assertEqual(after['revision'], before['revision'] + 1)
|
|
self.assertEqual((after['answer'], after['job_id']), ('清洗后的虚构答案-' + owner, before['job_id']))
|
|
|
|
def test_model_results_fail_closed_after_run_tenant_or_active_revocation(self):
|
|
for changed in ('run', 'tenant', 'active'):
|
|
with self.subTest(changed=changed):
|
|
self.grant(); self.assign(0)
|
|
job = self.job()
|
|
def revoke(candidate, *_args, **_kwargs):
|
|
if changed == 'run':
|
|
self.grant(run=False)
|
|
elif changed == 'tenant':
|
|
self.assign(1)
|
|
else:
|
|
self.grant(active=False)
|
|
return {**candidate, 'conditions': '合成条件'}, 10, 20
|
|
with mock.patch('knowledge_worker.enhance', side_effect=revoke) as call:
|
|
KnowledgeWorker(self.store).step()
|
|
self.assertEqual(call.call_count, 1)
|
|
final = self.store.job(self.tenant, job['id'])
|
|
self.assertEqual((final['status'], final['pending_items'], final['created_items']), ('failed', 1, 0))
|
|
self.assertIn('进度已保留', final['error'])
|
|
with self.db.connect() as db:
|
|
self.assertEqual(db.execute('SELECT lease_owner FROM knowledge_job WHERE id=?', (job['id'],)).fetchone()[0], '')
|
|
self.assertEqual(self.store.list_items(self.tenant)['total'], 0)
|
|
self.grant(); self.assign(0)
|
|
|
|
def test_new_own_constraint_during_model_wait_allows_admin_draft_in_same_account(self):
|
|
original = self.bob_draft()
|
|
before = self.raw_item(original['id'])
|
|
job = self.job()
|
|
def grant_own(candidate, *_args, **_kwargs):
|
|
self.grant(own=True)
|
|
return {**candidate, 'answer': '试图覆盖其他人的答案', 'conditions': '虚构条件'}, 10, 20
|
|
with mock.patch('knowledge_worker.enhance', side_effect=grant_own):
|
|
KnowledgeWorker(self.store).step()
|
|
self.assertEqual(self.raw_item(original['id'])['revision'], before['revision'] + 1)
|
|
self.assertEqual(self.raw_item(original['id'])['job_id'], before['job_id'])
|
|
self.assertEqual(self.store.job(self.tenant, job['id'])['status'], 'completed')
|
|
self.assertEqual(self.store.job(self.tenant, job['id'])['updated_items'], 1)
|
|
|
|
def test_paid_thread_rule_commit_and_unknown_creator_are_revalidated(self):
|
|
for mode in ('before_paid', 'before_rules', 'unknown_creator'):
|
|
with self.subTest(mode=mode):
|
|
self.grant(); self.assign(0)
|
|
job = self.job(model=mode != 'before_rules')
|
|
worker = KnowledgeWorker(self.store)
|
|
if mode == 'before_paid':
|
|
original = worker._call
|
|
def before_paid(*args):
|
|
self.grant(run=False)
|
|
return original(*args)
|
|
patcher = mock.patch.object(worker, '_call', side_effect=before_paid)
|
|
elif mode == 'before_rules':
|
|
original = worker.save_rules
|
|
def before_rules(*args):
|
|
self.grant(run=False)
|
|
return original(*args)
|
|
patcher = mock.patch.object(worker, 'save_rules', side_effect=before_rules)
|
|
else:
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE knowledge_job SET created_by=0 WHERE id=?', (job['id'],))
|
|
patcher = mock.patch.object(worker, 'owner', worker.owner)
|
|
with patcher, mock.patch('knowledge_worker.enhance_batch') as paid:
|
|
worker.step()
|
|
paid.assert_not_called()
|
|
final = self.store.job(self.tenant, job['id'])
|
|
self.assertEqual((final['status'], final['created_items']), ('failed', 0))
|
|
self.assertIn('进度已保留', final['error'])
|
|
if mode != 'unknown_creator':
|
|
self.assertEqual(final['pending_items'], 1)
|
|
self.assertEqual(self.store.list_items(self.tenant)['total'], 0)
|
|
self.grant()
|
|
job = self.job()
|
|
hook = _processing_client_options(self.db, self.tenant, job['id'])['event_hooks']['request'][0]
|
|
asyncio.run(hook(None))
|
|
self.grant(run=False)
|
|
with self.assertRaises(KnowledgeProcessingDenied):
|
|
asyncio.run(hook(None))
|
|
self.assertEqual(_processing_client_options(self.db, self.tenant, ''), {})
|
|
self.grant()
|
|
|
|
def test_automatic_watch_without_run_still_works_then_stops_on_watch_revoke(self):
|
|
self.job(model=False, auto_watch=True)
|
|
worker = KnowledgeWorker(self.store)
|
|
while worker.step():
|
|
pass
|
|
self.grant(own=True, run=False)
|
|
changed = self.payload(self.tenant)
|
|
changed['messages'][1]['content'] = '自动整理的合成新答案,请携带既往记录。'
|
|
self.archive.import_messages(changed, self.actor, '')
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE knowledge_dirty SET changed_at='2000-01-01'")
|
|
self.assertTrue(worker.step())
|
|
self.assertTrue(worker.step())
|
|
with self.db.connect() as db:
|
|
auto = db.execute("SELECT * FROM knowledge_job WHERE run_id='auto'").fetchone()
|
|
self.assertEqual((auto['status'], auto['created_by']), ('completed', self.cleaner))
|
|
self.assertEqual(self.store.list_items(self.tenant, actor=self.cleaner)['total'], 2)
|
|
changed['messages'][1]['content'] = '自动整理的另一条合成答案。'
|
|
self.archive.import_messages(changed, self.actor, '')
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE knowledge_dirty SET changed_at='2000-01-01'")
|
|
self.grant(own=True, run=False, watch=False)
|
|
self.assertFalse(worker.step())
|
|
self.grant(own=True, run=False)
|
|
self.assertTrue(worker.step())
|
|
self.grant(own=True, run=False, watch=False)
|
|
self.assertTrue(worker.step())
|
|
with self.db.connect() as db:
|
|
self.assertEqual(db.execute("SELECT COUNT(*) FROM knowledge_job WHERE run_id='auto' AND status='failed'").fetchone()[0], 1)
|
|
|
|
def test_denied_dirty_entries_do_not_starve_later_authorized_watch(self):
|
|
other_tenant = self.accounts[1]['tenant_id']
|
|
self.archive.import_messages(self.payload(other_tenant), self.actor, '')
|
|
with self.db.connect() as db:
|
|
for tenant, creator in ((self.tenant, self.cleaner), (other_tenant, self.actor)):
|
|
source, conversation = db.execute('SELECT source_account_id,conversation_id FROM archive_message '
|
|
'WHERE tenant_id=? LIMIT 1', (tenant,)).fetchone()
|
|
db.execute('INSERT INTO knowledge_watch VALUES (?,?,1,?)', (tenant, source, creator))
|
|
db.execute('INSERT INTO knowledge_dirty VALUES (?,?,?,?)',
|
|
(tenant, source, conversation, '2000-01-01' if tenant == self.tenant else '2002-01-01'))
|
|
if tenant == self.tenant:
|
|
db.executemany('INSERT INTO knowledge_dirty VALUES (?,?,?,?)',
|
|
[(tenant, source, f'blocked-{index}', '2001-01-01') for index in range(100)])
|
|
before = [tuple(row) for row in db.execute('SELECT * FROM knowledge_dirty WHERE tenant_id=? '
|
|
'ORDER BY conversation_id', (self.tenant,))]
|
|
self.grant(watch=False)
|
|
worker = KnowledgeWorker(self.store)
|
|
self.assertTrue(worker.enqueue_incremental())
|
|
with self.db.connect() as db:
|
|
self.assertEqual([(row['tenant_id'], row['created_by']) for row in db.execute(
|
|
"SELECT * FROM knowledge_job WHERE run_id='auto'")], [(other_tenant, self.actor)])
|
|
self.assertEqual([tuple(row) for row in db.execute('SELECT * FROM knowledge_dirty WHERE tenant_id=? '
|
|
'ORDER BY conversation_id', (self.tenant,))], before)
|
|
self.assertEqual(len(before), 101)
|
|
self.grant(own=True, run=False)
|
|
self.assertTrue(worker.enqueue_incremental())
|
|
with self.db.connect() as db:
|
|
self.assertEqual(db.execute("SELECT COUNT(*) FROM knowledge_job WHERE run_id='auto' AND tenant_id=? "
|
|
'AND created_by=?', (self.tenant, self.cleaner)).fetchone()[0], 1)
|
|
self.assertEqual(db.execute('SELECT COUNT(*) FROM knowledge_dirty WHERE tenant_id=?',
|
|
(self.tenant,)).fetchone()[0], 100)
|
|
|
|
def test_auto_cleanup_retains_referenced_owner_job_and_deletes_unreferenced(self):
|
|
job = self.job(model=False)
|
|
worker = KnowledgeWorker(self.store)
|
|
worker.step()
|
|
self.grant(own=True)
|
|
self.assertEqual(self.store.list_items(self.tenant, actor=self.cleaner)['total'], 1)
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE knowledge_job SET run_id='auto',updated_at='2000-01-01' WHERE id=?", (job['id'],))
|
|
db.execute('INSERT INTO knowledge_job(id,tenant_id,status,options_json,cutoff_at,created_by,created_at,updated_at,run_id) '
|
|
"SELECT 'old-unreferenced',tenant_id,'completed',options_json,cutoff_at,created_by,created_at,'2000-01-01','auto' "
|
|
'FROM knowledge_job WHERE id=?', (job['id'],))
|
|
self.assertFalse(worker.enqueue_incremental())
|
|
with self.db.connect() as db:
|
|
self.assertIsNotNone(db.execute('SELECT 1 FROM knowledge_job WHERE id=?', (job['id'],)).fetchone())
|
|
self.assertIsNone(db.execute("SELECT 1 FROM knowledge_job WHERE id='old-unreferenced'").fetchone())
|
|
self.assertEqual(self.store.list_items(self.tenant, actor=self.cleaner)['total'], 1)
|
|
|
|
def test_binding_and_bridge_rechecked_before_paid_call_and_before_commit(self):
|
|
for changed in ('binding-before-paid', 'binding-after-model', 'bridge-after-model'):
|
|
with self.subTest(changed=changed):
|
|
self.grant(own=True)
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE group_sso_bindings SET source_admin_id=165 WHERE user_id=?', (self.cleaner,))
|
|
job = self.job()
|
|
worker = KnowledgeWorker(self.store)
|
|
def revoke():
|
|
if changed.startswith('binding'):
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE group_sso_bindings SET source_admin_id=166 WHERE user_id=?', (self.cleaner,))
|
|
else:
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE model_config SET config_json=? WHERE id=1',
|
|
(json.dumps({'ZYT_LOGIN_API_URL': 'https://unregistered.invalid'}),))
|
|
if changed == 'binding-before-paid':
|
|
original = worker._call
|
|
def before_paid(*args):
|
|
revoke()
|
|
return original(*args)
|
|
with mock.patch.object(worker, '_call', side_effect=before_paid), mock.patch('knowledge_worker.enhance_batch') as paid:
|
|
worker.step()
|
|
paid.assert_not_called()
|
|
else:
|
|
def after_model(candidate, *_args, **_kwargs):
|
|
revoke()
|
|
return {**candidate, 'conditions': '合成条件'}, 10, 20
|
|
with mock.patch('knowledge_worker.enhance', side_effect=after_model):
|
|
worker.step()
|
|
final = self.store.job(self.tenant, job['id'])
|
|
self.assertEqual((final['status'], final['pending_items'], final['created_items']), ('failed', 1, 0))
|
|
self.assertIn('进度已保留', final['error'])
|
|
self.assertEqual(self.store.list_items(self.tenant)['total'], 0)
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE model_config SET config_json=? WHERE id=1',
|
|
(json.dumps({'ZYT_LOGIN_API_URL': 'https://desktop.synthetic.invalid'}),))
|
|
with self.db.connect() as db:
|
|
db.execute('UPDATE group_sso_bindings SET source_admin_id=165 WHERE user_id=?', (self.cleaner,))
|
|
|
|
def test_each_http_request_hook_rechecks_binding_and_active_desktop(self):
|
|
self.grant(own=True)
|
|
job = self.job()
|
|
hook = _processing_client_options(self.db, self.tenant, job['id'])['event_hooks']['request'][0]
|
|
asyncio.run(hook(None))
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE desktop_accounts SET status='disabled' WHERE id=?", (self.accounts[0]['id'],))
|
|
with self.assertRaises(KnowledgeProcessingDenied):
|
|
asyncio.run(hook(None))
|
|
with self.db.connect() as db:
|
|
db.execute("UPDATE desktop_accounts SET status='active' WHERE id=?", (self.accounts[0]['id'],))
|
|
db.execute('DELETE FROM group_sso_bindings WHERE user_id=?', (self.cleaner,))
|
|
with self.assertRaises(KnowledgeProcessingDenied):
|
|
asyncio.run(hook(None))
|