252 lines
14 KiB
Python
252 lines
14 KiB
Python
"""Read-only browser cache, search and pagination regressions on synthetic SQLite."""
|
|
import concurrent.futures
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import test_wxwork_message_browser as fixtures
|
|
import wxwork_message_browser as browser
|
|
|
|
|
|
class BrowserPerformanceTest(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(prefix='browser-cache-test-')
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.account = fixtures.WeChatMessageBrowserTests()._database(self.root, '100')
|
|
self.database = self.account / 'message.db'
|
|
browser.clear_browser_cache()
|
|
self.addCleanup(browser.clear_browser_cache)
|
|
patcher = mock.patch.object(browser, '_refresh_cache', side_effect=AssertionError('No decryption during navigation'))
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
|
|
def snapshot(self, **kwargs):
|
|
return browser.load_browser_snapshot(cache_roots=[self.root], selected_account='100', **kwargs)
|
|
|
|
def update(self, sql, args=(), path=None):
|
|
db = sqlite3.connect(path or self.database)
|
|
try:
|
|
db.execute(sql, args)
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
def test_warm_conversation_navigation_does_not_repeat_full_aggregates(self):
|
|
self.snapshot()
|
|
queries = []
|
|
original = browser.connect_sqlite
|
|
def connected(*args, **kwargs):
|
|
db = original(*args, **kwargs)
|
|
db.set_trace_callback(queries.append)
|
|
return db
|
|
with mock.patch.object(browser, 'connect_sqlite', side_effect=connected):
|
|
state = self.snapshot(selected_conversation='M:200')
|
|
self.snapshot(selected_conversation='S:100_300')
|
|
self.assertEqual(len(state['messages']), 2)
|
|
self.assertFalse(any('COUNT(' in query.upper() or 'GROUP BY' in query.upper() for query in queries))
|
|
self.assertTrue(all('WHERE rowid IN' in query for query in queries))
|
|
|
|
def test_cached_message_page_makes_no_sqlite_queries(self):
|
|
self.snapshot(selected_conversation='M:200')
|
|
with mock.patch.object(browser, 'connect_sqlite', side_effect=AssertionError('Cache hit must not open SQLite')):
|
|
self.assertEqual(len(self.snapshot(selected_conversation='M:200')['messages']), 2)
|
|
|
|
def test_new_message_invalidates_counts_order_and_cached_page(self):
|
|
self.snapshot(selected_conversation='M:200')
|
|
self.update('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('200', 'M:200', 2, 200, 'new synthetic', 's-new', ''))
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
self.assertEqual(result['messageCount'], 4)
|
|
self.assertEqual(result['conversations'][0]['id'], 'M:200')
|
|
self.assertEqual(result['messages'][-1]['id'], 's-new')
|
|
|
|
def test_wal_only_commit_invalidates_without_main_db_change(self):
|
|
writer = sqlite3.connect(self.database)
|
|
self.addCleanup(writer.close)
|
|
writer.execute('PRAGMA journal_mode=WAL')
|
|
writer.execute('PRAGMA wal_autocheckpoint=0')
|
|
writer.execute('PRAGMA wal_checkpoint(TRUNCATE)')
|
|
before = self.database.stat()
|
|
self.snapshot(selected_conversation='M:200')
|
|
writer.execute('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('200', 'M:200', 2, 300, 'wal synthetic', 's-wal', ''))
|
|
writer.commit()
|
|
after = self.database.stat()
|
|
self.assertEqual((before.st_size, before.st_mtime_ns), (after.st_size, after.st_mtime_ns))
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
self.assertEqual(result['messageCount'], 4)
|
|
self.assertEqual(result['messages'][-1]['id'], 's-wal')
|
|
|
|
def test_rename_invalidates_names_without_touching_messages(self):
|
|
self.snapshot(selected_conversation='M:200')
|
|
self.update('UPDATE user_table SET name=? WHERE id=?', ('新的合成昵称', '200'), self.account / 'user.db')
|
|
self.update('UPDATE conversation_table SET name=? WHERE id=?', ('新的会话名称', 'M:200'), self.account / 'session.db')
|
|
result = self.snapshot(selected_conversation='M:200', query='新的会话名称')
|
|
self.assertEqual(result['conversations'][0]['name'], '新的会话名称')
|
|
self.assertEqual(result['messages'][0]['sender'], '新的合成昵称')
|
|
|
|
def test_search_finds_conversations_older_than_the_newest_thousand(self):
|
|
db = sqlite3.connect(self.database)
|
|
try:
|
|
db.executemany('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)',
|
|
(('other', f'M:new-{i}', 2, 1000 + i, 'synthetic', f'n-{i}', '') for i in range(1200)))
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
result = self.snapshot(query='张女士', conversation_limit=8)
|
|
self.assertEqual([item['id'] for item in result['conversations']], ['M:200'])
|
|
|
|
def test_content_search_checks_all_conversations(self):
|
|
self.update('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('999', 'M:old', 2, 1, 'UNIQUE-OLD-PREVIEW', 'old', ''))
|
|
result = self.snapshot(query='unique-old-preview', conversation_limit=1)
|
|
self.assertEqual(result['selectedConversation'], 'M:old')
|
|
|
|
def test_conversation_and_message_offsets_are_stable_with_timestamp_ties(self):
|
|
for i in range(10):
|
|
self.update('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('200', 'M:200', 2, 500, f'page {i}', f'p-{i}', ''))
|
|
latest = self.snapshot(selected_conversation='M:200', message_limit=3)
|
|
older = self.snapshot(selected_conversation='M:200', message_limit=3, message_offset=3)
|
|
self.assertEqual([row['id'] for row in latest['messages']], ['p-7', 'p-8', 'p-9'])
|
|
self.assertEqual([row['id'] for row in older['messages']], ['p-4', 'p-5', 'p-6'])
|
|
self.assertEqual(latest['conversations'][0]['preview'], 'page 9')
|
|
page_two = self.snapshot(conversation_limit=1, conversation_offset=1)
|
|
self.assertEqual(page_two['conversations'][0]['id'], 'S:100_300')
|
|
self.assertEqual(self.snapshot(message_offset=100)['messages'], [])
|
|
|
|
def test_search_offset_is_applied_after_filtering(self):
|
|
for i in range(5):
|
|
self.update('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('x', f'R:{i}', 2, 400 + i, 'match-preview', f'r-{i}', ''))
|
|
result = self.snapshot(query='match-preview', conversation_limit=2, conversation_offset=2)
|
|
self.assertEqual([row['id'] for row in result['conversations']], ['R:2', 'R:1'])
|
|
|
|
def test_system_conversations_are_excluded_but_total_message_count_is_exact(self):
|
|
for cid in ('Y:upper', 'y:lower', '', None):
|
|
self.update('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)', ('x', cid, 2, 999, 'system', 'sys', ''))
|
|
result = self.snapshot()
|
|
self.assertEqual(result['messageCount'], 7)
|
|
self.assertEqual(result['conversationCount'], 2)
|
|
self.assertEqual(len(result['conversations']), 2)
|
|
|
|
def test_snapshots_do_not_expose_mutable_cache_objects(self):
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
result['files'][0]['rows'] = -1
|
|
result['messages'][0]['content'] = 'polluted'
|
|
result['conversations'][0]['name'] = 'polluted'
|
|
fresh = self.snapshot(selected_conversation='M:200')
|
|
self.assertGreaterEqual(fresh['files'][0]['rows'], 0)
|
|
self.assertNotEqual(fresh['messages'][0]['content'], 'polluted')
|
|
self.assertNotEqual(fresh['conversations'][0]['name'], 'polluted')
|
|
|
|
def test_database_replace_with_preserved_mtime_and_size_invalidates_inode(self):
|
|
original = self.snapshot(selected_conversation='M:200')
|
|
stamp = self.database.stat()
|
|
replacement = self.account / 'replacement.tmp'
|
|
replacement.write_bytes(self.database.read_bytes())
|
|
self.update('UPDATE message_table SET server_id=? WHERE server_id=?', ('z1', 's1'), replacement)
|
|
self.assertEqual(stamp.st_size, replacement.stat().st_size)
|
|
os.utime(replacement, ns=(stamp.st_atime_ns, stamp.st_mtime_ns))
|
|
os.replace(replacement, self.database)
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
self.assertNotEqual(original['messages'][0]['id'], result['messages'][0]['id'])
|
|
self.assertEqual(result['messages'][0]['id'], 'z1')
|
|
|
|
def test_multiple_accounts_do_not_rebuild_unselected_indexes(self):
|
|
for i in range(5):
|
|
fixtures.WeChatMessageBrowserTests()._database(self.root, str(200 + i))
|
|
self.snapshot(selected_conversation='M:200')
|
|
with mock.patch.object(browser, 'connect_sqlite', side_effect=AssertionError('Warm account summary must stay cached')):
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
self.assertEqual(result['accountCount'], 6)
|
|
self.assertLessEqual(len(browser._CACHES['index']), 4)
|
|
|
|
def test_reader_never_changes_source_database_or_adds_indexes(self):
|
|
before = hashlib.sha256(self.database.read_bytes()).hexdigest()
|
|
self.snapshot(query='张女士', selected_conversation='M:200')
|
|
self.assertEqual(hashlib.sha256(self.database.read_bytes()).hexdigest(), before)
|
|
db = sqlite3.connect(self.database)
|
|
try:
|
|
self.assertEqual(db.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='index'").fetchone()[0], 0)
|
|
finally:
|
|
db.close()
|
|
|
|
def test_refresh_explicitly_invalidates_even_unchanged_metadata(self):
|
|
self.snapshot()
|
|
with mock.patch.object(browser, '_refresh_cache', return_value=('', '')) as refresh, mock.patch.object(browser, '_columns', wraps=browser._columns) as columns:
|
|
self.snapshot(refresh_cache=True)
|
|
refresh.assert_called_once()
|
|
self.assertGreater(columns.call_count, 0)
|
|
|
|
def test_concurrent_navigation_uses_consistent_cache(self):
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
results = list(pool.map(lambda _: self.snapshot(selected_conversation='M:200'), range(8)))
|
|
self.assertTrue(all(result['messageCount'] == 3 for result in results))
|
|
self.assertTrue(all([row['id'] for row in result['messages']] == ['s1', 's2'] for result in results))
|
|
|
|
def test_concurrent_account_switches_cannot_leak_another_accounts_page(self):
|
|
other = fixtures.WeChatMessageBrowserTests()._database(self.root, '777')
|
|
self.update('UPDATE message_table SET server_id=? WHERE server_id=?', ('account-777', 's2'), other / 'message.db')
|
|
def read(account):
|
|
return browser.load_browser_snapshot(cache_roots=[self.root], selected_account=account, selected_conversation='M:200')
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
results = list(pool.map(read, ['100', '777'] * 8))
|
|
for index, result in enumerate(results):
|
|
account = '100' if index % 2 == 0 else '777'
|
|
self.assertEqual(result['selectedAccount'], account)
|
|
self.assertEqual(result['messages'][-1]['senderId'], account)
|
|
self.assertEqual(result['messages'][-1]['id'], 's2' if account == '100' else 'account-777')
|
|
|
|
def test_deletion_invalidates_message_and_conversation_counts(self):
|
|
self.snapshot(selected_conversation='M:200')
|
|
self.update('DELETE FROM message_table WHERE conversation_id=?', ('M:200',))
|
|
result = self.snapshot(selected_conversation='M:200')
|
|
self.assertEqual(result['messageCount'], 1)
|
|
self.assertEqual(result['conversationCount'], 1)
|
|
self.assertEqual(result['selectedConversation'], 'S:100_300')
|
|
self.assertEqual(result['messages'][0]['id'], 's3')
|
|
|
|
|
|
def _replacement_with_reused_rowids(self):
|
|
replacement = self.account / 'rowids-reused.tmp'
|
|
db = sqlite3.connect(replacement)
|
|
try:
|
|
db.execute('CREATE TABLE message_table(sender_id TEXT,conversation_id TEXT,content_type INTEGER,send_time INTEGER,content BLOB,server_id TEXT,client_id TEXT)')
|
|
db.executemany('INSERT INTO message_table VALUES(?,?,?,?,?,?,?)',
|
|
[('other', 'M:999', 2, 900 + i, 'OTHER CONVERSATION PRIVATE BODY', f'other-{i}', '') for i in range(3)])
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
return replacement
|
|
|
|
def test_database_replaced_between_index_and_message_read_cannot_leak_conversation(self):
|
|
browser._message_index(self.database)
|
|
replacement = self._replacement_with_reused_rowids()
|
|
original = browser._read_message_rows
|
|
def replace_before_read(*args, **kwargs):
|
|
os.replace(replacement, self.database)
|
|
return original(*args, **kwargs)
|
|
with mock.patch.object(browser, '_read_message_rows', side_effect=replace_before_read):
|
|
messages = browser._messages(self.database, '100', 'M:200', {}, 12)
|
|
self.assertEqual(messages, [])
|
|
refreshed = self.snapshot()
|
|
self.assertEqual(refreshed['selectedConversation'], 'M:999')
|
|
|
|
def test_database_replaced_between_index_and_preview_read_cannot_leak_conversation(self):
|
|
browser._message_index(self.database)
|
|
replacement = self._replacement_with_reused_rowids()
|
|
original = browser._read_message_rows
|
|
def replace_before_read(*args, **kwargs):
|
|
os.replace(replacement, self.database)
|
|
return original(*args, **kwargs)
|
|
with mock.patch.object(browser, '_read_message_rows', side_effect=replace_before_read):
|
|
rows, _, _ = browser._conversation_rows(self.database, '100', {}, {}, '', 8)
|
|
self.assertTrue(rows)
|
|
self.assertTrue(all(item['preview'] == '' for item in rows))
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|