54 lines
3.0 KiB
Python
54 lines
3.0 KiB
Python
"""40万条 synthetic archive rows; the source fixture contains no real customer data."""
|
|
from pathlib import Path
|
|
import sqlite3,sys,time,json
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'wechat_rpa'))
|
|
import admin_backend
|
|
from knowledge_store import KnowledgeStore
|
|
root=Path(__file__).resolve().parent
|
|
target=root/'benchmark.db'
|
|
assert not target.exists(), 'Do not overwrite an existing benchmark'
|
|
with sqlite3.connect(root/'backend.db') as original, sqlite3.connect(target) as db:
|
|
original.backup(db)
|
|
original.close()
|
|
db.close()
|
|
db=sqlite3.connect(target)
|
|
db.execute('PRAGMA journal_mode=WAL')
|
|
db.execute('PRAGMA synchronous=NORMAL')
|
|
def clone(table, amount, replacements):
|
|
columns=[row[1] for row in db.execute('PRAGMA table_info('+table+')')]
|
|
expressions=[replacements.get(column, 'base."'+column+'"') for column in columns]
|
|
db.execute('WITH RECURSIVE seq(n) AS (VALUES(0) UNION ALL SELECT n+1 FROM seq WHERE n<?), '
|
|
'base AS (SELECT * FROM '+table+' LIMIT 1) INSERT INTO '+table+' ('+','.join(columns)+') '
|
|
'SELECT '+','.join(expressions)+' FROM base CROSS JOIN seq',(amount-1,))
|
|
started=time.monotonic()
|
|
clone('archive_conversation',12000,{'id':"'bench-c-'||n",'external_id':"'bench-c-'||n"})
|
|
existing=db.execute('SELECT COUNT(*) FROM archive_message').fetchone()[0]
|
|
clone('archive_message',400000-existing,{'id':"'bench-m-'||n",'conversation_id':"'bench-c-'||(n%12000)",
|
|
'source_message_id':"'bench-m-'||n",'dedup_key':"'bench-m-'||n"})
|
|
clone('archive_message_version',400000-existing,{'id':"'bench-v-'||n",'message_id':"'bench-m-'||n",'version_no':'1'})
|
|
db.commit()
|
|
source,tenant=db.execute('SELECT id,tenant_id FROM archive_source_account LIMIT 1').fetchone()
|
|
db.close()
|
|
print('Fixture built in',round(time.monotonic()-started,2),'seconds',flush=True)
|
|
store=KnowledgeStore(admin_backend.Database(target))
|
|
started=time.monotonic()
|
|
result=store.preview_job(tenant,{'source_account_id':source,'max_messages':10000})
|
|
elapsed=time.monotonic()-started
|
|
assert result['available_messages']==400000
|
|
assert result['total_messages']==10000 and result['limited']
|
|
started=time.monotonic()
|
|
with store.database.connect() as connection:
|
|
fast=store._scope_counts(connection,tenant,{'source_account_id':source,'max_messages':10000},result['cutoff_at'],quality=False)
|
|
assert fast['available_messages']==result['available_messages']
|
|
count_seconds=time.monotonic()-started
|
|
record={'snapshot_count_seconds':round(count_seconds,3),'synthetic_messages':400000,'conversations':result['conversations'],'preview_seconds':round(elapsed,3),
|
|
'planned_messages':result['total_messages'],'database_bytes':target.stat().st_size}
|
|
(root/'benchmark-result.json').write_text(json.dumps(record,indent=2),encoding='utf-8')
|
|
print(json.dumps(record),flush=True)
|
|
# This disposable benchmark contains only copied synthetic fixture data.
|
|
assert target.resolve().parent==root.resolve()
|
|
target.unlink()
|
|
for suffix in ('-wal','-shm'):
|
|
candidate=Path(str(target)+suffix)
|
|
if candidate.exists(): candidate.unlink()
|