241 lines
9.9 KiB
Python
241 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Publish seeded discovery shuffling with automatic binary rollback."""
|
|
import base64
|
|
import fcntl
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
import pwd
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, '/www/server/xingyu-im/ops')
|
|
from baota_publish_test_users import BT, WEB, database, digest, env_values, extract, query
|
|
|
|
OLD = '313e8216c3a225d2eb6a8b6cacb516b7cd9b35c6ef2c03d01721c8d521f7a436'
|
|
BASE = Path('/www/server/xingyu-im/releases/nearby-distance-20260902-083423')
|
|
LOCAL = 'http://127.0.0.1:18888'
|
|
ALLOWED = {'backend/internal/app/social.go'}
|
|
|
|
|
|
def fetch(path):
|
|
with urllib.request.urlopen(LOCAL + path, timeout=20) as response:
|
|
return response.status, response.headers, response.read()
|
|
|
|
|
|
def health():
|
|
for _ in range(100):
|
|
try:
|
|
status, _, data = fetch('/healthz')
|
|
if status == 200 and json.loads(data)['data']['status'] == 'ok':
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise RuntimeError('backend failed local health check')
|
|
|
|
|
|
def restart():
|
|
os.chdir('/www/server/panel')
|
|
sys.path.insert(0, '/www/server/panel')
|
|
sys.path.insert(0, '/www/server/panel/class')
|
|
import public
|
|
from projectModel.goModel import main as GoProject
|
|
request = public.dict_obj()
|
|
request.project_name = 'xingyu_im'
|
|
manager = GoProject()
|
|
project = manager.get_project_find('xingyu_im')
|
|
assert project and project['project_config']['is_power_on'] == 1
|
|
result = manager.restart_project(request)
|
|
if not result.get('status'):
|
|
recovered = manager.start_project(request)
|
|
if not recovered.get('status'):
|
|
raise RuntimeError('BaoTa could not start project')
|
|
health()
|
|
|
|
|
|
def source_hashes(archive):
|
|
with zipfile.ZipFile(str(archive)) as package:
|
|
result = {}
|
|
for item in package.infolist():
|
|
if item.is_dir():
|
|
continue
|
|
name = item.filename.replace('\\', '/')
|
|
assert name not in result and not name.startswith('/') and '..' not in Path(name).parts
|
|
result[name] = hashlib.sha256(package.read(item)).hexdigest()
|
|
return result
|
|
|
|
|
|
def running(expected):
|
|
pids = []
|
|
for proc in Path('/proc').iterdir():
|
|
if not proc.name.isdigit():
|
|
continue
|
|
try:
|
|
if os.readlink(str(proc / 'exe')) == str(BT / 'xingyu-api'):
|
|
assert digest(proc / 'exe') == expected
|
|
pids.append(int(proc.name))
|
|
except (FileNotFoundError, PermissionError):
|
|
pass
|
|
assert len(pids) == 1, 'expected one running project executable'
|
|
return pids
|
|
|
|
|
|
def replace(source, stamp):
|
|
temporary = BT / ('xingyu-api.random-refresh-' + stamp)
|
|
assert not temporary.exists()
|
|
shutil.copy2(str(source), str(temporary))
|
|
www = pwd.getpwnam('www')
|
|
os.chown(str(temporary), www.pw_uid, www.pw_gid)
|
|
os.chmod(str(temporary), 0o750)
|
|
os.replace(str(temporary), str(BT / 'xingyu-api'))
|
|
|
|
|
|
def encode_json(value):
|
|
raw = json.dumps(value, separators=(',', ':'), ensure_ascii=False).encode()
|
|
return base64.urlsafe_b64encode(raw).rstrip(b'=').decode()
|
|
|
|
|
|
def user_token(secret, user_id, nickname, version):
|
|
now = int(time.time())
|
|
header = encode_json({'alg': 'HS256', 'typ': 'JWT'})
|
|
payload = encode_json({'role': 'user', 'name': nickname, 'ver': version, 'sub': str(user_id), 'iat': now, 'exp': now + 300})
|
|
signing_input = (header + '.' + payload).encode()
|
|
signature = base64.urlsafe_b64encode(hmac.new(secret.encode(), signing_input, hashlib.sha256).digest()).rstrip(b'=').decode()
|
|
return header + '.' + payload + '.' + signature
|
|
|
|
|
|
def api_json(path, token):
|
|
request = urllib.request.Request(
|
|
LOCAL + path,
|
|
headers={'Authorization': 'Bearer ' + token, 'Accept': 'application/json'},
|
|
)
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
assert response.status == 200
|
|
return json.loads(response.read())['data']
|
|
|
|
|
|
def item_ids(result):
|
|
return [int(item['id']) for item in result.get('items', [])]
|
|
|
|
|
|
def verify_seeded_endpoint(path, token):
|
|
joiner = '&' if '?' in path else '?'
|
|
first = api_json(path + joiner + 'page=1&pageSize=20&seed=1234567', token)
|
|
repeat = api_json(path + joiner + 'page=1&pageSize=20&seed=1234567', token)
|
|
second_page = api_json(path + joiner + 'page=2&pageSize=20&seed=1234567', token)
|
|
changed = api_json(path + joiner + 'page=1&pageSize=20&seed=7654321', token)
|
|
first_ids = item_ids(first)
|
|
repeat_ids = item_ids(repeat)
|
|
second_ids = item_ids(second_page)
|
|
changed_ids = item_ids(changed)
|
|
assert len(first_ids) >= 2, 'not enough rows to verify shuffling'
|
|
assert first_ids == repeat_ids, 'same seed returned a different order'
|
|
assert first_ids != changed_ids, 'different seeds returned the same order'
|
|
assert not set(first_ids).intersection(second_ids), 'seeded pages overlap'
|
|
assert len(first_ids + second_ids) == len(set(first_ids + second_ids)), 'seeded pagination contains duplicates'
|
|
invalid = api_json(path + joiner + 'page=1&pageSize=20&seed=invalid', token)
|
|
assert item_ids(invalid), 'invalid seed broke the normal list fallback'
|
|
return {
|
|
'firstPageItems': len(first_ids),
|
|
'secondPageItems': len(second_ids),
|
|
'sameSeedStable': True,
|
|
'differentSeedChanged': True,
|
|
'pagesDisjoint': True,
|
|
}
|
|
|
|
|
|
def verify_random_refresh(conn, values):
|
|
selected = query(conn, """SELECT user.id,profile.nickname,security.token_version
|
|
FROM users user JOIN user_profiles profile ON profile.user_id=user.id
|
|
JOIN user_security_controls security ON security.user_id=user.id
|
|
WHERE user.status=1 AND user.deleted_at IS NULL ORDER BY user.id LIMIT 1""")
|
|
assert selected, 'no active user available for random-refresh verification'
|
|
user_id, nickname, version = selected[0]
|
|
token = user_token(values['IM_JWT_SECRET'], user_id, nickname, version)
|
|
nearby = verify_seeded_endpoint('/api/v1/nearby/users', token)
|
|
discover = verify_seeded_endpoint('/api/v1/discover/recommendations', token)
|
|
return {'viewerUserId': user_id, 'nearby': nearby, 'discover': discover}
|
|
|
|
|
|
def publish(archive, expected):
|
|
assert os.geteuid() == 0
|
|
archive = Path(archive).resolve()
|
|
assert archive.parent == Path('/tmp') and digest(archive) == expected
|
|
lock = open('/www/server/xingyu-im/ops/application-release.lock', 'a')
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
assert digest(BT / 'xingyu-api') == OLD, 'production changed; re-audit required'
|
|
running(OLD)
|
|
health()
|
|
stamp = time.strftime('%Y%m%d-%H%M%S')
|
|
stage = Path('/www/server/xingyu-im/releases/random-refresh-' + stamp)
|
|
extract(archive, stage)
|
|
os.chmod(str(stage), 0o700)
|
|
old_source = source_hashes(BASE / 'backend-source.zip')
|
|
new_source = source_hashes(stage / 'backend-source.zip')
|
|
changed = {name for name in set(old_source) | set(new_source) if old_source.get(name) != new_source.get(name)}
|
|
assert changed == ALLOWED, 'unreviewed source changes: ' + repr(changed)
|
|
new_hash = digest(stage / 'xingyu-api')
|
|
assert (stage / 'xingyu-api').read_bytes()[:4] == b'\x7fELF' and new_hash != OLD
|
|
manifest = json.loads((stage / 'manifest.json').read_text())
|
|
for name in ['xingyu-api', 'backend-source.zip']:
|
|
assert digest(stage / name) == manifest['files'][name]
|
|
values = env_values()
|
|
conn, _ = database(values)
|
|
try:
|
|
migrations = dict(query(conn, 'SELECT version,checksum FROM schema_migrations'))
|
|
for name, sha in migrations.items():
|
|
assert new_source['backend/migrations/' + name] == sha
|
|
assert len(migrations) == len([name for name in new_source if name.startswith('backend/migrations/')])
|
|
frontend = {name: str((WEB / name).resolve()) for name in ['admin', 'app']}
|
|
backup = Path('/www/backup/xingyu-random-refresh-' + stamp)
|
|
backup.mkdir(mode=0o700)
|
|
shutil.copy2(str(BT / 'xingyu-api'), str(backup / 'xingyu-api-original'))
|
|
state = {
|
|
'backup': str(backup), 'stage': str(stage), 'previousBackendSha256': OLD,
|
|
'backendSha256': new_hash, 'archiveSha256': expected,
|
|
'changedSourceFiles': sorted(changed), 'frontendBefore': frontend,
|
|
'databaseMigrationsApplied': [], 'businessDataWrites': False,
|
|
}
|
|
(backup / 'before.json').write_text(json.dumps(state, indent=2))
|
|
print('BACKUP_VERIFIED=' + str(backup), flush=True)
|
|
changed_binary = False
|
|
try:
|
|
replace(stage / 'xingyu-api', stamp)
|
|
changed_binary = True
|
|
restart()
|
|
state['runningPids'] = running(new_hash)
|
|
assert digest(BT / 'xingyu-api') == new_hash
|
|
assert dict(query(conn, 'SELECT version,checksum FROM schema_migrations')) == migrations
|
|
assert {name: str((WEB / name).resolve()) for name in frontend} == frontend
|
|
for path in ['/api/v1/nearby/users', '/api/v1/discover/recommendations', '/admin/v1/users', '/ws']:
|
|
try:
|
|
fetch(path)
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 401
|
|
else:
|
|
raise RuntimeError('private endpoint became public')
|
|
state['randomRefresh'] = verify_random_refresh(conn, values)
|
|
state['healthAndAuthChecks'] = True
|
|
(backup / 'result.json').write_text(json.dumps(state, indent=2))
|
|
print(json.dumps(state), flush=True)
|
|
except Exception:
|
|
if changed_binary:
|
|
replace(backup / 'xingyu-api-original', stamp + '-rollback')
|
|
restart()
|
|
running(OLD)
|
|
(backup / 'rolled-back.txt').write_text('Original binary restored. No database restore was required.\n')
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
publish(*sys.argv[1:])
|