#!/usr/bin/env python3 """Publish reviewed avatar processing and H5 assets without changing user data.""" import fcntl import gzip import hashlib import json import os import pwd import re import shutil import subprocess import sys import time import urllib.error import urllib.request import zipfile from pathlib import Path sys.path.insert(0, '/www/server/xingyu-im/ops') from baota_publish_test_users import BT, WEB, DOMAIN, database, digest, env_values, extract, fetch, health, query, restart, switch OLD = '812ceedad7a7714ad9d04558946370d5665b0a2a9231b0e1282cb65def9f03ab' BASE = Path('/www/server/xingyu-im/releases/im-unread-20260831-124713') ALLOWED = {'backend/go.mod', 'backend/go.sum', 'backend/internal/app/media.go', 'backend/internal/app/avatar_images.go', 'backend/internal/app/avatar_images_test.go', 'backend/internal/app/media_batch.go', 'backend/internal/app/media_batch_test.go', 'backend/internal/app/media_upload_test.go', 'backend/docs/avatar-thumbnails.md'} def source_hashes(archive): with zipfile.ZipFile(str(archive)) as package: result = {} for item in package.infolist(): if item.is_dir(): continue name = item.filename.replace('\\', '/') assert name not in result and not name.startswith('/') and '..' not in Path(name).parts result[name] = hashlib.sha256(package.read(item)).hexdigest() return result def running(expected): pids = [] for proc in Path('/proc').iterdir(): if not proc.name.isdigit(): continue try: if os.readlink(str(proc / 'exe')) == str(BT / 'xingyu-api'): assert digest(proc / 'exe') == expected pids.append(int(proc.name)) except (FileNotFoundError, PermissionError): pass assert len(pids) == 1 return pids def marker(conn): configs = query(conn, 'SELECT config_key,config_value FROM system_configs ORDER BY config_key') profiles = query(conn, 'SELECT p.user_id,p.avatar_url FROM user_profiles p JOIN users u ON u.id=p.user_id WHERE u.is_test=1 ORDER BY p.user_id') return {'configValuesSha256': hashlib.sha256(repr(configs).encode()).hexdigest(), 'testAvatarsSha256': hashlib.sha256(repr(profiles).encode()).hexdigest(), 'testUsers': len(profiles)} def replace_binary(source, stamp): target = BT / ('xingyu-api.avatars-' + stamp) assert not target.exists() shutil.copy2(str(source), str(target)) www = pwd.getpwnam('www') os.chown(str(target), www.pw_uid, www.pw_gid) os.chmod(str(target), 0o750) os.replace(str(target), str(BT / 'xingyu-api')) def prepare_h5(stage, stamp): entry = WEB / 'app' previous = entry.resolve() releases = Path('/www/wwwroot/xingyu-h5/releases') assert entry.is_symlink() and previous.parent == releases release = releases / ('avatars-' + stamp) extract(stage / 'h5.zip', release) assert '/app/assets/' in (release / 'index.html').read_text() scripts = '\n'.join(p.read_text() for p in (release / 'assets').glob('*.js')) assert all(value in scripts for value in ['uploadAvatar', '-av1', '-thumb.jpg', 'https://im.bchongw.com']) for source in (previous / 'assets').rglob('*'): if source.is_file(): target = release / source.relative_to(previous) if not target.exists(): target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(source), str(target)) www = pwd.getpwnam('www') for parent, _, files in os.walk(str(release)): os.chown(parent, www.pw_uid, www.pw_gid) os.chmod(parent, 0o755) for name in files: path = os.path.join(parent, name) os.chown(path, www.pw_uid, www.pw_gid) os.chmod(path, 0o644) return entry, previous, release def public_checks(stage): verified = [] with zipfile.ZipFile(str(stage / 'h5.zip')) as package: index = package.read('index.html') status, headers, body = fetch('/app/') assert status == 200 and body == index assert 'no-store' in ','.join(headers.get_all('Cache-Control', [])) paths = set(re.findall(r'(?:src|href)="(/app/[^"?#]+)', index.decode())) for name in package.namelist(): if name.startswith(('assets/pages-edit-profile-', 'assets/pages-home-', 'assets/pages-chat-', 'assets/pages-messages-', 'assets/AvatarImage')) and name.endswith(('.js', '.css')): paths.add('/app/' + name) paths.add('/app/static/favicon.svg') for path in sorted(paths): if path.endswith(('.js', '.css', '.svg')): status, _, body = fetch(path) assert status == 200 and body == package.read(path[len('/app/'):]), path verified.append(path) assert fetch('/admin/')[0] == 200 for path, method in [('/api/v1/media/upload', 'POST'), ('/api/v1/im/conversations', 'GET'), ('/admin/v1/users', 'GET'), ('/ws', 'GET')]: try: with urllib.request.urlopen(urllib.request.Request(DOMAIN + path, method=method), timeout=20): raise RuntimeError('private endpoint became public') except urllib.error.HTTPError as error: assert error.code == 401 health() return verified def publish(archive, expected): assert os.geteuid() == 0 and re.fullmatch('[a-f0-9]{64}', expected) archive = Path(archive).resolve() assert archive.parent == Path('/tmp') and digest(archive) == expected lock = open('/www/server/xingyu-im/ops/application-release.lock', 'a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) assert digest(BT / 'xingyu-api') == OLD, 'production changed; re-audit required' running(OLD) health() subprocess.run(['/www/server/nginx/sbin/nginx', '-t'], check=True) stamp = time.strftime('%Y%m%d-%H%M%S') stage = Path('/www/server/xingyu-im/releases/avatars-' + stamp) extract(archive, stage) os.chmod(str(stage), 0o700) manifest = json.loads((stage / 'manifest.json').read_text()) for name, sha in manifest['files'].items(): path = (stage / name).resolve() assert stage in path.parents and path.is_file() and digest(path) == sha old_source = source_hashes(BASE / 'backend-source.zip') new_source = source_hashes(stage / 'backend-source.zip') changed = {name for name in set(old_source) | set(new_source) if old_source.get(name) != new_source.get(name)} assert changed == ALLOWED, 'unreviewed source changes: ' + repr(changed) for name in ['xingyu-api', 'avatar-check.test']: assert (stage / name).read_bytes()[:4] == b'\x7fELF' probe = stage / 'avatar-check.test' os.chmod(str(probe), 0o700) # These exact tests use temporary local files and in-memory database/storage # fixtures. They do not authenticate as a user or touch production records. cases = 'TestAvatarVariantsDimensionsAndSource|TestAvatarPhoneEXIFOrientation|TestAvatarSmallTransparentImageIsNotEnlarged|TestAvatarRejectsInvalidAndExcessivePixels|TestAvatarGIFAndWebP|TestAvatarUploadStoresAndServesAllVariants|TestAvatarUploadFinalizationFailureCleansEveryObject|TestOrdinaryMediaUploadPreservesOriginal|TestInvalidAvatarUploadCreatesNoRecords|TestMediaVariantUploadCleanup|TestLocalUploadCollisionPreservesExistingObject' subprocess.run([str(probe), '-test.run=^(' + cases + ')$', '-test.v'], cwd=str(stage), check=True, timeout=90) print('LINUX_AVATAR_TESTS_PASSED', flush=True) conn, options = database(env_values()) try: migrations = dict(query(conn, 'SELECT version,checksum FROM schema_migrations')) assert {name: sha for name, sha in new_source.items() if name.startswith('backend/migrations/')} == {'backend/migrations/' + name: sha for name, sha in migrations.items()} # Original binary rollback is safe for current COS objects; local # storage would additionally need the new av1 static filename rule. assert query(conn, "SELECT config_value FROM system_configs WHERE config_key='storage.provider'")[0][0] == 'tencent_cos' before = marker(conn) admin = str((WEB / 'admin').resolve()) backup = Path('/www/backup/xingyu-avatars-' + stamp) backup.mkdir(mode=0o700) shutil.copy2(str(BT / 'xingyu-api'), str(backup / 'xingyu-api-original')) dump = subprocess.Popen(['/opt/mysql-8.4.11/bin/mysqldump', '-h', '127.0.0.1', '-P', '3307', '-u', options['user'], '--single-transaction', '--no-tablespaces', '--set-gtid-purged=OFF', '--routines', '--triggers', 'im'], env=dict(os.environ, MYSQL_PWD=options['password']), stdout=subprocess.PIPE, stderr=subprocess.PIPE) with gzip.open(str(backup / 'im.sql.gz'), 'wb') as output: shutil.copyfileobj(dump.stdout, output) dump.communicate() assert dump.returncode == 0, 'database backup failed' os.chmod(str(backup / 'im.sql.gz'), 0o600) with gzip.open(str(backup / 'im.sql.gz'), 'rb') as check: assert b'MySQL dump' in check.read(4096) while check.read(1024 * 1024): pass entry, previous, release = prepare_h5(stage, stamp) state = {'stage': str(stage), 'backup': str(backup), 'archiveSha256': expected, 'previousBackendSha256': OLD, 'backendSha256': manifest['files']['xingyu-api'], 'previousH5': str(previous), 'h5': str(release), 'adminUnchanged': admin, 'changedBackendSourceFiles': sorted(changed), 'before': before, 'databaseMigrationsApplied': [], 'businessDataWrites': False, 'linuxAvatarTests': True} (backup / 'before.json').write_text(json.dumps(state, indent=2)) print('BACKUP_VERIFIED=' + str(backup), flush=True) binary_changed = switched = False try: replace_binary(stage / 'xingyu-api', stamp) binary_changed = True print('BACKEND_RESTARTING', flush=True) restart() state['runningPids'] = running(state['backendSha256']) switch(entry, release, stamp) switched = True state['publicAssetsVerified'] = public_checks(stage) assert marker(conn) == before, 'configuration or test avatars changed during release' assert dict(query(conn, 'SELECT version,checksum FROM schema_migrations')) == migrations assert str((WEB / 'admin').resolve()) == admin state['healthAndAuthChecks'] = True state['preservedConfigurationAndTestAvatars'] = True (backup / 'result.json').write_text(json.dumps(state, indent=2)) print('PUBLISHED=' + json.dumps(state), flush=True) except Exception: if switched: switch(entry, previous, stamp + '-rollback') if binary_changed: replace_binary(backup / 'xingyu-api-original', stamp + '-rollback') restart() running(OLD) print('ROLLED_BACK_APPLICATION; database untouched; backup=' + str(backup), flush=True) raise finally: conn.close() if __name__ == '__main__': publish(*sys.argv[1:])