Files
zyt/artifacts/private_dify_rollout_actions_20261027.py
T
2026-10-27 14:03:35 +08:00

300 lines
18 KiB
Python

"""Prepared metadata-bound API image rollout/rollback. No action on import."""
from pathlib import Path
import importlib.util
import json
import re
import shlex
import time
PROJECT = 'dify_v1110_eera'
SERVICE = 'api'
COMPOSE = '/www/dk_project/dk_app/dify_v1110/dify_v1110_Eera/docker-compose.yml'
WORKDIR = '/www/dk_project/dk_app/dify_v1110/dify_v1110_Eera'
ACTIONS = frozenset({'prepare-deploy', 'deploy', 'rollback', 'deployment-status'})
def _base(root):
spec = importlib.util.spec_from_file_location('private_rollout_fixed_base', root / 'artifacts/private_dify_ops_actions_20261027.py')
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
return module
def _prefix(extra=None):
parts = ['docker', 'compose', '--project-directory', WORKDIR, '-p', PROJECT, '-f', COMPOSE]
if extra:
parts += ['-f', extra]
return ' '.join(map(shlex.quote, parts))
def _current(client, base):
# Include exited/created containers: a failed replacement must remain recoverable.
raw, meta = base._exec(client, _prefix() + ' ps --all -q ' + SERVICE, cap=4096)
identifier = raw.decode('ascii').strip()
if not identifier:
return None
base._require(re.fullmatch(r'[0-9a-f]{12,64}', identifier) is not None, 'COMPOSE_API_IDENTITY_AMBIGUOUS')
template = '{"container_id":{{json .Id}},"image_id":{{json .Image}},"status":{{json .State.Status}},"restart_count":{{json .RestartCount}},"health":{{with (index .State "Health")}}{{json (index . "Status")}}{{else}}null{{end}},"project":{{json (index .Config.Labels "com.docker.compose.project")}},"service":{{json (index .Config.Labels "com.docker.compose.service")}}}'
data, command = base._json_command(client, 'docker inspect --format ' + shlex.quote(template) + ' ' + identifier)
base._require(isinstance(data, dict) and data.get('project') == PROJECT and data.get('service') == SERVICE,
'COMPOSE_API_BINDING_MISMATCH')
base._require(re.fullmatch(r'[0-9a-f]{64}', data.get('container_id', '')) is not None
and data['container_id'].startswith(identifier)
and re.fullmatch(r'sha256:[0-9a-f]{64}', data.get('image_id', '')) is not None,
'COMPOSE_API_METADATA_INVALID')
return data
def _compose_fingerprint(client, base):
"""Hash resolved inputs on their original host; never export config/Env values."""
code = '''import hashlib,json,pathlib,subprocess,sys
command=json.loads(sys.argv[1]); result=subprocess.run(command,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=25)
assert result.returncode==0 and len(result.stdout)<=16777216
model=json.loads(result.stdout); assert isinstance(model,dict) and 'api' in model.get('services',{})
canonical=json.dumps(model,sort_keys=True,separators=(',',':')).encode()
source=pathlib.Path(sys.argv[2]).read_bytes(); assert len(source)<=2097152
print(json.dumps({'resolved_sha256':hashlib.sha256(canonical).hexdigest(),'compose_sha256':hashlib.sha256(source).hexdigest()}))
'''
args = ['docker', 'compose', '--project-directory', WORKDIR, '-p', PROJECT, '-f', COMPOSE,
'config', '--format', 'json']
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
+ shlex.quote(json.dumps(args)) + ' ' + shlex.quote(COMPOSE), timeout=30)
base._require(isinstance(result, dict) and set(result) == {'resolved_sha256', 'compose_sha256'}
and all(isinstance(v, str) and re.fullmatch(r'[0-9a-f]{64}', v) for v in result.values()),
'COMPOSE_FINGERPRINT_INVALID')
return result
def _immutable_override(image_id):
if re.fullmatch(r'sha256:[0-9a-f]{64}', image_id or '') is None:
raise ValueError('invalid immutable image identity')
return json.dumps({'services': {SERVICE: {'image': image_id, 'pull_policy': 'never'}}}).encode()
def _verify_override(client, base, path, image_id):
code = '''import json,pathlib,sys
value=json.loads(pathlib.Path(sys.argv[1]).read_bytes())
assert value=={'services':{'api':{'image':sys.argv[2],'pull_policy':'never'}}}
print(json.dumps({'ok':True}))
'''
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
+ shlex.quote(path) + ' ' + shlex.quote(image_id), timeout=15)
base._require(result == {'ok': True}, 'ROLLOUT_OVERRIDE_CHANGED')
def _copy_compose_private(client, base, target, expected_sha256):
code = '''import hashlib,json,os,pathlib,sys
source=pathlib.Path(sys.argv[1]).read_bytes(); assert len(source)<=2097152
assert hashlib.sha256(source).hexdigest()==sys.argv[3]
fd=os.open(sys.argv[2],os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'wb') as output: output.write(source)
assert hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()==sys.argv[3]
print(json.dumps({'ok':True}))
'''
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
+ ' '.join(map(shlex.quote, [COMPOSE, target, expected_sha256])), timeout=20)
base._require(result == {'ok': True}, 'COMPOSE_PRIVATE_BACKUP_FAILED')
def _up_command(override):
# Image IDs and --no-build prevent a mutable tag/build/pull from selecting different bytes.
return _prefix(override) + ' up -d --no-deps --no-build --pull never --force-recreate ' + SERVICE
def _write_new_private(client, path, content):
sftp = client.open_sftp()
try:
sftp.get_channel().settimeout(20)
with sftp.file(path, 'wx') as handle:
handle.write(content)
sftp.chmod(path, 0o600)
finally:
sftp.close()
def _tree_check(client, base, identifier, manifest, which):
checks = [{'path': row['path'], 'sha256': row[which]} for row in manifest['targets']]
code = '''import hashlib,json,pathlib
rows=ROWS; root=pathlib.Path('/app/api'); good=0
for row in rows:
p=root/row['path']; assert (hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None)==row['sha256']; good+=1
print(json.dumps({'ok':True,'target_count':good}))
'''.replace('ROWS', repr(checks))
data, command = base._json_command(client, 'docker exec ' + shlex.quote(identifier) + ' python -B -c ' + shlex.quote(code), timeout=30)
base._require(data.get('ok') is True and data.get('target_count') == 17, 'ROLLOUT_TREE_CHECK_FAILED')
return data
def _stopped_candidate_tree_check(client, base, identifier, manifest):
"""Check only the 17 fixed files even when the candidate process cannot exec."""
checks = [{'path': row['path'], 'sha256': row['after_sha256']} for row in manifest['targets']]
code = '''import hashlib,io,json,os,selectors,subprocess,sys,tarfile,time
rows=json.loads(sys.argv[2]); good=0; overall=time.monotonic()+35
for row in rows:
command=['docker','cp',sys.argv[1]+':/app/api/'+row['path'],'-']
child=subprocess.Popen(command,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL)
poll=selectors.DefaultSelector(); poll.register(child.stdout,selectors.EVENT_READ)
try:
data=bytearray(); deadline=min(overall,time.monotonic()+8)
while True:
assert time.monotonic()<deadline
if not poll.select(.1): continue
chunk=os.read(child.stdout.fileno(),65536)
if not chunk: break
data.extend(chunk); assert len(data)<=2097152
assert child.wait(timeout=2)==0
finally:
if child.poll() is None: child.kill(); child.wait(timeout=5)
poll.close(); child.stdout.close()
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
members=archive.getmembers(); assert len(members)==1 and members[0].isfile() and members[0].size<=1048576
source=archive.extractfile(members[0]).read()
assert hashlib.sha256(source).hexdigest()==row['sha256']; good+=1
print(json.dumps({'ok':True,'target_count':good}))
'''
data, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
+ shlex.quote(identifier) + ' ' + shlex.quote(json.dumps(checks)), timeout=45)
base._require(data == {'ok': True, 'target_count': 17}, 'ROLLBACK_CANDIDATE_TREE_CHANGED')
return data
def run_action(client, action, repo_root):
root = Path(repo_root).resolve(strict=True)
base = _base(root)
safe_action = action if isinstance(action, str) and action in ACTIONS else 'invalid'
try:
base._require(isinstance(action, str) and action in ACTIONS, 'ACTION_NOT_ALLOWED')
transport = client.get_transport()
base._require(transport is not None and transport.is_active() and transport.is_authenticated(), 'AUTHENTICATED_SSH_REQUIRED')
base._require(transport.sock.getpeername()[0] == base.HOST, 'SSH_DESTINATION_MISMATCH')
state = base._load_state(root)
metadata = state.get('inspect', {})
base._require(metadata.get('compose_project') == PROJECT and metadata.get('compose_service') == SERVICE
and metadata.get('compose_config_files') == COMPOSE and metadata.get('compose_working_dir') == WORKDIR,
'INSPECTED_COMPOSE_METADATA_MISMATCH')
current = _current(client, base)
if action == 'deployment-status':
return {'ok': True, 'action': action, 'current': current,
'deployment_recorded': state.get('deployed') is True,
'rollback_recorded': state.get('rolled_back') is True, 'runtime_environment_read': False}
package, manifest, seal = base._package(root)
base._require(state.get('staged') is True, 'STAGE_REQUIRED')
if action == 'prepare-deploy':
spec = importlib.util.spec_from_file_location('fixed_candidate_preflight', root / 'artifacts/private_dify_preflight_20261027.py')
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
import sys
return module.prepare(client, base, sys.modules[__name__], root, state, current)
host_stage = state.get('host_stage')
base._require(isinstance(host_stage, str)
and re.fullmatch(r'/home/ps/\.zyt-dify-stop-20261027-[0-9a-f]{32}', host_stage) is not None,
'PRIVATE_ROLLOUT_STAGE_INVALID')
if action == 'deploy':
base._require(state.get('imports_passed') is True and state.get('built') is True, 'IMPORTS_AND_BUILT_IMAGE_REQUIRED')
base._require(state.get('preflight_passed') is True
and state.get('preflight_image_id') == state.get('candidate_image_id')
and state.get('preflight_compose_fingerprint') == _compose_fingerprint(client, base),
'INSTALLED_PREFLIGHT_REQUIRED')
observation = state.get('maintenance_observation',{})
approval = state.get('maintenance_approval',{})
base._require(observation.get('observation_only') is True and (
observation.get('all_samples_zero') is True or (
approval.get('user_explicitly_approved_one_connection_interruption') is True
and observation.get('established_inbound_samples') == [1,1,1]
and approval.get('baseline_container_id') == metadata.get('container_id')
and approval.get('candidate_image_id') == state.get('candidate_image_id'))),
'MAINTENANCE_CONNECTION_OBSERVATION_REQUIRED')
base._require(not state.get('deployment_started'), 'PRIOR_DEPLOY_OUTCOME_REQUIRES_STATUS_REVIEW')
base._require(current is not None and current['container_id'] == metadata['container_id']
and current['image_id'] == metadata['image_id'] and current.get('status') == 'running',
'PRE_DEPLOY_BASELINE_CHANGED')
image, image_meta = base._image_metadata(client, state['candidate_image_tag'])
base._require(image['image_id'] == state['candidate_image_id'] and image.get('manifest') == base.MANIFEST_SHA256
and image.get('base_image') == metadata['image_id'], 'CANDIDATE_IMAGE_PROVENANCE_MISMATCH')
original, base_meta = base._image_metadata(client, state['base_image_tag'])
base._require(original['image_id'] == metadata['image_id'], 'ROLLBACK_IMAGE_MISSING_OR_CHANGED')
_tree_check(client, base, current['container_id'], manifest, 'before_sha256')
forward = host_stage + '/api-candidate.override.json'
inverse = host_stage + '/api-baseline.override.json'
fingerprint = _compose_fingerprint(client, base)
_write_new_private(client, forward, _immutable_override(state['candidate_image_id']))
_write_new_private(client, inverse, _immutable_override(metadata['image_id']))
# This is an exact compose-file backup, not a self-contained .env/env_file backup.
_copy_compose_private(client, base, host_stage + '/docker-compose.before.private.yml', fingerprint['compose_sha256'])
state.update(deployment_started=True, forward_override=forward, rollback_override=inverse,
rollout_compose_fingerprint=fingerprint)
base._save_state(root, state)
base._require(_compose_fingerprint(client, base) == fingerprint, 'COMPOSE_INPUTS_CHANGED')
_verify_override(client, base, forward, state['candidate_image_id'])
command = _up_command(forward)
base._exec(client, command, timeout=120, cap=1048576, keep_stdout=False)
expected = state['candidate_image_id']
which = 'after_sha256'
else:
base._require(action == 'rollback' and state.get('deployment_started') is True, 'PRIOR_DEPLOY_REQUIRED')
base._require(not state.get('rollback_started'), 'PRIOR_ROLLBACK_OUTCOME_REQUIRES_STATUS_REVIEW')
# Failed creation may leave no service container; an exited candidate is also recoverable.
original_still_running = (current is not None and current['container_id'] == metadata['container_id']
and current['image_id'] == metadata['image_id'] and current.get('status') == 'running'
and current.get('health') in (None, 'healthy'))
base._require(current is None or current['image_id'] == state.get('candidate_image_id') or original_still_running,
'ROLLBACK_CURRENT_IMAGE_CHANGED')
original, image_meta = base._image_metadata(client, state['base_image_tag'])
base._require(original['image_id'] == metadata['image_id'], 'ROLLBACK_IMAGE_IDENTITY_MISMATCH')
override = state['rollback_override']
base._require(override == state['host_stage'] + '/api-baseline.override.json', 'ROLLBACK_OVERRIDE_PATH_INVALID')
fingerprint = state.get('rollout_compose_fingerprint')
base._require(isinstance(fingerprint, dict) and _compose_fingerprint(client, base) == fingerprint,
'ROLLBACK_COMPOSE_INPUTS_CHANGED_OR_UNRECORDED')
_verify_override(client, base, override, metadata['image_id'])
if original_still_running:
checked = _tree_check(client, base, current['container_id'], manifest, 'before_sha256')
state.update(rolled_back=True, rollback_container_id=current['container_id'],
rollback_result='original_baseline_already_running')
base._save_state(root, state)
return {'ok': True, 'action': action, 'current': current, 'source_tree': checked,
'baseline_already_running': True, 'service_recreated': False,
'model_or_cancel_acceptance_claimed': False}
if current is not None:
if current.get('status') == 'running':
_tree_check(client, base, current['container_id'], manifest, 'after_sha256')
else:
_stopped_candidate_tree_check(client, base, current['container_id'], manifest)
# An exclusive per-action marker prevents concurrent callers from rolling back twice.
_write_new_private(client, state['host_stage'] + '/rollback.started.private', b'rollback\n')
state['rollback_started'] = True
base._save_state(root, state)
command = _up_command(override)
base._exec(client, command, timeout=120, cap=1048576, keep_stdout=False)
expected = metadata['image_id']
which = 'before_sha256'
# Basic process/container readiness only; actual model/cancel validation stays separate.
final = None
for unused in range(15):
final = _current(client, base)
if final is not None and final['image_id'] == expected and final.get('status') == 'running' and final.get('health') in (None, 'healthy'):
break
time.sleep(2)
base._require(final is not None and final['image_id'] == expected and final.get('status') == 'running'
and final.get('health') in (None, 'healthy'), 'ROLLOUT_READINESS_UNCONFIRMED')
checked = _tree_check(client, base, final['container_id'], manifest, which)
base._require(_compose_fingerprint(client, base) == state['rollout_compose_fingerprint'],
'POST_ROLLOUT_COMPOSE_INPUTS_CHANGED')
if action == 'deploy':
state.update(deployed=True, deployed_container_id=final['container_id'])
else:
state.update(rolled_back=True, rollback_container_id=final['container_id'])
base._save_state(root, state)
return {'ok': True, 'action': action, 'current': final, 'source_tree': checked,
'service_recreated': SERVICE, 'other_services_recreated': False,
'direct_database_write_performed_by_helper': False,
'entrypoint_database_effects_verified': False, 'runtime_environment_baked': False,
'resolved_configuration_exported': False,
'model_or_cancel_acceptance_claimed': False}
except base.OpsError as error:
return {'ok': False, 'action': safe_action, 'error_code': error.code,
'diagnostic': getattr(error,'diagnostic',None),
'raw_remote_output_suppressed': True, 'automatic_retry_performed': False}
except Exception as error:
return {'ok': False, 'action': safe_action, 'error_code': 'ROLLOUT_ACTION_FAILED',
'error_type': type(error).__name__, 'raw_remote_output_suppressed': True,
'automatic_retry_performed': False}