300 lines
18 KiB
Python
300 lines
18 KiB
Python
"""Prepared metadata-bound API image rollout/rollback. No action on import."""
|
|
from pathlib import Path
|
|
import importlib.util
|
|
import json
|
|
import re
|
|
import shlex
|
|
import time
|
|
|
|
PROJECT = 'dify_v1110_eera'
|
|
SERVICE = 'api'
|
|
COMPOSE = '/www/dk_project/dk_app/dify_v1110/dify_v1110_Eera/docker-compose.yml'
|
|
WORKDIR = '/www/dk_project/dk_app/dify_v1110/dify_v1110_Eera'
|
|
ACTIONS = frozenset({'prepare-deploy', 'deploy', 'rollback', 'deployment-status'})
|
|
|
|
|
|
def _base(root):
|
|
spec = importlib.util.spec_from_file_location('private_rollout_fixed_base', root / 'artifacts/private_dify_ops_actions_20261027.py')
|
|
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _prefix(extra=None):
|
|
parts = ['docker', 'compose', '--project-directory', WORKDIR, '-p', PROJECT, '-f', COMPOSE]
|
|
if extra:
|
|
parts += ['-f', extra]
|
|
return ' '.join(map(shlex.quote, parts))
|
|
|
|
|
|
def _current(client, base):
|
|
# Include exited/created containers: a failed replacement must remain recoverable.
|
|
raw, meta = base._exec(client, _prefix() + ' ps --all -q ' + SERVICE, cap=4096)
|
|
identifier = raw.decode('ascii').strip()
|
|
if not identifier:
|
|
return None
|
|
base._require(re.fullmatch(r'[0-9a-f]{12,64}', identifier) is not None, 'COMPOSE_API_IDENTITY_AMBIGUOUS')
|
|
template = '{"container_id":{{json .Id}},"image_id":{{json .Image}},"status":{{json .State.Status}},"restart_count":{{json .RestartCount}},"health":{{with (index .State "Health")}}{{json (index . "Status")}}{{else}}null{{end}},"project":{{json (index .Config.Labels "com.docker.compose.project")}},"service":{{json (index .Config.Labels "com.docker.compose.service")}}}'
|
|
data, command = base._json_command(client, 'docker inspect --format ' + shlex.quote(template) + ' ' + identifier)
|
|
base._require(isinstance(data, dict) and data.get('project') == PROJECT and data.get('service') == SERVICE,
|
|
'COMPOSE_API_BINDING_MISMATCH')
|
|
base._require(re.fullmatch(r'[0-9a-f]{64}', data.get('container_id', '')) is not None
|
|
and data['container_id'].startswith(identifier)
|
|
and re.fullmatch(r'sha256:[0-9a-f]{64}', data.get('image_id', '')) is not None,
|
|
'COMPOSE_API_METADATA_INVALID')
|
|
return data
|
|
|
|
|
|
def _compose_fingerprint(client, base):
|
|
"""Hash resolved inputs on their original host; never export config/Env values."""
|
|
code = '''import hashlib,json,pathlib,subprocess,sys
|
|
command=json.loads(sys.argv[1]); result=subprocess.run(command,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=25)
|
|
assert result.returncode==0 and len(result.stdout)<=16777216
|
|
model=json.loads(result.stdout); assert isinstance(model,dict) and 'api' in model.get('services',{})
|
|
canonical=json.dumps(model,sort_keys=True,separators=(',',':')).encode()
|
|
source=pathlib.Path(sys.argv[2]).read_bytes(); assert len(source)<=2097152
|
|
print(json.dumps({'resolved_sha256':hashlib.sha256(canonical).hexdigest(),'compose_sha256':hashlib.sha256(source).hexdigest()}))
|
|
'''
|
|
args = ['docker', 'compose', '--project-directory', WORKDIR, '-p', PROJECT, '-f', COMPOSE,
|
|
'config', '--format', 'json']
|
|
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
|
|
+ shlex.quote(json.dumps(args)) + ' ' + shlex.quote(COMPOSE), timeout=30)
|
|
base._require(isinstance(result, dict) and set(result) == {'resolved_sha256', 'compose_sha256'}
|
|
and all(isinstance(v, str) and re.fullmatch(r'[0-9a-f]{64}', v) for v in result.values()),
|
|
'COMPOSE_FINGERPRINT_INVALID')
|
|
return result
|
|
|
|
|
|
def _immutable_override(image_id):
|
|
if re.fullmatch(r'sha256:[0-9a-f]{64}', image_id or '') is None:
|
|
raise ValueError('invalid immutable image identity')
|
|
return json.dumps({'services': {SERVICE: {'image': image_id, 'pull_policy': 'never'}}}).encode()
|
|
|
|
|
|
def _verify_override(client, base, path, image_id):
|
|
code = '''import json,pathlib,sys
|
|
value=json.loads(pathlib.Path(sys.argv[1]).read_bytes())
|
|
assert value=={'services':{'api':{'image':sys.argv[2],'pull_policy':'never'}}}
|
|
print(json.dumps({'ok':True}))
|
|
'''
|
|
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
|
|
+ shlex.quote(path) + ' ' + shlex.quote(image_id), timeout=15)
|
|
base._require(result == {'ok': True}, 'ROLLOUT_OVERRIDE_CHANGED')
|
|
|
|
|
|
def _copy_compose_private(client, base, target, expected_sha256):
|
|
code = '''import hashlib,json,os,pathlib,sys
|
|
source=pathlib.Path(sys.argv[1]).read_bytes(); assert len(source)<=2097152
|
|
assert hashlib.sha256(source).hexdigest()==sys.argv[3]
|
|
fd=os.open(sys.argv[2],os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
|
with os.fdopen(fd,'wb') as output: output.write(source)
|
|
assert hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()==sys.argv[3]
|
|
print(json.dumps({'ok':True}))
|
|
'''
|
|
result, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
|
|
+ ' '.join(map(shlex.quote, [COMPOSE, target, expected_sha256])), timeout=20)
|
|
base._require(result == {'ok': True}, 'COMPOSE_PRIVATE_BACKUP_FAILED')
|
|
|
|
|
|
def _up_command(override):
|
|
# Image IDs and --no-build prevent a mutable tag/build/pull from selecting different bytes.
|
|
return _prefix(override) + ' up -d --no-deps --no-build --pull never --force-recreate ' + SERVICE
|
|
|
|
|
|
def _write_new_private(client, path, content):
|
|
sftp = client.open_sftp()
|
|
try:
|
|
sftp.get_channel().settimeout(20)
|
|
with sftp.file(path, 'wx') as handle:
|
|
handle.write(content)
|
|
sftp.chmod(path, 0o600)
|
|
finally:
|
|
sftp.close()
|
|
|
|
|
|
def _tree_check(client, base, identifier, manifest, which):
|
|
checks = [{'path': row['path'], 'sha256': row[which]} for row in manifest['targets']]
|
|
code = '''import hashlib,json,pathlib
|
|
rows=ROWS; root=pathlib.Path('/app/api'); good=0
|
|
for row in rows:
|
|
p=root/row['path']; assert (hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None)==row['sha256']; good+=1
|
|
print(json.dumps({'ok':True,'target_count':good}))
|
|
'''.replace('ROWS', repr(checks))
|
|
data, command = base._json_command(client, 'docker exec ' + shlex.quote(identifier) + ' python -B -c ' + shlex.quote(code), timeout=30)
|
|
base._require(data.get('ok') is True and data.get('target_count') == 17, 'ROLLOUT_TREE_CHECK_FAILED')
|
|
return data
|
|
|
|
|
|
def _stopped_candidate_tree_check(client, base, identifier, manifest):
|
|
"""Check only the 17 fixed files even when the candidate process cannot exec."""
|
|
checks = [{'path': row['path'], 'sha256': row['after_sha256']} for row in manifest['targets']]
|
|
code = '''import hashlib,io,json,os,selectors,subprocess,sys,tarfile,time
|
|
rows=json.loads(sys.argv[2]); good=0; overall=time.monotonic()+35
|
|
for row in rows:
|
|
command=['docker','cp',sys.argv[1]+':/app/api/'+row['path'],'-']
|
|
child=subprocess.Popen(command,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL)
|
|
poll=selectors.DefaultSelector(); poll.register(child.stdout,selectors.EVENT_READ)
|
|
try:
|
|
data=bytearray(); deadline=min(overall,time.monotonic()+8)
|
|
while True:
|
|
assert time.monotonic()<deadline
|
|
if not poll.select(.1): continue
|
|
chunk=os.read(child.stdout.fileno(),65536)
|
|
if not chunk: break
|
|
data.extend(chunk); assert len(data)<=2097152
|
|
assert child.wait(timeout=2)==0
|
|
finally:
|
|
if child.poll() is None: child.kill(); child.wait(timeout=5)
|
|
poll.close(); child.stdout.close()
|
|
with tarfile.open(fileobj=io.BytesIO(data)) as archive:
|
|
members=archive.getmembers(); assert len(members)==1 and members[0].isfile() and members[0].size<=1048576
|
|
source=archive.extractfile(members[0]).read()
|
|
assert hashlib.sha256(source).hexdigest()==row['sha256']; good+=1
|
|
print(json.dumps({'ok':True,'target_count':good}))
|
|
'''
|
|
data, command = base._json_command(client, 'python3 -c ' + shlex.quote(code) + ' '
|
|
+ shlex.quote(identifier) + ' ' + shlex.quote(json.dumps(checks)), timeout=45)
|
|
base._require(data == {'ok': True, 'target_count': 17}, 'ROLLBACK_CANDIDATE_TREE_CHANGED')
|
|
return data
|
|
|
|
|
|
def run_action(client, action, repo_root):
|
|
root = Path(repo_root).resolve(strict=True)
|
|
base = _base(root)
|
|
safe_action = action if isinstance(action, str) and action in ACTIONS else 'invalid'
|
|
try:
|
|
base._require(isinstance(action, str) and action in ACTIONS, 'ACTION_NOT_ALLOWED')
|
|
transport = client.get_transport()
|
|
base._require(transport is not None and transport.is_active() and transport.is_authenticated(), 'AUTHENTICATED_SSH_REQUIRED')
|
|
base._require(transport.sock.getpeername()[0] == base.HOST, 'SSH_DESTINATION_MISMATCH')
|
|
state = base._load_state(root)
|
|
metadata = state.get('inspect', {})
|
|
base._require(metadata.get('compose_project') == PROJECT and metadata.get('compose_service') == SERVICE
|
|
and metadata.get('compose_config_files') == COMPOSE and metadata.get('compose_working_dir') == WORKDIR,
|
|
'INSPECTED_COMPOSE_METADATA_MISMATCH')
|
|
current = _current(client, base)
|
|
if action == 'deployment-status':
|
|
return {'ok': True, 'action': action, 'current': current,
|
|
'deployment_recorded': state.get('deployed') is True,
|
|
'rollback_recorded': state.get('rolled_back') is True, 'runtime_environment_read': False}
|
|
package, manifest, seal = base._package(root)
|
|
base._require(state.get('staged') is True, 'STAGE_REQUIRED')
|
|
if action == 'prepare-deploy':
|
|
spec = importlib.util.spec_from_file_location('fixed_candidate_preflight', root / 'artifacts/private_dify_preflight_20261027.py')
|
|
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
|
import sys
|
|
return module.prepare(client, base, sys.modules[__name__], root, state, current)
|
|
host_stage = state.get('host_stage')
|
|
base._require(isinstance(host_stage, str)
|
|
and re.fullmatch(r'/home/ps/\.zyt-dify-stop-20261027-[0-9a-f]{32}', host_stage) is not None,
|
|
'PRIVATE_ROLLOUT_STAGE_INVALID')
|
|
if action == 'deploy':
|
|
base._require(state.get('imports_passed') is True and state.get('built') is True, 'IMPORTS_AND_BUILT_IMAGE_REQUIRED')
|
|
base._require(state.get('preflight_passed') is True
|
|
and state.get('preflight_image_id') == state.get('candidate_image_id')
|
|
and state.get('preflight_compose_fingerprint') == _compose_fingerprint(client, base),
|
|
'INSTALLED_PREFLIGHT_REQUIRED')
|
|
observation = state.get('maintenance_observation',{})
|
|
approval = state.get('maintenance_approval',{})
|
|
base._require(observation.get('observation_only') is True and (
|
|
observation.get('all_samples_zero') is True or (
|
|
approval.get('user_explicitly_approved_one_connection_interruption') is True
|
|
and observation.get('established_inbound_samples') == [1,1,1]
|
|
and approval.get('baseline_container_id') == metadata.get('container_id')
|
|
and approval.get('candidate_image_id') == state.get('candidate_image_id'))),
|
|
'MAINTENANCE_CONNECTION_OBSERVATION_REQUIRED')
|
|
base._require(not state.get('deployment_started'), 'PRIOR_DEPLOY_OUTCOME_REQUIRES_STATUS_REVIEW')
|
|
base._require(current is not None and current['container_id'] == metadata['container_id']
|
|
and current['image_id'] == metadata['image_id'] and current.get('status') == 'running',
|
|
'PRE_DEPLOY_BASELINE_CHANGED')
|
|
image, image_meta = base._image_metadata(client, state['candidate_image_tag'])
|
|
base._require(image['image_id'] == state['candidate_image_id'] and image.get('manifest') == base.MANIFEST_SHA256
|
|
and image.get('base_image') == metadata['image_id'], 'CANDIDATE_IMAGE_PROVENANCE_MISMATCH')
|
|
original, base_meta = base._image_metadata(client, state['base_image_tag'])
|
|
base._require(original['image_id'] == metadata['image_id'], 'ROLLBACK_IMAGE_MISSING_OR_CHANGED')
|
|
_tree_check(client, base, current['container_id'], manifest, 'before_sha256')
|
|
forward = host_stage + '/api-candidate.override.json'
|
|
inverse = host_stage + '/api-baseline.override.json'
|
|
fingerprint = _compose_fingerprint(client, base)
|
|
_write_new_private(client, forward, _immutable_override(state['candidate_image_id']))
|
|
_write_new_private(client, inverse, _immutable_override(metadata['image_id']))
|
|
# This is an exact compose-file backup, not a self-contained .env/env_file backup.
|
|
_copy_compose_private(client, base, host_stage + '/docker-compose.before.private.yml', fingerprint['compose_sha256'])
|
|
state.update(deployment_started=True, forward_override=forward, rollback_override=inverse,
|
|
rollout_compose_fingerprint=fingerprint)
|
|
base._save_state(root, state)
|
|
base._require(_compose_fingerprint(client, base) == fingerprint, 'COMPOSE_INPUTS_CHANGED')
|
|
_verify_override(client, base, forward, state['candidate_image_id'])
|
|
command = _up_command(forward)
|
|
base._exec(client, command, timeout=120, cap=1048576, keep_stdout=False)
|
|
expected = state['candidate_image_id']
|
|
which = 'after_sha256'
|
|
else:
|
|
base._require(action == 'rollback' and state.get('deployment_started') is True, 'PRIOR_DEPLOY_REQUIRED')
|
|
base._require(not state.get('rollback_started'), 'PRIOR_ROLLBACK_OUTCOME_REQUIRES_STATUS_REVIEW')
|
|
# Failed creation may leave no service container; an exited candidate is also recoverable.
|
|
original_still_running = (current is not None and current['container_id'] == metadata['container_id']
|
|
and current['image_id'] == metadata['image_id'] and current.get('status') == 'running'
|
|
and current.get('health') in (None, 'healthy'))
|
|
base._require(current is None or current['image_id'] == state.get('candidate_image_id') or original_still_running,
|
|
'ROLLBACK_CURRENT_IMAGE_CHANGED')
|
|
original, image_meta = base._image_metadata(client, state['base_image_tag'])
|
|
base._require(original['image_id'] == metadata['image_id'], 'ROLLBACK_IMAGE_IDENTITY_MISMATCH')
|
|
override = state['rollback_override']
|
|
base._require(override == state['host_stage'] + '/api-baseline.override.json', 'ROLLBACK_OVERRIDE_PATH_INVALID')
|
|
fingerprint = state.get('rollout_compose_fingerprint')
|
|
base._require(isinstance(fingerprint, dict) and _compose_fingerprint(client, base) == fingerprint,
|
|
'ROLLBACK_COMPOSE_INPUTS_CHANGED_OR_UNRECORDED')
|
|
_verify_override(client, base, override, metadata['image_id'])
|
|
if original_still_running:
|
|
checked = _tree_check(client, base, current['container_id'], manifest, 'before_sha256')
|
|
state.update(rolled_back=True, rollback_container_id=current['container_id'],
|
|
rollback_result='original_baseline_already_running')
|
|
base._save_state(root, state)
|
|
return {'ok': True, 'action': action, 'current': current, 'source_tree': checked,
|
|
'baseline_already_running': True, 'service_recreated': False,
|
|
'model_or_cancel_acceptance_claimed': False}
|
|
if current is not None:
|
|
if current.get('status') == 'running':
|
|
_tree_check(client, base, current['container_id'], manifest, 'after_sha256')
|
|
else:
|
|
_stopped_candidate_tree_check(client, base, current['container_id'], manifest)
|
|
# An exclusive per-action marker prevents concurrent callers from rolling back twice.
|
|
_write_new_private(client, state['host_stage'] + '/rollback.started.private', b'rollback\n')
|
|
state['rollback_started'] = True
|
|
base._save_state(root, state)
|
|
command = _up_command(override)
|
|
base._exec(client, command, timeout=120, cap=1048576, keep_stdout=False)
|
|
expected = metadata['image_id']
|
|
which = 'before_sha256'
|
|
# Basic process/container readiness only; actual model/cancel validation stays separate.
|
|
final = None
|
|
for unused in range(15):
|
|
final = _current(client, base)
|
|
if final is not None and final['image_id'] == expected and final.get('status') == 'running' and final.get('health') in (None, 'healthy'):
|
|
break
|
|
time.sleep(2)
|
|
base._require(final is not None and final['image_id'] == expected and final.get('status') == 'running'
|
|
and final.get('health') in (None, 'healthy'), 'ROLLOUT_READINESS_UNCONFIRMED')
|
|
checked = _tree_check(client, base, final['container_id'], manifest, which)
|
|
base._require(_compose_fingerprint(client, base) == state['rollout_compose_fingerprint'],
|
|
'POST_ROLLOUT_COMPOSE_INPUTS_CHANGED')
|
|
if action == 'deploy':
|
|
state.update(deployed=True, deployed_container_id=final['container_id'])
|
|
else:
|
|
state.update(rolled_back=True, rollback_container_id=final['container_id'])
|
|
base._save_state(root, state)
|
|
return {'ok': True, 'action': action, 'current': final, 'source_tree': checked,
|
|
'service_recreated': SERVICE, 'other_services_recreated': False,
|
|
'direct_database_write_performed_by_helper': False,
|
|
'entrypoint_database_effects_verified': False, 'runtime_environment_baked': False,
|
|
'resolved_configuration_exported': False,
|
|
'model_or_cancel_acceptance_claimed': False}
|
|
except base.OpsError as error:
|
|
return {'ok': False, 'action': safe_action, 'error_code': error.code,
|
|
'diagnostic': getattr(error,'diagnostic',None),
|
|
'raw_remote_output_suppressed': True, 'automatic_retry_performed': False}
|
|
except Exception as error:
|
|
return {'ok': False, 'action': safe_action, 'error_code': 'ROLLOUT_ACTION_FAILED',
|
|
'error_type': type(error).__name__, 'raw_remote_output_suppressed': True,
|
|
'automatic_retry_performed': False}
|