"""Fixed privileged lifecycle for the managed runtime deployment.""" from dataclasses import dataclass import json import os import re import signal import stat import subprocess import threading import time from host_agent_apply import _new_stopped_runtime_proof from host_agent_state import HostOperationState, HostStateError DOCKER = '/usr/bin/docker' PROJECT = 'truf-docker' PROJECT_DIRECTORY = '/opt/truf' COMPOSE_FILES = ('/opt/truf/compose.yaml', '/opt/truf/compose.edge.yaml') DEPLOYMENT_PROFILE_FILE = '/etc/truf/deployment-profile' STANDALONE_PROFILE_NAME = 'standalone-edge-v1' SHARED_HOST_PROFILE_NAME = 'shared-host-edge-v1' EDGE_ENV_FILE = '/etc/truf-edge/edge.env' RUNTIME_IMAGE = 'truf-local:runtime' EDGE_IMAGE = 'truf-local:edge' RUNTIME_SERVICE = 'runtime' EDGE_SERVICE = 'edge' COMMAND_OUTPUT_LIMIT = 16 * 1024 INSPECT_TIMEOUT = 15.0 EDGE_STOP_TIMEOUT = 45.0 RUNTIME_STOP_TIMEOUT = 660.0 REMOVE_TIMEOUT = 45.0 RECREATE_TIMEOUT = 120.0 RUNTIME_HEALTH_TIMEOUT = 240.0 HEALTH_COMMAND_TIMEOUT = 30.0 EDGE_START_TIMEOUT = 45.0 EDGE_STABILITY_SECONDS = 10.0 EDGE_VERIFY_TIMEOUT = 100.0 POLL_SECONDS = 2.0 _HEX_ID = re.compile(r'[0-9a-f]{64}') _IMAGE_ID = re.compile(r'sha256:[0-9a-f]{64}') _RUNTIME_ENTRYPOINT = ( '/usr/bin/tini', '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B', '/opt/truf/app/container_runtime.py', ) _RUNTIME_HEALTH_TEST = ( 'CMD', '/usr/local/bin/python3', '-I', '-S', '-B', '/opt/truf/app/container_runtime.py', 'health', '--config', '/data/config/config.yaml', ) _RUNTIME_TMPFS = { '/run/truf': 'rw,nosuid,nodev,noexec,size=64m,mode=0700,uid=10001,gid=10001', '/tmp': 'rw,nosuid,nodev,noexec,size=128m,mode=1777', } _EDGE_TMPFS = { '/tmp': 'rw,nosuid,nodev,noexec,size=16m,mode=1777', '/run': 'rw,nosuid,nodev,noexec,size=4m,mode=0700,uid=10001,gid=10001', } _INSPECT_FORMAT = ( '{"id":{{json .Id}},"image":{{json .Image}},' '"status":{{json .State.Status}},"running":{{json .State.Running}},' '"paused":{{json .State.Paused}},' '"restarting":{{json .State.Restarting}},"dead":{{json .State.Dead}},' '"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},' '"oom_killed":{{json .State.OOMKilled}},' '"restarts":{{json .RestartCount}},"user":{{json .Config.User}},' '"entrypoint":{{json .Config.Entrypoint}},' '"command":{{json .Config.Cmd}},' '"stop_timeout":{{json .Config.StopTimeout}},' '"stop_signal":{{with index .Config "StopSignal"}}{{json .}}' '{{else}}""{{end}},' '"mounts":"{{range .Mounts}}{{.Type}}|{{if eq .Type "volume"}}' '{{.Name}}{{end}}|{{.Source}}|' '{{.Destination}}|{{.RW}};{{end}}",' '"readonly":{{json .HostConfig.ReadonlyRootfs}},' '"privileged":{{json .HostConfig.Privileged}},' '"network":{{json .HostConfig.NetworkMode}},' '"pid_mode":{{json .HostConfig.PidMode}},' '"ipc_mode":{{json .HostConfig.IpcMode}},' '"userns_mode":{{json .HostConfig.UsernsMode}},' '"cgroupns_mode":{{json .HostConfig.CgroupnsMode}},' '"uts_mode":{{json .HostConfig.UTSMode}},' '"group_add":{{json .HostConfig.GroupAdd}},' '"oci_runtime":{{json .HostConfig.Runtime}},' '"devices":{{if .HostConfig.Devices}}{{len .HostConfig.Devices}}' '{{else}}0{{end}},' '"device_requests":{{if .HostConfig.DeviceRequests}}' '{{len .HostConfig.DeviceRequests}}{{else}}0{{end}},' '"device_cgroup_rules":{{if .HostConfig.DeviceCgroupRules}}' '{{len .HostConfig.DeviceCgroupRules}}{{else}}0{{end}},' '"ports":{{json .HostConfig.PortBindings}},' '"tmpfs":{{json .HostConfig.Tmpfs}},' '"cpus":{{json .HostConfig.NanoCpus}},' '"memory":{{json .HostConfig.Memory}},' '"pids_limit":{{json .HostConfig.PidsLimit}},' '"shm_size":{{json .HostConfig.ShmSize}},' '"log_config":{{json .HostConfig.LogConfig}},' '"cap_drop":{{json .HostConfig.CapDrop}},' '"cap_add":{{json .HostConfig.CapAdd}},' '"security_opt":{{json .HostConfig.SecurityOpt}},' '"restart_policy":{{json .HostConfig.RestartPolicy}},' '"project":{{json (index .Config.Labels "com.docker.compose.project")}},' '"service":{{json (index .Config.Labels "com.docker.compose.service")}},' '"oneoff":{{json (index .Config.Labels "com.docker.compose.oneoff")}},' '"config_hash":{{json (index .Config.Labels ' '"com.docker.compose.config-hash")}},' '"config_files":{{json (index .Config.Labels ' '"com.docker.compose.project.config_files")}},' '"working_dir":{{json (index .Config.Labels ' '"com.docker.compose.project.working_dir")}},' '"health_test":{{with index .Config "Healthcheck"}}{{json .Test}}' '{{else}}null{{end}},' '"health":{{with index .State "Health"}}{{json .Status}}' '{{else}}null{{end}}}' ) _INSPECT_KEYS = { 'id', 'image', 'status', 'running', 'paused', 'restarting', 'dead', 'pid', 'exit_code', 'oom_killed', 'restarts', 'user', 'entrypoint', 'command', 'stop_timeout', 'stop_signal', 'mounts', 'readonly', 'privileged', 'network', 'pid_mode', 'ipc_mode', 'userns_mode', 'cgroupns_mode', 'uts_mode', 'group_add', 'oci_runtime', 'devices', 'device_requests', 'device_cgroup_rules', 'ports', 'tmpfs', 'cpus', 'memory', 'pids_limit', 'shm_size', 'log_config', 'cap_drop', 'cap_add', 'security_opt', 'restart_policy', 'project', 'service', 'oneoff', 'config_hash', 'config_files', 'working_dir', 'health_test', 'health', } class HostLifecycleError(RuntimeError): def __init__(self, category): super().__init__('host runtime lifecycle failed') self.category = str(category) @dataclass(frozen=True, slots=True) class _DeploymentProfile: name: str compose_files: tuple runtime_network: str runtime_ports: dict runtime_data_volume: str runtime_cpus: int runtime_memory: int edge_cap_add: tuple edge_caddyfile: str STANDALONE_PROFILE = _DeploymentProfile( name=STANDALONE_PROFILE_NAME, compose_files=COMPOSE_FILES, runtime_network=f'{PROJECT}_default', runtime_ports={'443/tcp': [{'HostIp': '', 'HostPort': '443'}]}, runtime_data_volume='truf-docker_data', runtime_cpus=2_000_000_000, runtime_memory=6 * 1024 ** 3, edge_cap_add=('NET_BIND_SERVICE',), edge_caddyfile='/etc/caddy/Caddyfile', ) SHARED_HOST_PROFILE = _DeploymentProfile( name=SHARED_HOST_PROFILE_NAME, compose_files=( '/opt/truf/compose.yaml', '/opt/truf/compose.shared-host.yaml', ), runtime_network='host', runtime_ports={}, runtime_data_volume='truf-remote-server-data', runtime_cpus=900_000_000, runtime_memory=720 * 1024 ** 2, edge_cap_add=(), edge_caddyfile='/etc/caddy/Caddyfile.shared-host', ) def _compose_command(profile): return ( DOCKER, 'compose', '--ansi', 'never', '--project-name', PROJECT, '--env-file', EDGE_ENV_FILE, '--project-directory', PROJECT_DIRECTORY, *(item for path in profile.compose_files for item in ('--file', path)), ) _COMPOSE = _compose_command(STANDALONE_PROFILE) _CONFIG_FILES_LABEL = ','.join(COMPOSE_FILES) def _deployment_profile(path=DEPLOYMENT_PROFILE_FILE): descriptor = None try: descriptor = os.open( path, os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0) | getattr(os, 'O_NOFOLLOW', 0), ) before = os.fstat(descriptor) if ( not stat.S_ISREG(before.st_mode) or before.st_uid != 0 or before.st_gid != 0 or stat.S_IMODE(before.st_mode) != 0o444 or before.st_nlink != 1 or before.st_size > 64 ): raise HostLifecycleError('profile') payload = os.read(descriptor, 65) after = os.fstat(descriptor) if ( len(payload) > 64 or before.st_dev != after.st_dev or before.st_ino != after.st_ino or before.st_mode != after.st_mode or before.st_uid != after.st_uid or before.st_gid != after.st_gid or before.st_nlink != after.st_nlink or before.st_size != after.st_size or before.st_mtime_ns != after.st_mtime_ns ): raise HostLifecycleError('profile') except FileNotFoundError: return STANDALONE_PROFILE except HostLifecycleError: raise except OSError: raise HostLifecycleError('profile') from None finally: if descriptor is not None: os.close(descriptor) try: name = payload.decode('ascii').strip() except UnicodeDecodeError: raise HostLifecycleError('profile') from None profiles = { STANDALONE_PROFILE_NAME: STANDALONE_PROFILE, SHARED_HOST_PROFILE_NAME: SHARED_HOST_PROFILE, } if name not in profiles: raise HostLifecycleError('profile') return profiles[name] @dataclass(frozen=True, slots=True) class _ContainerState: container_id: str image_id: str status: str running: bool paused: bool restarting: bool dead: bool pid: int exit_code: int oom_killed: bool restarts: int user: str entrypoint: object command: object stop_timeout: int stop_signal: str mounts: tuple readonly: bool privileged: bool network: str pid_mode: str ipc_mode: str userns_mode: str cgroupns_mode: str uts_mode: str group_add: tuple oci_runtime: str devices: int device_requests: int device_cgroup_rules: int ports: dict tmpfs: dict cpus: int memory: int pids_limit: int shm_size: int log_config: dict cap_drop: tuple cap_add: tuple security_opt: tuple restart_policy: dict project: str service: str oneoff: str config_hash: str config_files: str working_dir: str health_test: object health: object @dataclass(frozen=True, slots=True) class DeploymentSnapshot: runtime: _ContainerState edge: _ContainerState runtime_image_id: str edge_image_id: str @dataclass(slots=True) class _DeploymentAttempt: snapshot: DeploymentSnapshot token: object phase: str = 'preflight' mutation_started: bool = False edge_stop_issued: bool = False runtime_stop_issued: bool = False forward_runtime_id: str = None forward_edge_id: str = None forward_edge_config_hash: str = None def _subprocess_runner(command, timeout): environment = { 'HOME': '/root', 'LANG': 'C.UTF-8', 'LC_ALL': 'C.UTF-8', 'PATH': '/usr/bin:/bin', } process = None reader = None output_fd = None output_lock = threading.Lock() stop_output = threading.Event() captured = bytearray() overflow = threading.Event() read_failed = threading.Event() def close_output(): nonlocal output_fd with output_lock: descriptor = output_fd output_fd = None if descriptor is not None: try: os.close(descriptor) except OSError: pass def terminate(): if process is None: return try: os.killpg(process.pid, signal.SIGKILL) except OSError: if process.poll() is None: try: process.kill() except OSError: pass if process.poll() is None: try: process.wait(timeout=1) except subprocess.TimeoutExpired: pass def read_output(): try: while not stop_output.is_set(): chunk = os.read(output_fd, 4096) if not chunk: return if len(captured) <= COMMAND_OUTPUT_LIMIT: captured.extend(chunk[:COMMAND_OUTPUT_LIMIT + 1 - len(captured)]) if len(captured) > COMMAND_OUTPUT_LIMIT: overflow.set() terminate() except Exception: if not stop_output.is_set(): read_failed.set() terminate() finally: close_output() try: process = subprocess.Popen( command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, cwd='/', env=environment, close_fds=True, start_new_session=True, ) output_fd = os.dup(process.stdout.fileno()) process.stdout.close() reader = threading.Thread(target=read_output, daemon=True) reader.start() try: return_code = process.wait(timeout=max(0.001, float(timeout))) except subprocess.TimeoutExpired: terminate() raise HostLifecycleError('timeout') from None reader.join(timeout=1) if ( reader.is_alive() or read_failed.is_set() or overflow.is_set() or return_code != 0 ): terminate() raise HostLifecycleError('command') return bytes(captured) except HostLifecycleError: raise except Exception: terminate() raise HostLifecycleError('command') from None finally: if process is not None and ( process.poll() is None or (reader is not None and reader.is_alive()) ): terminate() if reader is not None and reader.is_alive(): stop_output.set() close_output() reader.join(timeout=1) close_output() if ( process is not None and process.stdout is not None and not process.stdout.closed ): process.stdout.close() def _json_payload(payload): try: if not payload or len(payload) > COMMAND_OUTPUT_LIMIT: raise ValueError('bounded JSON required') return json.loads(payload.decode('utf-8')) except (UnicodeDecodeError, ValueError, TypeError): raise HostLifecycleError('evidence') from None def _string_tuple(value): if value is None: return () if not isinstance(value, list) or any(type(item) is not str for item in value): raise HostLifecycleError('evidence') return tuple(value) def _optional_string_tuple(value): return None if value is None else _string_tuple(value) def _mount_tuple(value): if type(value) is not str or len(value) > 4096 or not value.endswith(';'): raise HostLifecycleError('evidence') mounts = [] entries = value.split(';') if entries[-1] != '' or len(entries) > 17: raise HostLifecycleError('evidence') for entry in entries[:-1]: fields = entry.split('|') if len(fields) != 5 or fields[4] not in ('true', 'false'): raise HostLifecycleError('evidence') mount_type, name, source, destination, writable = fields identity = name if mount_type == 'volume' else source if not mount_type or not identity or not destination or any( character in mount_type + identity + destination for character in '\r\n"' ): raise HostLifecycleError('evidence') mounts.append((mount_type, identity, destination, writable == 'true')) return tuple(sorted(mounts)) def _string_map(value): if value is None: return {} if ( not isinstance(value, dict) or len(value) > 16 or any(type(key) is not str or type(item) is not str for key, item in value.items()) ): raise HostLifecycleError('evidence') return dict(value) def _port_map(value): if value is None: return {} if not isinstance(value, dict) or len(value) > 8: raise HostLifecycleError('evidence') result = {} for port, bindings in value.items(): if type(port) is not str or not isinstance(bindings, list) or len(bindings) > 8: raise HostLifecycleError('evidence') normalized = [] for binding in bindings: if ( not isinstance(binding, dict) or set(binding) != {'HostIp', 'HostPort'} or type(binding['HostIp']) is not str or type(binding['HostPort']) is not str ): raise HostLifecycleError('evidence') normalized.append(dict(binding)) result[port] = normalized return result def _container_state(payload): value = _json_payload(payload) try: if not isinstance(value, dict) or set(value) != _INSPECT_KEYS: raise ValueError('inspect shape') for name in ('running', 'paused', 'restarting', 'dead', 'oom_killed', 'readonly', 'privileged'): if type(value[name]) is not bool: raise ValueError('inspect boolean') for name in ( 'pid', 'exit_code', 'restarts', 'stop_timeout', 'devices', 'device_requests', 'device_cgroup_rules', 'cpus', 'memory', 'pids_limit', 'shm_size', ): if type(value[name]) is not int or value[name] < 0: raise ValueError('inspect integer') for name in ( 'id', 'image', 'status', 'user', 'stop_signal', 'network', 'pid_mode', 'ipc_mode', 'userns_mode', 'cgroupns_mode', 'uts_mode', 'oci_runtime', 'project', 'service', 'oneoff', 'config_hash', 'config_files', 'working_dir', ): if type(value[name]) is not str: raise ValueError('inspect string') if value['health'] is not None and type(value['health']) is not str: raise ValueError('inspect health') if not isinstance(value['restart_policy'], dict): raise ValueError('inspect restart policy') if not isinstance(value['log_config'], dict): raise ValueError('inspect log config') return _ContainerState( container_id=value['id'], image_id=value['image'], status=value['status'], running=value['running'], paused=value['paused'], restarting=value['restarting'], dead=value['dead'], pid=value['pid'], exit_code=value['exit_code'], oom_killed=value['oom_killed'], restarts=value['restarts'], user=value['user'], entrypoint=_optional_string_tuple(value['entrypoint']), command=_optional_string_tuple(value['command']), stop_timeout=value['stop_timeout'], stop_signal=value['stop_signal'], mounts=_mount_tuple(value['mounts']), readonly=value['readonly'], privileged=value['privileged'], network=value['network'], pid_mode=value['pid_mode'], ipc_mode=value['ipc_mode'], userns_mode=value['userns_mode'], cgroupns_mode=value['cgroupns_mode'], uts_mode=value['uts_mode'], group_add=_string_tuple(value['group_add']), oci_runtime=value['oci_runtime'], devices=value['devices'], device_requests=value['device_requests'], device_cgroup_rules=value['device_cgroup_rules'], ports=_port_map(value['ports']), tmpfs=_string_map(value['tmpfs']), cpus=value['cpus'], memory=value['memory'], pids_limit=value['pids_limit'], shm_size=value['shm_size'], log_config=dict(value['log_config']), cap_drop=_string_tuple(value['cap_drop']), cap_add=_string_tuple(value['cap_add']), security_opt=_string_tuple(value['security_opt']), restart_policy=dict(value['restart_policy']), project=value['project'], service=value['service'], oneoff=value['oneoff'], config_hash=value['config_hash'], config_files=value['config_files'], working_dir=value['working_dir'], health_test=_optional_string_tuple(value['health_test']), health=value['health'], ) except (KeyError, TypeError, ValueError): raise HostLifecycleError('evidence') from None class FixedDeploymentLifecycle: def __init__(self, *, _runner=None, _clock=None, _sleep=None, _profile=None): self._runner = _runner or _subprocess_runner self._clock = _clock or time.monotonic self._sleep = _sleep or time.sleep self._profile = _profile or _deployment_profile() self._compose_command = _compose_command(self._profile) self._token = object() def _run(self, command, timeout): try: payload = self._runner(tuple(command), float(timeout)) if not isinstance(payload, bytes) or len(payload) > COMMAND_OUTPUT_LIMIT: raise HostLifecycleError('command') return payload except HostLifecycleError: raise except Exception: raise HostLifecycleError('command') from None def _compose(self, *arguments, timeout=INSPECT_TIMEOUT): return self._run((*self._compose_command, *arguments), timeout) def _image_id(self, image): payload = self._run( (DOCKER, 'image', 'inspect', '--format', '{{json .Id}}', image), INSPECT_TIMEOUT, ) value = _json_payload(payload) if type(value) is not str or _IMAGE_ID.fullmatch(value) is None: raise HostLifecycleError('evidence') return value def _resolve(self, service, timeout=INSPECT_TIMEOUT): payload = self._compose( 'ps', '--all', '--quiet', service, timeout=timeout, ) try: lines = payload.decode('ascii').splitlines() except UnicodeDecodeError: raise HostLifecycleError('evidence') from None if len(lines) != 1 or _HEX_ID.fullmatch(lines[0]) is None: raise HostLifecycleError('evidence') return lines[0] def _resolve_optional(self, service, timeout=INSPECT_TIMEOUT): payload = self._compose( 'ps', '--all', '--quiet', service, timeout=timeout, ) try: lines = payload.decode('ascii').splitlines() except UnicodeDecodeError: raise HostLifecycleError('evidence') from None if not lines: return None if len(lines) != 1 or _HEX_ID.fullmatch(lines[0]) is None: raise HostLifecycleError('evidence') return lines[0] def _inspect(self, container_id, timeout=INSPECT_TIMEOUT): if _HEX_ID.fullmatch(container_id) is None: raise HostLifecycleError('evidence') state = _container_state(self._run( (DOCKER, 'container', 'inspect', '--format', _INSPECT_FORMAT, container_id), timeout, )) if state.container_id != container_id: raise HostLifecycleError('identity') return state def _require_metadata( self, state, service, image_id, runtime_id, *, config_hash=None, ): expected_policy = ( {'Name': 'on-failure', 'MaximumRetryCount': 3} if service == RUNTIME_SERVICE else {'Name': 'unless-stopped', 'MaximumRetryCount': 0} ) expected_cap_add = ( () if service == RUNTIME_SERVICE else self._profile.edge_cap_add ) expected_network = ( self._profile.runtime_network if service == RUNTIME_SERVICE else f'container:{runtime_id}' ) runtime = service == RUNTIME_SERVICE expected_mounts = ( { ('volume', self._profile.runtime_data_volume, '/data', True), ('bind', '/etc/truf/runtime', '/data/config', False), ('bind', '/etc/truf/worker-packages', '/data/worker-packages', False), ('bind', '/var/lib/truf/runtime-document-candidates', '/data/runtime-document-candidates', True), ('bind', '/run/truf/host-agent.sock', '/run/truf/host-agent.sock', False), ('bind', '/var/lib/truf/host-agent/results', '/data/host-agent-results', False), ('bind', '/run/truf-postgres', '/run/truf-postgres', True), } if runtime else { ('volume', '/data', True), ('volume', '/config', True), ('bind', '/var/log/caddy', True), ('bind', '/etc/caddy/denylist', False), } ) actual_mounts = ( set(state.mounts) if runtime else { (mount_type, destination, writable) for mount_type, _identity, destination, writable in state.mounts } ) expected_ports = self._profile.runtime_ports if runtime else {} expected_log = { 'Type': 'json-file', 'Config': {'max-size': '16m' if runtime else '8m', 'max-file': '4'}, } if ( _HEX_ID.fullmatch(state.container_id) is None or state.image_id != image_id or _IMAGE_ID.fullmatch(state.image_id) is None or state.user != '10001:10001' or state.readonly is not True or state.privileged is not False or state.project != PROJECT or state.service != service or state.oneoff != 'False' or _HEX_ID.fullmatch(state.config_hash) is None or (config_hash is not None and state.config_hash != config_hash) or state.config_files != ','.join(self._profile.compose_files) or state.working_dir != PROJECT_DIRECTORY or state.network != expected_network or set(state.cap_drop) != {'ALL'} or set(state.cap_add) != set(expected_cap_add) or set(state.security_opt) != {'no-new-privileges:true'} or state.restart_policy != expected_policy or state.entrypoint != ( _RUNTIME_ENTRYPOINT if runtime else ('/usr/local/bin/truf-edge-entrypoint',) ) or state.command != ( ('run', '--config', '/data/config/config.yaml') if runtime else None ) or state.health_test != (_RUNTIME_HEALTH_TEST if runtime else None) or state.stop_timeout != (600 if runtime else 30) or state.stop_signal != ('SIGTERM' if runtime else '') or len(state.mounts) != len(expected_mounts) or actual_mounts != expected_mounts or state.pid_mode != '' or state.ipc_mode != 'private' or state.userns_mode != '' or state.cgroupns_mode != 'private' or state.uts_mode != '' or state.group_add or state.oci_runtime != 'runc' or state.devices != 0 or state.device_requests != 0 or state.device_cgroup_rules != 0 or state.ports != expected_ports or state.tmpfs != (_RUNTIME_TMPFS if runtime else _EDGE_TMPFS) or state.cpus != ( self._profile.runtime_cpus if runtime else 1_000_000_000 ) or state.memory != ( self._profile.runtime_memory if runtime else 256 * 1024 ** 2 ) or state.pids_limit != (512 if runtime else 128) or state.shm_size != (256 * 1024 ** 2 if runtime else 64 * 1024 ** 2) or state.log_config != expected_log ): raise HostLifecycleError('identity') def _require_running( self, state, service, image_id, runtime_id, *, fresh, config_hash=None, ): self._require_metadata( state, service, image_id, runtime_id, config_hash=config_hash, ) if ( state.status != 'running' or state.running is not True or state.paused or state.restarting or state.dead or state.oom_killed or state.pid <= 0 or (fresh and state.restarts != 0) ): raise HostLifecycleError('health') def _require_stopped(self, current, previous, service, runtime_id): self._require_metadata( current, service, previous.image_id, runtime_id, config_hash=previous.config_hash, ) if ( current.status != 'exited' or current.running or current.paused or current.restarting or current.dead or current.oom_killed or current.pid != 0 or current.exit_code != 0 or current.restarts != previous.restarts ): raise HostLifecycleError('stop') def preflight(self): self._compose('config', '--quiet') runtime_image = self._image_id(RUNTIME_IMAGE) edge_image = self._image_id(EDGE_IMAGE) runtime_id = self._resolve(RUNTIME_SERVICE) edge_id = self._resolve(EDGE_SERVICE) runtime = self._inspect(runtime_id) edge = self._inspect(edge_id) self._require_running( runtime, RUNTIME_SERVICE, runtime_image, runtime_id, fresh=False, ) self._require_running( edge, EDGE_SERVICE, edge_image, runtime_id, fresh=False, ) return DeploymentSnapshot(runtime, edge, runtime_image, edge_image) def begin_attempt(self, snapshot): if not isinstance(snapshot, DeploymentSnapshot): raise HostLifecycleError('state') return _DeploymentAttempt(snapshot, self._token) def _require_attempt(self, attempt): if ( not isinstance(attempt, _DeploymentAttempt) or attempt.token is not self._token ): raise HostLifecycleError('state') return attempt def stop_cleanly(self, attempt): attempt = self._require_attempt(attempt) snapshot = attempt.snapshot attempt.mutation_started = True attempt.edge_stop_issued = True attempt.phase = 'stopping-edge' self._compose( 'stop', '--timeout', '30', EDGE_SERVICE, timeout=EDGE_STOP_TIMEOUT, ) stopped_edge = self._inspect(snapshot.edge.container_id) self._require_stopped( stopped_edge, snapshot.edge, EDGE_SERVICE, snapshot.runtime.container_id, ) attempt.runtime_stop_issued = True attempt.phase = 'stopping-runtime' self._compose( 'stop', '--timeout', '600', RUNTIME_SERVICE, timeout=RUNTIME_STOP_TIMEOUT, ) stopped_runtime = self._inspect(snapshot.runtime.container_id) self._require_stopped( stopped_runtime, snapshot.runtime, RUNTIME_SERVICE, snapshot.runtime.container_id, ) attempt.phase = 'stopped' return attempt def authorize_replacement(self, attempt, operation_id): attempt = self._require_attempt(attempt) if attempt.phase != 'stopped': raise HostLifecycleError('state') snapshot = attempt.snapshot runtime_id = snapshot.runtime.container_id if ( self._resolve(RUNTIME_SERVICE) != runtime_id or self._resolve(EDGE_SERVICE) != snapshot.edge.container_id ): raise HostLifecycleError('identity') self._require_stopped( self._inspect(snapshot.edge.container_id), snapshot.edge, EDGE_SERVICE, runtime_id, ) self._require_stopped( self._inspect(runtime_id), snapshot.runtime, RUNTIME_SERVICE, runtime_id, ) if ( self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id ): raise HostLifecycleError('identity') return _new_stopped_runtime_proof(operation_id, purpose='forward') def _remaining(self, deadline, maximum): remaining = deadline - self._clock() if remaining <= 0: raise HostLifecycleError('health') return min(float(maximum), remaining) def _strict_runtime_health(self, timeout): try: payload = self._compose( 'exec', '-T', '--user', '10001:10001', RUNTIME_SERVICE, '/usr/local/bin/python3', '-I', '-S', '-B', '/opt/truf/app/container_runtime.py', 'health', '--config', '/data/config/config.yaml', '--require-worker-api', '--require-discovery-producers', timeout=min(HEALTH_COMMAND_TIMEOUT, timeout), ) value = _json_payload(payload) if ( not isinstance(value, dict) or set(value) != { 'healthy', 'activation_state', 'postgres', 'workers', } or value.get('healthy') is not True or value.get('activation_state') != 'ACTIVE' or value.get('postgres') != 'READY' or not isinstance(value.get('workers'), list) or any(type(name) is not str for name in value['workers']) or not { 'worker-api', 'result-ingester', 'jsonl-projector', }.issubset(value['workers']) ): raise HostLifecycleError('health') except HostLifecycleError as error: if error.category == 'health': raise raise HostLifecycleError('health') from None def _wait_runtime(self, runtime_id, image_id, config_hash): deadline = self._clock() + RUNTIME_HEALTH_TIMEOUT while True: if self._resolve( RUNTIME_SERVICE, self._remaining(deadline, INSPECT_TIMEOUT), ) != runtime_id: raise HostLifecycleError('identity') current = self._inspect( runtime_id, self._remaining(deadline, INSPECT_TIMEOUT), ) self._require_running( current, RUNTIME_SERVICE, image_id, runtime_id, fresh=True, config_hash=config_hash, ) if current.health == 'healthy': try: self._strict_runtime_health( self._remaining(deadline, HEALTH_COMMAND_TIMEOUT), ) except HostLifecycleError as error: if error.category != 'health': raise self._sleep(min( POLL_SECONDS, self._remaining(deadline, POLL_SECONDS), )) continue else: return if current.health not in ('starting', 'unhealthy'): raise HostLifecycleError('health') self._sleep(min(POLL_SECONDS, self._remaining(deadline, POLL_SECONDS))) def _wait_edge(self, edge_id, edge_image_id, runtime_id, config_hash): deadline = self._clock() + EDGE_VERIFY_TIMEOUT running_deadline = self._clock() + EDGE_START_TIMEOUT while True: if self._resolve( EDGE_SERVICE, self._remaining(running_deadline, INSPECT_TIMEOUT), ) != edge_id: raise HostLifecycleError('identity') current = self._inspect( edge_id, self._remaining(running_deadline, INSPECT_TIMEOUT), ) self._require_metadata( current, EDGE_SERVICE, edge_image_id, runtime_id, config_hash=config_hash, ) if ( current.status == 'running' and current.running is True and current.paused is False and current.restarting is False and current.dead is False and current.oom_killed is False and current.pid > 0 and current.restarts == 0 ): break if ( current.restarting or current.dead or current.oom_killed or current.status == 'exited' ): raise HostLifecycleError('health') self._sleep(min( POLL_SECONDS, self._remaining(running_deadline, POLL_SECONDS), )) self._compose( 'exec', '-T', '--user', '10001:10001', EDGE_SERVICE, 'caddy', 'validate', '--config', self._profile.edge_caddyfile, '--adapter', 'caddyfile', timeout=self._remaining(deadline, EDGE_START_TIMEOUT), ) stable_until = self._clock() + EDGE_STABILITY_SECONDS if stable_until > deadline: raise HostLifecycleError('health') while self._clock() < stable_until: self._sleep(min(POLL_SECONDS, stable_until - self._clock())) if self._resolve( EDGE_SERVICE, self._remaining(deadline, INSPECT_TIMEOUT), ) != edge_id: raise HostLifecycleError('identity') current = self._inspect( edge_id, self._remaining(deadline, INSPECT_TIMEOUT), ) self._require_running( current, EDGE_SERVICE, edge_image_id, runtime_id, fresh=True, config_hash=config_hash, ) def recreate_and_verify(self, attempt): attempt = self._require_attempt(attempt) snapshot = attempt.snapshot runtime_id = snapshot.runtime.container_id self._require_stopped( self._inspect(snapshot.edge.container_id), snapshot.edge, EDGE_SERVICE, runtime_id, ) self._require_stopped( self._inspect(runtime_id), snapshot.runtime, RUNTIME_SERVICE, runtime_id, ) if ( self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id ): raise HostLifecycleError('identity') self._compose('rm', '--force', EDGE_SERVICE, timeout=REMOVE_TIMEOUT) self._compose('rm', '--force', RUNTIME_SERVICE, timeout=REMOVE_TIMEOUT) attempt.phase = 'forward-removed' if self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id: raise HostLifecycleError('identity') self._compose( 'up', '--detach', '--no-deps', '--no-build', '--pull', 'never', '--force-recreate', RUNTIME_SERVICE, timeout=RECREATE_TIMEOUT, ) new_runtime_id = self._resolve(RUNTIME_SERVICE) attempt.forward_runtime_id = new_runtime_id if new_runtime_id == runtime_id: raise HostLifecycleError('identity') new_runtime = self._inspect(new_runtime_id) self._require_running( new_runtime, RUNTIME_SERVICE, snapshot.runtime_image_id, new_runtime_id, fresh=True, config_hash=snapshot.runtime.config_hash, ) self._wait_runtime( new_runtime_id, snapshot.runtime_image_id, snapshot.runtime.config_hash, ) if self._image_id(EDGE_IMAGE) != snapshot.edge_image_id: raise HostLifecycleError('identity') self._compose( 'up', '--detach', '--no-deps', '--no-build', '--pull', 'never', '--force-recreate', EDGE_SERVICE, timeout=RECREATE_TIMEOUT, ) new_edge_id = self._resolve(EDGE_SERVICE) attempt.forward_edge_id = new_edge_id if new_edge_id == snapshot.edge.container_id: raise HostLifecycleError('identity') new_edge = self._inspect(new_edge_id) self._require_metadata( new_edge, EDGE_SERVICE, snapshot.edge_image_id, new_runtime_id, ) attempt.forward_edge_config_hash = new_edge.config_hash self._wait_edge( new_edge_id, snapshot.edge_image_id, new_runtime_id, attempt.forward_edge_config_hash, ) attempt.phase = 'forward-healthy' @staticmethod def _attempt_edge_config_hash(attempt, edge_id): if edge_id == attempt.snapshot.edge.container_id: return attempt.snapshot.edge.config_hash if ( edge_id == attempt.forward_edge_id and _HEX_ID.fullmatch(attempt.forward_edge_config_hash or '') ): return attempt.forward_edge_config_hash raise HostLifecycleError('rollback') def _require_quiescent(self, current, service, image_id, runtime_id, config_hash): self._require_metadata( current, service, image_id, runtime_id, config_hash=config_hash, ) if ( current.status != 'exited' or current.running or current.paused or current.restarting or current.dead or current.pid != 0 ): raise HostLifecycleError('rollback') def _quiesce_service( self, service, current_id, image_id, runtime_id, config_hash, timeout, grace, ): if current_id is None: return current = self._inspect(current_id) self._require_metadata( current, service, image_id, runtime_id, config_hash=config_hash, ) if current.running: try: self._compose( 'stop', '--timeout', str(grace), service, timeout=timeout, ) except HostLifecycleError: pass if self._resolve_optional(service) != current_id: raise HostLifecycleError('rollback') current = self._inspect(current_id) self._require_quiescent( current, service, image_id, runtime_id, config_hash, ) def quiesce_for_rollback(self, attempt, operation_id): attempt = self._require_attempt(attempt) snapshot = attempt.snapshot runtime_id = self._resolve_optional(RUNTIME_SERVICE) edge_id = self._resolve_optional(EDGE_SERVICE) if edge_id is not None and runtime_id is None: raise HostLifecycleError('rollback') edge_config_hash = ( self._attempt_edge_config_hash(attempt, edge_id) if edge_id is not None else None ) self._quiesce_service( EDGE_SERVICE, edge_id, snapshot.edge_image_id, runtime_id, edge_config_hash, EDGE_STOP_TIMEOUT, 30, ) self._quiesce_service( RUNTIME_SERVICE, runtime_id, snapshot.runtime_image_id, runtime_id, snapshot.runtime.config_hash, RUNTIME_STOP_TIMEOUT, 600, ) if ( self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id or self._image_id(EDGE_IMAGE) != snapshot.edge_image_id ): raise HostLifecycleError('rollback') attempt.phase = 'rollback-quiescent' return _new_stopped_runtime_proof(operation_id, purpose='rollback') def recreate_restored_and_verify(self, attempt): attempt = self._require_attempt(attempt) if attempt.phase != 'rollback-quiescent': raise HostLifecycleError('state') snapshot = attempt.snapshot edge_id = self._resolve_optional(EDGE_SERVICE) runtime_id = self._resolve_optional(RUNTIME_SERVICE) if edge_id is not None: self._require_quiescent( self._inspect(edge_id), EDGE_SERVICE, snapshot.edge_image_id, runtime_id, self._attempt_edge_config_hash(attempt, edge_id), ) self._compose('rm', '--force', EDGE_SERVICE, timeout=REMOVE_TIMEOUT) if runtime_id is not None: self._require_quiescent( self._inspect(runtime_id), RUNTIME_SERVICE, snapshot.runtime_image_id, runtime_id, snapshot.runtime.config_hash, ) self._compose('rm', '--force', RUNTIME_SERVICE, timeout=REMOVE_TIMEOUT) if self._image_id(RUNTIME_IMAGE) != snapshot.runtime_image_id: raise HostLifecycleError('rollback') self._compose( 'up', '--detach', '--no-deps', '--no-build', '--pull', 'never', '--force-recreate', RUNTIME_SERVICE, timeout=RECREATE_TIMEOUT, ) restored_runtime_id = self._resolve(RUNTIME_SERVICE) if restored_runtime_id in { snapshot.runtime.container_id, attempt.forward_runtime_id, }: raise HostLifecycleError('rollback') restored_runtime = self._inspect(restored_runtime_id) self._require_running( restored_runtime, RUNTIME_SERVICE, snapshot.runtime_image_id, restored_runtime_id, fresh=True, config_hash=snapshot.runtime.config_hash, ) self._wait_runtime( restored_runtime_id, snapshot.runtime_image_id, snapshot.runtime.config_hash, ) if self._image_id(EDGE_IMAGE) != snapshot.edge_image_id: raise HostLifecycleError('rollback') self._compose( 'up', '--detach', '--no-deps', '--no-build', '--pull', 'never', '--force-recreate', EDGE_SERVICE, timeout=RECREATE_TIMEOUT, ) restored_edge_id = self._resolve(EDGE_SERVICE) if restored_edge_id in {snapshot.edge.container_id, attempt.forward_edge_id}: raise HostLifecycleError('rollback') restored_edge = self._inspect(restored_edge_id) self._require_metadata( restored_edge, EDGE_SERVICE, snapshot.edge_image_id, restored_runtime_id, ) self._wait_edge( restored_edge_id, snapshot.edge_image_id, restored_runtime_id, restored_edge.config_hash, ) attempt.phase = 'rollback-healthy' def contain_for_failed_hold(self, attempt, operation_id): try: self.quiesce_for_rollback(attempt, operation_id) attempt.phase = 'failed-hold-contained' return True except Exception: attempt.phase = 'failed-hold-uncontained' return False def execute_fixed_forward(session, lifecycle=None): if getattr(session, '_entered', False) is not True: raise HostLifecycleError('state') lifecycle = lifecycle or FixedDeploymentLifecycle() session.backup() snapshot = lifecycle.preflight() attempt = lifecycle.begin_attempt(snapshot) session.revalidate_for_stop() lifecycle.stop_cleanly(attempt) proof = lifecycle.authorize_replacement( attempt, session.request.operation_id, ) resulting_identity = session.replace(proof) lifecycle.recreate_and_verify(attempt) return resulting_identity def _forward_failure(request, error): category = getattr(error, 'category', '') if category == 'health': return 'health_check_failed', 'health_check_failed' if request.action.value == 'restart': return 'restart_failed', 'restart_failed' return 'apply_failed', 'apply_failed' def _preflight_failure(error): category = getattr(error, 'category', '') if category == 'validation': return 'validation_failed', 'validation_failed' if category in ('identity', 'partial'): return 'operation_conflict', 'bounded_result' return 'apply_failed', 'apply_failed' def _terminal_result_for_phase(session, phase): name = phase['phase'] if name == 'succeeded': identity = { 'active_config_sha256': ( session.request.candidate_config_sha256 or session.request.active_config_sha256 ), 'active_secrets_sha256': ( session.request.candidate_secrets_sha256 or session.request.active_secrets_sha256 ), } return name, None, None, identity if name in {'failed', 'rolled_back'}: category = phase.get('forward_category') or 'apply_failed' detail = phase.get('safe_detail') or category return name, category, detail, session.original_identity() if name == 'failed_hold': return name, 'rollback_failed', 'rollback_failed', None raise HostLifecycleError('state') def execute_fixed_operation(session, lifecycle=None, state=None): """Run one forward attempt and at most one fixed rollback attempt.""" if getattr(session, '_entered', False) is not True: raise HostLifecycleError('state') lifecycle = lifecycle or FixedDeploymentLifecycle() state = state or HostOperationState(session.request) try: terminal = state.terminal_result() if terminal is not None: return terminal phase = state.initialize(session.publication_state) except HostStateError: raise HostLifecycleError('state') from None if getattr(session, '_failed_hold_replay', False): if phase['phase'] == 'forward_started': try: phase = state.advance( 'forward_started', 'rollback_started', phase['publication_state'], forward_category=phase.get('forward_category'), safe_detail=phase.get('safe_detail'), ) except HostStateError: raise HostLifecycleError('state') from None if phase['phase'] == 'rollback_started': try: phase = state.advance( 'rollback_started', 'failed_hold', phase['publication_state'], forward_category=phase.get('forward_category'), safe_detail='rollback_failed', containment_confirmed=False, ) except HostStateError: raise HostLifecycleError('state') from None elif phase['phase'] != 'failed_hold': raise HostLifecycleError('state') if phase['phase'] in {'succeeded', 'failed', 'rolled_back', 'failed_hold'}: result, category, detail, identity = _terminal_result_for_phase( session, phase, ) try: return state.publish_result( result, safe_category=category, safe_detail=detail, resulting_identity=identity, ) except HostStateError: raise HostLifecycleError('state') from None current_phase = phase['phase'] attempt = None cancellation = None rollback_required = current_phase in {'forward_started', 'rollback_started'} rollback_required = rollback_required or session.publication_state != 'original' forward_category = phase.get('forward_category') if rollback_required and forward_category is None: forward_category = ( 'restart_failed' if session.request.action.value == 'restart' else 'apply_failed' ) if not rollback_required: try: session.backup() snapshot = lifecycle.preflight() attempt = lifecycle.begin_attempt(snapshot) session.revalidate_for_stop() state.advance( 'prepared', 'forward_started', session.publication_state, ) current_phase = 'forward_started' lifecycle.stop_cleanly(attempt) proof = lifecycle.authorize_replacement( attempt, session.request.operation_id, ) resulting_identity = session.replace(proof) lifecycle.recreate_and_verify(attempt) state.advance( 'forward_started', 'succeeded', session.publication_state, ) current_phase = 'succeeded' result = state.publish_result( 'succeeded', safe_category=None, safe_detail=None, resulting_identity=resulting_identity, ) return result except BaseException as error: is_cancellation = not isinstance(error, Exception) if current_phase == 'succeeded': if is_cancellation: raise raise HostLifecycleError('state') from None if ( isinstance(error, HostStateError) and error.category == 'uncertain' ): if error.cancellation is not None: raise error.cancellation raise HostLifecycleError('state') from None if attempt is None or not attempt.mutation_started: if is_cancellation: raise category, detail = _preflight_failure(error) try: state.advance( 'prepared', 'failed', session.publication_state, forward_category=category, safe_detail=detail, ) current_phase = 'failed' result = state.publish_result( 'failed', safe_category=category, safe_detail=detail, resulting_identity=session.original_identity(), ) return result except HostStateError: raise HostLifecycleError('state') from None rollback_required = True if is_cancellation: cancellation = error forward_category, _ = _forward_failure(session.request, error) if attempt is None: try: session.backup() snapshot = lifecycle.preflight() attempt = lifecycle.begin_attempt(snapshot) except BaseException as error: if not isinstance(error, Exception) and not rollback_required: raise category, detail = _preflight_failure(error) try: if current_phase != 'rollback_started': state.advance( current_phase, 'rollback_started', session.publication_state, forward_category=forward_category or category, safe_detail=detail, ) current_phase = 'rollback_started' state.publish_failed_hold( forward_category=forward_category or category, publication_state=session.publication_state, containment_confirmed=False, ) state.advance( 'rollback_started', 'failed_hold', session.publication_state, forward_category=forward_category or category, safe_detail='rollback_failed', containment_confirmed=False, ) current_phase = 'failed_hold' result = state.publish_result( 'failed_hold', safe_category='rollback_failed', safe_detail='rollback_failed', resulting_identity=None, ) if not isinstance(error, Exception): raise error return result except HostStateError: if not isinstance(error, Exception): raise error raise HostLifecycleError('state') from None rollback_terminal_transition = False try: if current_phase != 'rollback_started': state.advance( current_phase, 'rollback_started', session.publication_state, forward_category=forward_category, safe_detail=forward_category, ) current_phase = 'rollback_started' proof = lifecycle.quiesce_for_rollback( attempt, session.request.operation_id, ) original_identity = session.restore_backups(proof) lifecycle.recreate_restored_and_verify(attempt) detail = forward_category rollback_terminal_transition = True state.advance( 'rollback_started', 'rolled_back', session.publication_state, forward_category=forward_category, safe_detail=detail, ) current_phase = 'rolled_back' result = state.publish_result( 'rolled_back', safe_category=forward_category, safe_detail=detail, resulting_identity=original_identity, ) if cancellation is not None: raise cancellation return result except BaseException as error: if current_phase == 'rolled_back': if cancellation is not None: raise cancellation if not isinstance(error, Exception): raise raise HostLifecycleError('state') from None if ( isinstance(error, HostStateError) and error.category == 'uncertain' and rollback_terminal_transition ): if error.cancellation is not None: raise error.cancellation raise HostLifecycleError('state') from None if ( isinstance(error, HostStateError) and error.category == 'uncertain' and error.cancellation is not None ): cancellation = error.cancellation if not isinstance(error, Exception): cancellation = error try: state.publish_failed_hold( forward_category=forward_category, publication_state=session.publication_state, containment_confirmed=False, ) except HostStateError: if cancellation is not None: raise cancellation raise HostLifecycleError('state') from None contained = lifecycle.contain_for_failed_hold( attempt, session.request.operation_id, ) try: state.advance( 'rollback_started', 'failed_hold', session.publication_state, forward_category=forward_category, safe_detail='rollback_failed', containment_confirmed=contained, ) current_phase = 'failed_hold' result = state.publish_result( 'failed_hold', safe_category='rollback_failed', safe_detail='rollback_failed', resulting_identity=None, ) if cancellation is not None: raise cancellation return result except HostStateError: if cancellation is not None: raise cancellation raise HostLifecycleError('state') from None