Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,378 @@
|
||||
import hmac
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from process_identity import current_process_identity, verify_retained_process
|
||||
from lifecycle_authority import (
|
||||
LIFECYCLE_PHASES,
|
||||
PHASE_ACTIVATING,
|
||||
LifecycleAuthorityError,
|
||||
build_code_manifest,
|
||||
code_manifest_sha256,
|
||||
normalize_code_manifest,
|
||||
verify_code_manifest,
|
||||
verify_supervisor_command_line,
|
||||
)
|
||||
from runtime_security import (
|
||||
atomic_write_private_json,
|
||||
canonical_path,
|
||||
durable_unlink,
|
||||
PrivateFileLock,
|
||||
read_private_json,
|
||||
sha256_file,
|
||||
write_private_json_exclusive,
|
||||
)
|
||||
|
||||
|
||||
INSTANCE_SCHEMA = 2
|
||||
CONTROL_SCHEMA = 1
|
||||
SHUTDOWN_RECEIPT_SCHEMA = 1
|
||||
|
||||
|
||||
class InstanceMetadataError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class InstanceLockError(OSError):
|
||||
pass
|
||||
|
||||
|
||||
def instance_lock_path(instance_path):
|
||||
return os.path.splitext(os.path.abspath(instance_path))[0] + '.lock'
|
||||
|
||||
|
||||
def shutdown_receipt_path(instance_path):
|
||||
return os.path.splitext(os.path.abspath(instance_path))[0] + '.exit.json'
|
||||
|
||||
|
||||
class SupervisorInstanceLock:
|
||||
"""Lifetime singleton lock keyed by the canonical private instance path."""
|
||||
|
||||
def __init__(self, instance_path, lock_path=None):
|
||||
self.instance_path = canonical_path(instance_path)
|
||||
self.path = os.path.normcase(os.path.abspath(lock_path or instance_lock_path(instance_path)))
|
||||
self._lock = PrivateFileLock(self.path)
|
||||
self._acquired = False
|
||||
|
||||
@property
|
||||
def acquired(self):
|
||||
return self._acquired
|
||||
|
||||
def acquire(self):
|
||||
if self._acquired:
|
||||
return self
|
||||
try:
|
||||
self._lock.acquire()
|
||||
except BlockingIOError as exc:
|
||||
raise InstanceLockError('another supervisor owns this private runtime control lock') from exc
|
||||
except OSError as exc:
|
||||
raise InstanceLockError(str(exc)) from exc
|
||||
self._acquired = True
|
||||
return self
|
||||
|
||||
def release(self):
|
||||
if not self._acquired:
|
||||
return
|
||||
self._acquired = False
|
||||
self._lock.release()
|
||||
|
||||
def __enter__(self):
|
||||
return self.acquire()
|
||||
|
||||
def __exit__(self, exc_type, value, traceback):
|
||||
self.release()
|
||||
|
||||
|
||||
def utc_now_iso():
|
||||
return datetime.now(timezone.utc).isoformat(timespec='seconds')
|
||||
|
||||
|
||||
def is_loopback_host(host):
|
||||
try:
|
||||
return ipaddress.ip_address(str(host)).is_loopback
|
||||
except ValueError:
|
||||
return str(host).strip().lower() == 'localhost'
|
||||
|
||||
|
||||
def build_instance_metadata(
|
||||
launch_nonce,
|
||||
supervisor_path,
|
||||
config_path,
|
||||
control_host,
|
||||
control_port,
|
||||
manages_postgres,
|
||||
identity=None,
|
||||
instance_id=None,
|
||||
token=None,
|
||||
activation_state=PHASE_ACTIVATING,
|
||||
expected_config_sha256=None,
|
||||
expected_supervisor_sha256=None,
|
||||
code_manifest=None,
|
||||
expected_code_manifest_sha256=None,
|
||||
canonical_dsn_sha256='',
|
||||
lifecycle_mode='background',
|
||||
instance_file=None,
|
||||
):
|
||||
if not launch_nonce:
|
||||
raise InstanceMetadataError('launch nonce is required')
|
||||
if not is_loopback_host(control_host):
|
||||
raise InstanceMetadataError('control endpoint must be loopback-only')
|
||||
identity = identity or current_process_identity()
|
||||
supervisor_path = canonical_path(supervisor_path)
|
||||
config_path = canonical_path(config_path)
|
||||
actual_supervisor_sha256 = sha256_file(supervisor_path)
|
||||
actual_config_sha256 = sha256_file(config_path)
|
||||
if expected_supervisor_sha256 and not hmac.compare_digest(actual_supervisor_sha256, str(expected_supervisor_sha256)):
|
||||
raise InstanceMetadataError('supervisor script changed after parent authority capture')
|
||||
if expected_config_sha256 and not hmac.compare_digest(actual_config_sha256, str(expected_config_sha256)):
|
||||
raise InstanceMetadataError('supervisor config changed after parent authority capture')
|
||||
code_manifest = normalize_code_manifest(code_manifest or build_code_manifest())
|
||||
manifest_sha256 = code_manifest_sha256(code_manifest)
|
||||
if expected_code_manifest_sha256 and not hmac.compare_digest(manifest_sha256, str(expected_code_manifest_sha256)):
|
||||
raise InstanceMetadataError('code manifest changed after parent authority capture')
|
||||
lifecycle_mode = str(lifecycle_mode)
|
||||
if lifecycle_mode not in ('background', 'foreground'):
|
||||
raise InstanceMetadataError('invalid supervisor lifecycle mode')
|
||||
return {
|
||||
'schema': INSTANCE_SCHEMA,
|
||||
'instance_id': instance_id or secrets.token_urlsafe(24),
|
||||
'token': token or secrets.token_urlsafe(48),
|
||||
'launch_nonce': str(launch_nonce),
|
||||
'pid': int(identity.pid),
|
||||
'process_creation_time': str(identity.creation_time),
|
||||
'executable': canonical_path(identity.executable),
|
||||
'supervisor_path': supervisor_path,
|
||||
'supervisor_sha256': actual_supervisor_sha256,
|
||||
'config_path': config_path,
|
||||
'config_sha256': actual_config_sha256,
|
||||
'code_manifest': code_manifest,
|
||||
'code_manifest_sha256': manifest_sha256,
|
||||
'canonical_dsn_sha256': str(canonical_dsn_sha256 or ''),
|
||||
'instance_file': canonical_path(instance_file) if instance_file else '',
|
||||
'lifecycle_mode': lifecycle_mode,
|
||||
'control': {'host': str(control_host), 'port': int(control_port)},
|
||||
'startup_time': utc_now_iso(),
|
||||
'manages_postgres': bool(manages_postgres),
|
||||
'activation_state': str(activation_state),
|
||||
'private_file_ready': True,
|
||||
}
|
||||
|
||||
|
||||
def validate_instance_metadata(value):
|
||||
if not isinstance(value, dict) or value.get('schema') != INSTANCE_SCHEMA:
|
||||
raise InstanceMetadataError('unsupported supervisor instance schema')
|
||||
required_strings = (
|
||||
'instance_id', 'token', 'launch_nonce', 'process_creation_time', 'executable',
|
||||
'supervisor_path', 'supervisor_sha256', 'config_path', 'config_sha256', 'startup_time',
|
||||
'code_manifest_sha256', 'lifecycle_mode',
|
||||
)
|
||||
for key in required_strings:
|
||||
if not isinstance(value.get(key), str) or not value[key]:
|
||||
raise InstanceMetadataError(f'invalid supervisor instance field: {key}')
|
||||
for key in ('supervisor_sha256', 'config_sha256', 'code_manifest_sha256'):
|
||||
if not re.fullmatch(r'[0-9a-f]{64}', value[key]):
|
||||
raise InstanceMetadataError(f'invalid supervisor instance hash: {key}')
|
||||
canonical_dsn_sha256 = str(value.get('canonical_dsn_sha256') or '')
|
||||
if canonical_dsn_sha256 and not re.fullmatch(r'[0-9a-f]{64}', canonical_dsn_sha256):
|
||||
raise InstanceMetadataError('invalid supervisor instance DSN authority')
|
||||
try:
|
||||
manifest = normalize_code_manifest(value.get('code_manifest'))
|
||||
except (OSError, ValueError) as exc:
|
||||
raise InstanceMetadataError(str(exc)) from exc
|
||||
if not hmac.compare_digest(code_manifest_sha256(manifest), value['code_manifest_sha256']):
|
||||
raise InstanceMetadataError('supervisor instance code manifest digest mismatch')
|
||||
if len(value['instance_id']) > 256 or len(value['token']) < 32 or len(value['token']) > 512:
|
||||
raise InstanceMetadataError('invalid supervisor instance credentials')
|
||||
try:
|
||||
pid = int(value.get('pid'))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise InstanceMetadataError('invalid supervisor instance PID') from exc
|
||||
if pid <= 0:
|
||||
raise InstanceMetadataError('invalid supervisor instance PID')
|
||||
control = value.get('control')
|
||||
if not isinstance(control, dict) or not is_loopback_host(control.get('host')):
|
||||
raise InstanceMetadataError('invalid supervisor control endpoint')
|
||||
try:
|
||||
port = int(control.get('port'))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise InstanceMetadataError('invalid supervisor control port') from exc
|
||||
if not 0 < port <= 65535:
|
||||
raise InstanceMetadataError('invalid supervisor control port')
|
||||
if value.get('private_file_ready') is not True:
|
||||
raise InstanceMetadataError('supervisor instance is not marked private-file-ready')
|
||||
activation_state = str(value.get('activation_state') or PHASE_ACTIVATING).upper()
|
||||
if activation_state not in LIFECYCLE_PHASES:
|
||||
raise InstanceMetadataError('invalid supervisor activation state')
|
||||
lifecycle_mode = str(value.get('lifecycle_mode') or '')
|
||||
if lifecycle_mode not in ('background', 'foreground'):
|
||||
raise InstanceMetadataError('invalid supervisor lifecycle mode')
|
||||
normalized = dict(value)
|
||||
normalized['pid'] = pid
|
||||
normalized['control'] = {'host': str(control['host']), 'port': port}
|
||||
normalized['executable'] = canonical_path(value['executable'])
|
||||
normalized['supervisor_path'] = canonical_path(value['supervisor_path'])
|
||||
normalized['config_path'] = canonical_path(value['config_path'])
|
||||
normalized['code_manifest'] = manifest
|
||||
normalized['code_manifest_sha256'] = value['code_manifest_sha256']
|
||||
normalized['canonical_dsn_sha256'] = canonical_dsn_sha256
|
||||
normalized['instance_file'] = canonical_path(value['instance_file']) if value.get('instance_file') else ''
|
||||
normalized['lifecycle_mode'] = lifecycle_mode
|
||||
normalized['manages_postgres'] = bool(value.get('manages_postgres'))
|
||||
normalized['activation_state'] = activation_state
|
||||
return normalized
|
||||
|
||||
|
||||
def write_instance_metadata(path, metadata):
|
||||
write_private_json_exclusive(path, validate_instance_metadata(metadata))
|
||||
|
||||
|
||||
def load_instance_metadata(path):
|
||||
return validate_instance_metadata(read_private_json(path))
|
||||
|
||||
|
||||
def update_instance_activation(path, instance_id, activation_state):
|
||||
activation_state = str(activation_state).upper()
|
||||
if activation_state not in LIFECYCLE_PHASES:
|
||||
raise InstanceMetadataError('invalid supervisor activation state')
|
||||
current = load_instance_metadata(path)
|
||||
if not hmac.compare_digest(current['instance_id'], str(instance_id)):
|
||||
raise InstanceMetadataError('supervisor activation instance mismatch')
|
||||
current['activation_state'] = activation_state
|
||||
atomic_write_private_json(path, validate_instance_metadata(current))
|
||||
return current
|
||||
|
||||
|
||||
def remove_instance_if_matches(path, instance_id, instance_lock=None, lock_path=None):
|
||||
owned_lock = None
|
||||
if instance_lock is None:
|
||||
try:
|
||||
owned_lock = SupervisorInstanceLock(path, lock_path=lock_path).acquire()
|
||||
instance_lock = owned_lock
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
current = load_instance_metadata(path)
|
||||
except (OSError, ValueError):
|
||||
if owned_lock:
|
||||
owned_lock.release()
|
||||
return False
|
||||
if not hmac.compare_digest(current['instance_id'], str(instance_id)):
|
||||
if owned_lock:
|
||||
owned_lock.release()
|
||||
return False
|
||||
try:
|
||||
before = os.stat(path, follow_symlinks=False)
|
||||
confirmed = load_instance_metadata(path)
|
||||
after = os.stat(path, follow_symlinks=False)
|
||||
identity_before = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns)
|
||||
identity_after = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns)
|
||||
if identity_before != identity_after or not hmac.compare_digest(confirmed['instance_id'], str(instance_id)):
|
||||
return False
|
||||
durable_unlink(path)
|
||||
return True
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
finally:
|
||||
if owned_lock:
|
||||
owned_lock.release()
|
||||
|
||||
|
||||
def verify_instance_process(
|
||||
metadata,
|
||||
supervisor_path=None,
|
||||
config_path=None,
|
||||
allow_config_drift=False,
|
||||
allow_code_drift=False,
|
||||
):
|
||||
metadata = validate_instance_metadata(metadata)
|
||||
if supervisor_path and metadata['supervisor_path'] != canonical_path(supervisor_path):
|
||||
raise InstanceMetadataError('supervisor path does not match instance metadata')
|
||||
if config_path and metadata['config_path'] != canonical_path(config_path):
|
||||
raise InstanceMetadataError('config path does not match instance metadata')
|
||||
if not allow_code_drift:
|
||||
try:
|
||||
verify_code_manifest(metadata['code_manifest'], metadata['code_manifest_sha256'])
|
||||
if sha256_file(metadata['supervisor_path']) != metadata['supervisor_sha256']:
|
||||
raise InstanceMetadataError('supervisor script hash does not match instance metadata')
|
||||
except OSError as exc:
|
||||
raise InstanceMetadataError(f'unable to recompute supervisor code authority: {exc}') from exc
|
||||
except ValueError as exc:
|
||||
raise InstanceMetadataError(str(exc)) from exc
|
||||
try:
|
||||
config_matches = sha256_file(metadata['config_path']) == metadata['config_sha256']
|
||||
except OSError as exc:
|
||||
if not allow_config_drift:
|
||||
raise InstanceMetadataError(f'unable to recompute supervisor config authority hash: {exc}') from exc
|
||||
config_matches = False
|
||||
if not config_matches and not allow_config_drift:
|
||||
raise InstanceMetadataError('supervisor config hash drifted; only authenticated shutdown is allowed')
|
||||
process = verify_retained_process(
|
||||
metadata['pid'],
|
||||
metadata['process_creation_time'],
|
||||
metadata['executable'],
|
||||
)
|
||||
try:
|
||||
arguments = process.command_line()
|
||||
if metadata['lifecycle_mode'] == 'background' and '--background-child' not in arguments:
|
||||
raise InstanceMetadataError('retained Python process is not a background supervisor child')
|
||||
if metadata['lifecycle_mode'] == 'foreground' and '--background-child' in arguments:
|
||||
raise InstanceMetadataError('retained Python process lifecycle mode mismatch')
|
||||
try:
|
||||
verify_supervisor_command_line(
|
||||
arguments,
|
||||
metadata['supervisor_path'],
|
||||
metadata['config_path'],
|
||||
)
|
||||
except LifecycleAuthorityError as exc:
|
||||
raise InstanceMetadataError(str(exc)) from exc
|
||||
except BaseException:
|
||||
process.close()
|
||||
raise
|
||||
return process
|
||||
|
||||
|
||||
def write_shutdown_receipt(instance_path, instance_id, exit_code):
|
||||
value = {
|
||||
'schema': SHUTDOWN_RECEIPT_SCHEMA,
|
||||
'instance_id': str(instance_id),
|
||||
'exit_code': int(exit_code),
|
||||
'completed_at': utc_now_iso(),
|
||||
}
|
||||
atomic_write_private_json(shutdown_receipt_path(instance_path), value)
|
||||
|
||||
|
||||
def load_shutdown_receipt(instance_path, instance_id):
|
||||
value = read_private_json(shutdown_receipt_path(instance_path))
|
||||
if value.get('schema') != SHUTDOWN_RECEIPT_SCHEMA:
|
||||
raise InstanceMetadataError('unsupported supervisor shutdown receipt schema')
|
||||
if not hmac.compare_digest(str(value.get('instance_id') or ''), str(instance_id)):
|
||||
raise InstanceMetadataError('supervisor shutdown receipt instance mismatch')
|
||||
try:
|
||||
code = int(value.get('exit_code'))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise InstanceMetadataError('invalid supervisor shutdown receipt exit code') from exc
|
||||
return code
|
||||
|
||||
|
||||
def remove_shutdown_receipt(instance_path, instance_id=None):
|
||||
path = shutdown_receipt_path(instance_path)
|
||||
try:
|
||||
if instance_id is not None:
|
||||
load_shutdown_receipt(instance_path, instance_id)
|
||||
durable_unlink(path)
|
||||
return True
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def authenticate_request(request, instance_id, token):
|
||||
if not isinstance(request, dict) or request.get('schema') != CONTROL_SCHEMA:
|
||||
return False
|
||||
request_instance = request.get('instance_id')
|
||||
request_token = request.get('token')
|
||||
if not isinstance(request_instance, str) or not isinstance(request_token, str):
|
||||
return False
|
||||
return hmac.compare_digest(request_instance, str(instance_id)) and hmac.compare_digest(request_token, str(token))
|
||||
Reference in New Issue
Block a user