sashatrask
457 lines
20 KiB
Python
457 lines
20 KiB
Python
import os
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
|
|
from test_supervisor_foreground_shutdown import (
|
|
isolated_runtime,
|
|
lifecycle_context,
|
|
runtime,
|
|
)
|
|
import process_identity
|
|
import supervisor
|
|
|
|
|
|
ManagedSource = supervisor.ManagedSource
|
|
BoundedRotatingLogPump = supervisor.BoundedRotatingLogPump
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def no_startup_io(monkeypatch, isolated_runtime):
|
|
blocked = mock.Mock(side_effect=AssertionError('unexpected startup I/O'))
|
|
for name in (
|
|
'exact_process_identity_state', 'process_running_status', 'read_private_json',
|
|
'durable_unlink', 'verify_instance_process', 'send_control_request',
|
|
):
|
|
monkeypatch.setattr(supervisor, name, blocked)
|
|
for name in ('require_private_directory', 'append_bounded_log_record', 'finalize_source_runs'):
|
|
monkeypatch.setattr(supervisor, name, mock.Mock())
|
|
monkeypatch.setattr(supervisor, 'BoundedRotatingLogPump', blocked)
|
|
monkeypatch.setattr(process_identity, 'open_process', blocked)
|
|
|
|
|
|
@pytest.fixture
|
|
def launch(monkeypatch, tmp_path):
|
|
source = ManagedSource(
|
|
'fixture', str(tmp_path / 'config.yaml'), str(tmp_path), str(tmp_path),
|
|
{'log_dir': str(tmp_path / 'logs'), 'state_dir': str(tmp_path / 'state')},
|
|
{'enabled': True},
|
|
)
|
|
source.build_command = mock.Mock(return_value=['mock-child'])
|
|
source.build_env = mock.Mock(return_value={})
|
|
process = mock.Mock(pid=41001, stdout=mock.sentinel.stream)
|
|
process.poll.return_value = None
|
|
process.wait.side_effect = supervisor.subprocess.TimeoutExpired('mock-child', 5)
|
|
pump = mock.Mock(thread=None, handle=mock.sentinel.log_handle, error='')
|
|
pump.attach.side_effect = OSError('log attachment failed')
|
|
factory = mock.Mock(return_value=process)
|
|
monkeypatch.setattr(supervisor, 'OwnedProcess', factory)
|
|
monkeypatch.setattr(supervisor, 'BoundedRotatingLogPump', mock.Mock(return_value=pump))
|
|
retained = mock.MagicMock()
|
|
retained.__enter__.return_value.identity = mock.sentinel.payload_identity
|
|
monkeypatch.setattr(supervisor, 'open_process', mock.Mock(return_value=retained))
|
|
|
|
def confirm_exit(*args, **kwargs):
|
|
process.poll.return_value = 0
|
|
return 0
|
|
|
|
return SimpleNamespace(source=source, process=process, pump=pump, factory=factory, confirm_exit=confirm_exit)
|
|
|
|
|
|
@pytest.mark.parametrize('failure', ['terminate', 'timeout', 'unconfirmed_wait', 'poll'])
|
|
def test_unconfirmed_launch_rollback_retains_owner_until_stop_retry(launch, failure):
|
|
h = launch
|
|
if failure == 'terminate':
|
|
h.process.terminate.side_effect = OSError('stop unavailable')
|
|
elif failure == 'unconfirmed_wait':
|
|
h.process.wait.side_effect = None
|
|
h.process.wait.return_value = 0
|
|
elif failure == 'poll':
|
|
h.process.poll.side_effect = OSError('exit observation unavailable')
|
|
|
|
assert not h.source.start(force=True)
|
|
assert h.source.process is h.process
|
|
assert h.source.payload_identity is mock.sentinel.payload_identity
|
|
assert h.source.startup_cleanup_pending is True
|
|
assert h.source.log_pump is h.pump
|
|
assert h.source.started_at is None
|
|
assert h.source.status == 'failed'
|
|
assert 'unconfirmed' in h.source.last_action_error
|
|
h.pump.join.assert_not_called()
|
|
h.process.kill.assert_not_called()
|
|
supervisor.finalize_source_runs.assert_not_called()
|
|
|
|
h.process.terminate.side_effect = None
|
|
h.process.poll.side_effect = None
|
|
h.process.wait.side_effect = h.confirm_exit
|
|
assert h.source.stop(final=True)
|
|
assert h.source.process is None
|
|
assert h.source.payload_identity is None
|
|
assert h.source.startup_cleanup_pending is False
|
|
assert h.source.log_pump is None
|
|
assert not h.source.is_running()
|
|
h.factory.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize('exit_observation', [None, 0, OSError('observer unavailable')])
|
|
def test_retained_rollback_refuses_start_schedule_and_poll(launch, exit_observation):
|
|
h = launch
|
|
assert not h.source.start(force=True)
|
|
h.process.poll.reset_mock()
|
|
if isinstance(exit_observation, BaseException):
|
|
h.process.poll.side_effect = exit_observation
|
|
else:
|
|
h.process.poll.return_value = exit_observation
|
|
for force in (False, True):
|
|
assert not h.source.start(force=force)
|
|
assert not h.source.resume()
|
|
h.source.schedule_start()
|
|
h.source.poll()
|
|
assert h.source.status == 'failed'
|
|
assert h.source.process is h.process
|
|
assert h.source.payload_identity is mock.sentinel.payload_identity
|
|
assert h.source.startup_cleanup_pending is True
|
|
h.process.poll.assert_not_called()
|
|
h.factory.assert_called_once()
|
|
supervisor.BoundedRotatingLogPump.assert_called_once()
|
|
if exit_observation == 0:
|
|
assert h.source.stop(final=True)
|
|
assert h.source.process is None
|
|
assert h.source.payload_identity is None
|
|
assert h.source.startup_cleanup_pending is False
|
|
|
|
|
|
@pytest.mark.parametrize('already_exited', [False, True])
|
|
def test_confirmed_launch_rollback_clears_owner_and_allows_explicit_retry(launch, already_exited):
|
|
h = launch
|
|
if already_exited:
|
|
h.process.poll.return_value = 0
|
|
else:
|
|
h.process.wait.side_effect = h.confirm_exit
|
|
assert not h.source.start(force=True)
|
|
assert h.source.process is None
|
|
assert h.source.payload_identity is None
|
|
assert h.source.startup_cleanup_pending is False
|
|
assert h.source.log_pump is None
|
|
assert h.source.status == 'failed'
|
|
if already_exited:
|
|
h.process.terminate.assert_not_called()
|
|
h.process.poll.return_value = None
|
|
h.pump.attach.side_effect = None
|
|
assert h.source.start(force=True)
|
|
assert h.factory.call_count == 2
|
|
|
|
|
|
def test_constructor_failure_without_returned_owner_still_closes_log(launch):
|
|
launch.factory.side_effect = RuntimeError('construction failed')
|
|
assert not launch.source.start(force=True)
|
|
assert launch.source.process is None
|
|
assert launch.source.payload_identity is None
|
|
assert launch.source.startup_cleanup_pending is False
|
|
assert launch.source.log_pump is None
|
|
launch.pump.join.assert_called_once_with(timeout=1)
|
|
launch.process.terminate.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize('stage', ['attach', 'terminate', 'wait', 'poll'])
|
|
@pytest.mark.parametrize('error', [KeyboardInterrupt(), SystemExit(7)])
|
|
def test_rollback_interruption_propagates_without_losing_owner(launch, stage, error):
|
|
h = launch
|
|
target = h.pump.attach if stage == 'attach' else getattr(h.process, stage)
|
|
target.side_effect = error
|
|
with pytest.raises(type(error)):
|
|
h.source.start(force=True)
|
|
assert h.source.process is h.process
|
|
assert h.source.payload_identity is mock.sentinel.payload_identity
|
|
assert h.source.startup_cleanup_pending is True
|
|
assert h.source.status == 'failed'
|
|
assert h.source.log_pump is h.pump
|
|
h.process.terminate.side_effect = None
|
|
h.process.poll.side_effect = None
|
|
h.process.wait.side_effect = h.confirm_exit
|
|
assert h.source.stop(final=True)
|
|
assert h.source.startup_cleanup_pending is False
|
|
|
|
|
|
@pytest.mark.parametrize('error', [OSError('log cleanup failed'), KeyboardInterrupt(), SystemExit(7)])
|
|
def test_log_cleanup_error_does_not_undo_confirmed_exit(launch, error):
|
|
h = launch
|
|
h.process.wait.side_effect = h.confirm_exit
|
|
h.pump.join.side_effect = error
|
|
if isinstance(error, Exception):
|
|
assert not h.source.start(force=True)
|
|
else:
|
|
with pytest.raises(type(error)):
|
|
h.source.start(force=True)
|
|
assert h.source.process is None
|
|
assert h.source.payload_identity is None
|
|
assert h.source.startup_cleanup_pending is False
|
|
h.pump.join.side_effect = None
|
|
assert h.source.stop(final=True)
|
|
|
|
|
|
def test_log_pump_with_failed_thread_start_can_close_on_rollback_retry():
|
|
pump = object.__new__(BoundedRotatingLogPump)
|
|
pump.thread = mock.Mock(ident=None)
|
|
pump.thread.join.side_effect = RuntimeError('cannot join thread before it is started')
|
|
pump.close = mock.Mock()
|
|
pump.join(timeout=1)
|
|
pump.thread.join.assert_not_called()
|
|
pump.close.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize('publication_error', [None, OSError('private metadata unavailable'), KeyboardInterrupt()])
|
|
def test_rollback_pending_closes_admission_before_failed_hold_publication(launch, publication_error):
|
|
h = launch
|
|
context = lifecycle_context(
|
|
managed_sources=[h.source], postgres_controller=mock.Mock(),
|
|
instance_file='mock-instance', instance_id='exact-instance',
|
|
)
|
|
h.source.start_gate = lambda: supervisor.lifecycle_start_allowed(context)
|
|
assert not h.source.start(force=True)
|
|
assert not supervisor.lifecycle_start_allowed(context)
|
|
supervisor.update_instance_activation.side_effect = publication_error
|
|
with context['control_lock']:
|
|
if publication_error is None:
|
|
assert supervisor.shutdown_checkpoint(context)
|
|
else:
|
|
with pytest.raises(type(publication_error)):
|
|
supervisor.shutdown_checkpoint(context)
|
|
assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD
|
|
assert context['runtime_failed'] is True
|
|
assert context['authority_release_safe'] is False
|
|
assert not context['start_gate_open']
|
|
assert context['shutdown_event'].is_set()
|
|
context['postgres_controller'].request_stop.assert_not_called()
|
|
context['postgres_controller'].close.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize('stop_error', [None, OSError('owned child uncertain'), KeyboardInterrupt()])
|
|
def test_owned_child_uncertain_defers_postgres_until_retained_retry_confirms_exit(launch, stop_error):
|
|
h = launch
|
|
assert not h.source.start(force=True)
|
|
controller = mock.Mock(
|
|
lifecycle_action_required=True, terminal=True, stop_succeeded=True,
|
|
authority_release_safe=True, detail='',
|
|
)
|
|
controller.close.return_value = True
|
|
context = lifecycle_context(managed_sources=[h.source], postgres_controller=controller)
|
|
if stop_error is not None:
|
|
h.process.terminate.side_effect = stop_error
|
|
assert not supervisor.retain_unsafe_authority([h.source], context, max_attempts=1)
|
|
assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD
|
|
assert context['authority_release_safe'] is False
|
|
assert h.source.process is h.process
|
|
assert h.source.payload_identity is mock.sentinel.payload_identity
|
|
controller.request_stop.assert_not_called()
|
|
controller.close.assert_not_called()
|
|
supervisor.finalize_source_runs.assert_not_called()
|
|
h.process.terminate.side_effect = None
|
|
h.process.wait.side_effect = h.confirm_exit
|
|
assert supervisor.retain_unsafe_authority([h.source], context, max_attempts=1)
|
|
assert context['authority_release_safe'] is True
|
|
assert context['runtime_failed'] is True
|
|
assert h.source.process is None
|
|
assert h.source.payload_identity is None
|
|
assert not supervisor.lifecycle_start_allowed(context)
|
|
controller.request_stop.assert_called_once()
|
|
controller.close.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize('background', [False, True])
|
|
@pytest.mark.parametrize('interrupted', [False, True])
|
|
def test_main_retains_locks_and_control_until_startup_rollback_is_confirmed(runtime, launch, monkeypatch, background, interrupted):
|
|
h, child = runtime, launch
|
|
h.args.background_child = background
|
|
peer = mock.Mock(source='later', enabled=True, manual_only=False, status='stopped')
|
|
peer.stop.return_value = True
|
|
peer.is_running.return_value = False
|
|
supervisor.load_supervisor_runtime.return_value[-1].append('later')
|
|
|
|
def construct(name, *args):
|
|
if name == 'later':
|
|
return peer
|
|
child.source.authority_check = args[7]
|
|
child.source.start_gate = args[8]
|
|
return child.source
|
|
|
|
monkeypatch.setattr(supervisor, 'ManagedSource', mock.Mock(side_effect=construct))
|
|
if interrupted:
|
|
child.pump.attach.side_effect = KeyboardInterrupt()
|
|
real_retain = h.retain._mock_wraps
|
|
|
|
def retry(sources, context):
|
|
assert context['lifecycle_phase'] == supervisor.PHASE_FAILED_HOLD
|
|
assert not supervisor.lifecycle_start_allowed(context)
|
|
assert context['runtime_failed'] is True
|
|
assert child.source.process is child.process
|
|
assert child.source.payload_identity is mock.sentinel.payload_identity
|
|
h.instance_lock.release.assert_not_called()
|
|
h.cluster_lock.release.assert_not_called()
|
|
h.server.shutdown.assert_not_called()
|
|
supervisor.write_shutdown_receipt.assert_not_called()
|
|
h.controller.request_stop.assert_not_called()
|
|
h.controller.close.assert_not_called()
|
|
peer.start.assert_not_called()
|
|
child.process.wait.side_effect = child.confirm_exit
|
|
return real_retain(sources, context, max_attempts=1)
|
|
|
|
h.retain.side_effect = retry
|
|
assert supervisor.main() != 0
|
|
h.retain.assert_called_once()
|
|
peer.start.assert_not_called()
|
|
child.factory.assert_called_once()
|
|
assert child.source.process is None
|
|
assert child.source.payload_identity is None
|
|
assert supervisor.write_shutdown_receipt.call_args.args[-1] != 0
|
|
h.instance_lock.release.assert_called_once()
|
|
h.cluster_lock.release.assert_called_once()
|
|
|
|
|
|
@pytest.fixture
|
|
def stale_instance(runtime, monkeypatch):
|
|
h = runtime
|
|
h.stale = {
|
|
'schema': 2, 'instance_id': 'previous-instance', 'instance_file': h.instance_file,
|
|
'pid': 123, 'process_creation_time': 'exact-old-creation',
|
|
'executable': os.path.abspath('mock-python.exe'), 'startup_time': '2999-01-01T00:00:00Z',
|
|
}
|
|
h.legacy = False
|
|
|
|
def load_metadata(path):
|
|
if not h.metadata and h.legacy:
|
|
raise ValueError('legacy manifest')
|
|
return dict(h.metadata or h.stale)
|
|
|
|
supervisor.os.path.exists.return_value = True
|
|
supervisor.load_instance_metadata.side_effect = load_metadata
|
|
monkeypatch.setattr(supervisor, 'read_private_json', mock.Mock(side_effect=lambda path: dict(h.stale)))
|
|
monkeypatch.setattr(supervisor, 'durable_unlink', mock.Mock())
|
|
h.identity_state = 'reused'
|
|
|
|
def inspect(pid, creation_time, executable):
|
|
supervisor.SupervisorInstanceLock.return_value.acquire.assert_called()
|
|
supervisor.ClusterAuthorityLock.return_value.acquire.assert_called()
|
|
h.instance_lock.release.assert_not_called()
|
|
h.cluster_lock.release.assert_not_called()
|
|
supervisor.controller_from_config.assert_not_called()
|
|
assert (pid, creation_time, executable) == (
|
|
h.stale['pid'], h.stale['process_creation_time'], h.stale['executable'],
|
|
)
|
|
return h.identity_state
|
|
|
|
monkeypatch.setattr(supervisor, 'exact_process_identity_state', mock.Mock(side_effect=inspect))
|
|
return h
|
|
|
|
|
|
@pytest.mark.parametrize('legacy', [False, True])
|
|
@pytest.mark.parametrize('identity_state', ['dead', 'reused'])
|
|
def test_locked_stale_metadata_recovery_uses_exact_identity_not_pid_or_timestamp(stale_instance, legacy, identity_state):
|
|
h = stale_instance
|
|
h.identity_state = identity_state
|
|
h.legacy = legacy
|
|
assert supervisor.main() == 0
|
|
supervisor.exact_process_identity_state.assert_called_once_with(
|
|
h.stale['pid'], h.stale['process_creation_time'], h.stale['executable'],
|
|
)
|
|
supervisor.process_running_status.assert_not_called()
|
|
supervisor.verify_instance_process.assert_not_called()
|
|
supervisor.send_control_request.assert_not_called()
|
|
if legacy:
|
|
supervisor.durable_unlink.assert_called_once_with(h.instance_file)
|
|
supervisor.remove_instance_if_matches.assert_not_called()
|
|
else:
|
|
supervisor.remove_instance_if_matches.assert_called_once_with(
|
|
h.instance_file, h.stale['instance_id'], instance_lock=h.instance_lock,
|
|
)
|
|
supervisor.durable_unlink.assert_not_called()
|
|
supervisor.remove_shutdown_receipt.assert_called_once_with(h.instance_file, h.stale['instance_id'])
|
|
|
|
|
|
@pytest.mark.parametrize('observed', ['dead', 'reused', 'alive', 'unknown'])
|
|
def test_foreground_reconciliation_uses_existing_exact_identity_api(stale_instance, monkeypatch, observed):
|
|
h = stale_instance
|
|
process = mock.Mock(spec=process_identity.RetainedProcess)
|
|
process.identity = SimpleNamespace(
|
|
creation_time='exact-new-creation' if observed == 'reused' else h.stale['process_creation_time'],
|
|
executable=h.stale['executable'],
|
|
)
|
|
process.is_running.return_value = observed != 'dead'
|
|
opened = mock.Mock(return_value=process)
|
|
if observed == 'unknown':
|
|
opened.side_effect = process_identity.ProcessIdentityError('process inspection denied')
|
|
monkeypatch.setattr(process_identity, 'open_process', opened)
|
|
monkeypatch.setattr(supervisor, 'exact_process_identity_state', mock.Mock(
|
|
wraps=process_identity.exact_process_identity_state,
|
|
))
|
|
recoverable = observed in ('dead', 'reused')
|
|
assert supervisor.main() == (0 if recoverable else 1)
|
|
opened.assert_called_once_with(h.stale['pid'])
|
|
assert supervisor.remove_instance_if_matches.called is recoverable
|
|
assert supervisor.remove_shutdown_receipt.called is recoverable
|
|
assert process.close.call_count == int(observed != 'unknown')
|
|
process.wait.assert_not_called()
|
|
supervisor.process_running_status.assert_not_called()
|
|
supervisor.send_control_request.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize('identity_state', ['alive', 'unknown', 'mismatched'])
|
|
@pytest.mark.parametrize('legacy', [False, True])
|
|
def test_live_or_uncertain_owner_metadata_is_never_replaced(stale_instance, identity_state, legacy):
|
|
h = stale_instance
|
|
h.identity_state = identity_state
|
|
h.stale['startup_time'] = '1900-01-01T00:00:00Z'
|
|
h.legacy = legacy
|
|
assert supervisor.main() != 0
|
|
supervisor.remove_instance_if_matches.assert_not_called()
|
|
supervisor.durable_unlink.assert_not_called()
|
|
supervisor.remove_shutdown_receipt.assert_not_called()
|
|
supervisor.write_instance_metadata.assert_not_called()
|
|
supervisor.controller_from_config.assert_not_called()
|
|
h.instance_lock.release.assert_called_once()
|
|
h.cluster_lock.release.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize('legacy', [False, True])
|
|
def test_metadata_bound_to_another_instance_path_is_not_removed(stale_instance, legacy):
|
|
h = stale_instance
|
|
h.stale['instance_file'] += '.other'
|
|
h.legacy = legacy
|
|
assert supervisor.main() != 0
|
|
supervisor.remove_instance_if_matches.assert_not_called()
|
|
supervisor.durable_unlink.assert_not_called()
|
|
supervisor.remove_shutdown_receipt.assert_not_called()
|
|
supervisor.controller_from_config.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize('operation', ['identity', 'remove'])
|
|
@pytest.mark.parametrize('error,expected', [(OSError('unavailable'), 1), (KeyboardInterrupt(), 0), (SystemExit(7), 7)])
|
|
def test_reconciliation_error_or_interruption_preserves_metadata_for_retry(stale_instance, operation, error, expected):
|
|
h = stale_instance
|
|
target = supervisor.exact_process_identity_state if operation == 'identity' else supervisor.remove_instance_if_matches
|
|
previous = target.side_effect
|
|
target.side_effect = error
|
|
assert supervisor.main() == expected
|
|
supervisor.durable_unlink.assert_not_called()
|
|
supervisor.remove_shutdown_receipt.assert_not_called()
|
|
supervisor.write_instance_metadata.assert_not_called()
|
|
supervisor.controller_from_config.assert_not_called()
|
|
target.side_effect = previous
|
|
h.instance_lock.release.reset_mock()
|
|
h.cluster_lock.release.reset_mock()
|
|
assert supervisor.main() == 0
|
|
supervisor.remove_shutdown_receipt.assert_called_once_with(h.instance_file, h.stale['instance_id'])
|
|
|
|
|
|
def test_changed_instance_during_matching_removal_fails_closed(stale_instance):
|
|
supervisor.remove_instance_if_matches.return_value = False
|
|
assert supervisor.main() != 0
|
|
supervisor.remove_instance_if_matches.assert_called_once_with(
|
|
stale_instance.instance_file, stale_instance.stale['instance_id'],
|
|
instance_lock=stale_instance.instance_lock,
|
|
)
|
|
supervisor.remove_shutdown_receipt.assert_not_called()
|
|
supervisor.durable_unlink.assert_not_called()
|
|
supervisor.controller_from_config.assert_not_called()
|