truf-server/tests/test_supervisor_safety.py

import json
import os
from pathlib import Path
import socket
import sqlite3
import subprocess
import sys
import tempfile
import threading
import time
from types import SimpleNamespace
import unittest
from unittest import mock


ROOT = Path(__file__).resolve().parents[1]
APP_DIR = ROOT / 'app'
sys.path.insert(0, str(APP_DIR))

import supervisor
import supervisor_instance
from scanner_db import ScannerDB
from process_identity import ProcessIdentity, ProcessIdentityError, current_process_identity
from paths import apply_path_config
from lifecycle_authority import (
    PHASE_ACTIVE, PHASE_ACTIVATING, PHASE_FAILED_HOLD, PHASE_STOPPING,
)
from runtime_security import ClusterAuthorityLock, ensure_private_directory, harden_private_file
from supervisor_instance import (
    InstanceMetadataError,
    InstanceLockError,
    SupervisorInstanceLock,
    build_instance_metadata,
    verify_instance_process,
    write_instance_metadata,
)


class RecheckCommandParsingTests(unittest.TestCase):
    def test_legacy_vertex_maps_to_authenticated_import_without_recheck_all(self):
        parsed, error = supervisor.parse_recheck_command(['recheck', 'gcp', 'legacy-vertex'])
        self.assertIsNone(error)
        self.assertEqual(parsed['service'], 'gcp')
        self.assertEqual(parsed['runner_args'], ['--import-legacy-gcp-vertex'])


class FakeOwnedProcess:
    next_pid = 40000

    def __init__(self, *args, **kwargs):
        type(self).next_pid += 1
        self.pid = type(self).next_pid
        self.returncode = None
        self.terminated = False
        self.kwargs = kwargs

    def poll(self):
        return self.returncode

    def terminate(self):
        self.terminated = True
        self.returncode = 0

    def kill(self):
        self.returncode = 1

    def wait(self, timeout=None):
        if self.returncode is None:
            self.returncode = 0
        return self.returncode

    def exit_code(self):
        return self.returncode


class UnkillableProcess(FakeOwnedProcess):
    def terminate(self):
        self.terminated = True

    def kill(self):
        pass

    def wait(self, timeout=None):
        raise subprocess.TimeoutExpired('unkillable', timeout)


def source_fixture(temp_dir, gate):
    ensure_private_directory(temp_dir, reject_reparse=True)
    config_path = os.path.join(temp_dir, 'config.yaml')
    Path(config_path).write_text('{}\n', encoding='ascii')
    harden_private_file(config_path)
    ensure_private_directory(os.path.join(temp_dir, 'logs'), reject_reparse=True)
    ensure_private_directory(os.path.join(temp_dir, 'state'), reject_reparse=True)
    supervisor_config = {
        'log_dir': os.path.join(temp_dir, 'logs'),
        'state_dir': os.path.join(temp_dir, 'state'),
        'defaults': {'enabled': True, 'restart': True},
    }
    return supervisor.ManagedSource(
        'sample', config_path, str(APP_DIR), temp_dir, supervisor_config,
        {'enabled': True}, dependency_gate=gate,
    )


def docker_shadow_fixture(temp_dir, gate):
    ensure_private_directory(temp_dir, reject_reparse=True)
    config_path = os.path.join(temp_dir, 'config.yaml')
    Path(config_path).write_text('{}\n', encoding='ascii')
    harden_private_file(config_path)
    ensure_private_directory(os.path.join(temp_dir, 'logs'), reject_reparse=True)
    ensure_private_directory(os.path.join(temp_dir, 'state'), reject_reparse=True)
    supervisor_config = {
        'log_dir': os.path.join(temp_dir, 'logs'),
        'state_dir': os.path.join(temp_dir, 'state'),
        'defaults': {'enabled': True, 'restart': True, 'repeat': True},
    }
    return supervisor.ManagedDockerShadow(
        config_path, str(APP_DIR), temp_dir, supervisor_config,
        {'enabled': True}, dependency_gate=gate,
    )


def prepare_supervisor_fixture(temp_dir, control_port=0):
    ensure_private_directory(temp_dir, reject_reparse=True)
    project_dir = os.path.join(temp_dir, 'project')
    runtime_dir = os.path.join(temp_dir, 'runtime')
    paths = {
        'project_dir': project_dir,
        'runtime_dir': runtime_dir,
        'results_dir': os.path.join(runtime_dir, 'results'),
        'result_spool_dir': os.path.join(runtime_dir, 'result_spool'),
        'queue_dir': os.path.join(runtime_dir, 'queues'),
        'state_dir': os.path.join(runtime_dir, 'state'),
        'log_dir': os.path.join(runtime_dir, 'logs'),
        'control_dir': os.path.join(runtime_dir, 'control'),
        'keycheck_dir': os.path.join(runtime_dir, 'keychecks'),
        'postman_cache_dir': os.path.join(runtime_dir, 'postman_cache'),
        'gharchive_cache_dir': os.path.join(runtime_dir, 'state', 'gharchive_cache'),
        'work_dir': os.path.join(temp_dir, 'work'),
        'postgres_dir': os.path.join(runtime_dir, 'postgres'),
    }
    for path in paths.values():
        ensure_private_directory(path, reject_reparse=True)
    trufflehog_path = os.path.join(temp_dir, 'trufflehog.exe')
    Path(trufflehog_path).write_bytes(b'fixture executable')
    harden_private_file(trufflehog_path)
    config_path = os.path.join(temp_dir, 'config.yaml')
    value = {
        'global': {
            'root_dir': temp_dir,
            'trufflehog_path': trufflehog_path,
            **{key: value for key, value in paths.items() if key != 'postgres_dir'},
        },
        'supervisor': {
            'interactive': False,
            'autostart': False,
            'control_port': control_port,
            'background_start_timeout_sec': 10,
            'background_activation_timeout_sec': 20,
            'background_shutdown_timeout_sec': 15,
            'dashboard': {'enabled': False},
            'sources': {'fixture': {'enabled': True}},
        },
        'keychecks': {'enabled': False},
        'sources': {'fixture': {'enabled': True}},
    }
    Path(config_path).write_text(json.dumps(value), encoding='ascii')
    harden_private_file(config_path)
    supervisor_log = os.path.join(paths['log_dir'], 'supervisor.log')
    Path(supervisor_log).write_text('', encoding='ascii')
    harden_private_file(supervisor_log)
    return config_path, paths


def private_background_config(temp_dir, **values):
    ensure_private_directory(temp_dir, reject_reparse=True)
    control_dir = os.path.join(temp_dir, 'control')
    ensure_private_directory(control_dir, reject_reparse=True)
    return {'log_dir': temp_dir, 'control_dir': control_dir, **values}


class DesiredStateGateTests(unittest.TestCase):
    def test_docker_shadow_is_manual_singleton_without_relaunch(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            shadow = docker_shadow_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            command = shadow.build_command()

            self.assertTrue(shadow.manual_only)
            self.assertTrue(shadow.once)
            self.assertFalse(shadow.repeat)
            self.assertFalse(shadow.restart)
            self.assertFalse(shadow.use_per_source_state)
            self.assertEqual(shadow.mode_label(), 'manual-once')
            self.assertEqual(command[5], 'docker-shadow')
            self.assertEqual(command[-2:], ['--config', shadow.config_path])
            self.assertEqual(supervisor.autostart_sources([shadow]), [])

            shadow.desired_state = 'running'
            shadow.manual_stop = False
            shadow.process = FakeOwnedProcess()
            shadow.process.returncode = 0
            shadow.started_at = time.time()
            shadow.poll()

            self.assertEqual(shadow.status, 'done')
            self.assertEqual(shadow.desired_state, 'stopped')
            self.assertEqual(shadow.restarts, 0)
            self.assertEqual(shadow.next_start_at, 0)

    def test_docker_shadow_does_not_retain_intent_across_dependency_loss(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            gate = supervisor.DependencyGate(ready=False)
            shadow = docker_shadow_fixture(temp_dir, gate)

            self.assertFalse(shadow.start(force=True))
            self.assertEqual(shadow.desired_state, 'stopped')
            self.assertFalse(shadow.runtime_blocked)
            self.assertIn('unavailable', shadow.last_action_error)

    def test_unresolved_handoff_exit_enters_hold_without_restart(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.restart = True
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = supervisor.SOURCE_INFRASTRUCTURE_HOLD_EXIT
            source.started_at = time.time()

            source.poll()

            self.assertEqual(source.status, 'failed')
            self.assertEqual(source.desired_state, 'stopped')
            self.assertTrue(source.manual_stop)
            self.assertEqual(source.restarts, 0)
            self.assertEqual(source.next_start_at, 0)

    def test_failed_nonrepeat_batch_with_restart_disabled_stays_failed(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.once = True
            source.repeat = False
            source.restart = False
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = 1
            source.started_at = time.time()

            source.poll()

            self.assertEqual(source.status, 'failed')
            self.assertEqual(source.desired_state, 'stopped')
            self.assertEqual(source.restarts, 0)
            self.assertEqual(source.next_start_at, 0)

    def test_failed_periodic_keychecks_wait_for_each_interval(self):
        clock = {'now': 1000.0}
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor.time, 'time', side_effect=lambda: clock['now']), \
                mock.patch.object(supervisor, 'open_process', side_effect=OSError('fixture process')), \
                mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            ensure_private_directory(temp_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            ensure_private_directory(os.path.join(temp_dir, 'logs'), reject_reparse=True)
            ensure_private_directory(os.path.join(temp_dir, 'state'), reject_reparse=True)
            source = supervisor.ManagedKeychecks(
                config_path,
                str(APP_DIR),
                temp_dir,
                {'log_dir': os.path.join(temp_dir, 'logs'), 'state_dir': os.path.join(temp_dir, 'state')},
                {
                    'enabled': True,
                    'repeat': True,
                    'restart': False,
                    'interval': 60,
                    'summary_tsv': os.path.join(temp_dir, 'summary.tsv'),
                },
                dependency_gate=supervisor.DependencyGate(ready=True),
            )
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = 1
            source.started_at = 900.0

            source.poll()

            self.assertEqual(source.status, 'waiting')
            self.assertEqual(source.desired_state, 'running')
            self.assertFalse(source.manual_stop)
            self.assertEqual(source.last_exit_code, 1)
            self.assertEqual(source.next_start_at, 1060.0)
            self.assertEqual(source.restarts, 1)
            self.assertEqual(source.restart_streak, 0)
            self.assertEqual(source.row()[5], '1')
            self.assertNotEqual(source.row()[6], '-')
            self.assertEqual(source.row()[11], 'running')

            scheduled = source.next_start_at
            source.poll()
            source.poll()
            self.assertEqual(source.next_start_at, scheduled)
            owned.assert_not_called()

            clock['now'] = 1059.0
            source.poll()
            owned.assert_not_called()
            clock['now'] = 1060.0
            source.poll()
            self.assertEqual(owned.call_count, 1)
            self.assertEqual(source.status, 'running')
            self.assertEqual(source.next_start_at, 0)

            source.process.returncode = 1
            clock['now'] = 1061.0
            source.poll()
            self.assertEqual(source.status, 'waiting')
            self.assertEqual(source.last_exit_code, 1)
            self.assertEqual(source.next_start_at, 1121.0)
            self.assertEqual(source.restarts, 2)
            self.assertEqual(source.restart_streak, 0)
            source.poll()
            self.assertEqual(owned.call_count, 1)
            source.stop(final=True)

    def test_manual_and_coordinated_stop_cancel_pending_periodic_launch(self):
        for final in (False, True):
            with self.subTest(final=final), tempfile.TemporaryDirectory() as temp_dir, \
                    mock.patch.object(supervisor.time, 'time', return_value=1000.0), \
                    mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
                source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
                source.once = True
                source.repeat = True
                source.restart = False
                source.interval = 60
                source.desired_state = 'running'
                source.manual_stop = False
                source.process = FakeOwnedProcess()
                source.process.returncode = 1
                source.started_at = 900.0

                source.poll()
                self.assertEqual(source.next_start_at, 1060.0)
                self.assertTrue(source.stop(final=final))
                self.assertEqual(source.status, 'stopped')
                self.assertEqual(source.desired_state, 'stopped')
                self.assertEqual(source.next_start_at, 0)
                source.poll()
                owned.assert_not_called()

    def test_successful_repeat_batch_schedules_configured_interval(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.once = True
            source.repeat = True
            source.restart = False
            source.interval = 3600
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = 0
            source.started_at = time.time()
            before = time.time()

            source.poll()

            self.assertEqual(source.status, 'waiting')
            self.assertEqual(source.desired_state, 'running')
            self.assertEqual(source.restarts, 1)
            self.assertGreaterEqual(source.next_start_at, before + source.interval)

    def test_gate_loss_preserves_finished_failed_repeat_interval(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(
            supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess,
        ) as owned:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            source.once = True
            source.repeat = True
            source.restart = False
            source.interval = 3600
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = 1
            source.started_at = time.time()
            before = time.time()

            gate.set_ready(False)
            scheduled = source.next_start_at
            gate.set_ready(True)

            self.assertEqual(source.last_exit_code, 1)
            self.assertGreaterEqual(scheduled, before + source.interval)
            self.assertEqual(source.next_start_at, scheduled)
            self.assertEqual(source.status, 'waiting')
            self.assertEqual(source.desired_state, 'running')
            owned.assert_not_called()

    def test_gate_loss_preserves_finished_successful_repeat_interval(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(
            supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess,
        ) as owned:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            source.once = True
            source.repeat = True
            source.restart = False
            source.interval = 3600
            source.desired_state = 'running'
            source.manual_stop = False
            source.process = FakeOwnedProcess()
            source.process.returncode = 0
            source.started_at = time.time()
            before = time.time()

            gate.set_ready(False)
            scheduled = source.next_start_at
            gate.set_ready(True)

            self.assertGreaterEqual(scheduled, before + source.interval)
            self.assertEqual(source.next_start_at, scheduled)
            self.assertEqual(source.status, 'waiting')
            self.assertEqual(source.desired_state, 'running')
            owned.assert_not_called()

    def test_supervisor_text_output_writer_rotates_with_bounded_retention(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            path = os.path.join(temp_dir, 'supervisor.log')
            writer = supervisor.BoundedRotatingTextWriter(path, 64, 2)
            try:
                writer.write('x' * 400)
                writer.flush()
            finally:
                writer.close()
            logs = [candidate for candidate in Path(temp_dir).glob('supervisor*.log')]
            self.assertLessEqual(len(logs), 3)
            self.assertTrue(logs)
            self.assertTrue(all(candidate.stat().st_size <= 64 for candidate in logs))

    def test_live_source_output_rotates_with_bounded_segments_and_tail_reads(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.log_max_mb = 1
            source.log_keep = 2
            source.once = True
            source.repeat = False
            source.restart = False
            source.build_command = lambda: [
                sys.executable,
                '-c',
                'import sys; sys.stdout.buffer.write(b"x" * (3 * 1024 * 1024 + 123)); sys.stdout.buffer.flush()',
            ]
            self.assertTrue(source.start(force=True))
            deadline = time.monotonic() + 20
            while time.monotonic() < deadline and source.status == 'running':
                time.sleep(0.02)
                source.poll()
            self.assertEqual(source.status, 'done', source.last_action_error)
            paths = [Path(source.log_path), *Path(source.log_path).parent.glob('sample.*.log')]
            paths = list(dict.fromkeys(path for path in paths if path.exists()))
            self.assertLessEqual(len(paths), source.log_keep + 1)
            self.assertTrue(paths)
            self.assertTrue(all(path.stat().st_size <= 1024 * 1024 for path in paths))
            tail = source.tail_log_lines(5000)
            self.assertLessEqual(sum(len(line.encode('utf-8')) for line in tail), supervisor.MAX_LOG_TAIL_BYTES)

    def test_owned_process_start_failure_closes_log_and_returns_false(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(
            supervisor,
            'OwnedProcess',
            side_effect=RuntimeError('host setup failed'),
        ):
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            self.assertFalse(source.start(force=True))
            self.assertEqual(source.status, 'failed')
            self.assertIsNone(source.process)
            self.assertIsNone(source.log_handle)
            self.assertIn('host setup failed', source.last_action_error)

    def test_running_intent_waits_for_ready_and_stop_pause_are_preserved(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            gate = supervisor.DependencyGate(ready=False)
            source = source_fixture(temp_dir, gate)

            self.assertFalse(source.start(force=True))
            self.assertEqual(source.desired_state, 'running')
            self.assertEqual(source.status, 'blocked')
            owned.assert_not_called()

            source.stop()
            self.assertEqual(source.desired_state, 'stopped')
            gate.set_ready(True)
            owned.assert_not_called()

            gate.set_ready(False)
            source.resume()
            source.pause()
            self.assertEqual(source.desired_state, 'paused')
            gate.set_ready(True)
            owned.assert_not_called()

    def test_readiness_loss_stops_owned_child_without_changing_running_intent(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            self.assertTrue(source.start(force=True))
            first = source.process

            gate.set_ready(False)
            self.assertTrue(first.terminated)
            self.assertEqual(source.desired_state, 'running')
            self.assertEqual(source.status, 'blocked')
            gate.set_ready(True)

            self.assertEqual(owned.call_count, 2)
            self.assertEqual(source.desired_state, 'running')
            self.assertEqual(source.status, 'running')
            source.stop(final=True)

    def test_keycheck_recheck_remains_queued_while_blocked(self):
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            ensure_private_directory(temp_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            ensure_private_directory(os.path.join(temp_dir, 'logs'), reject_reparse=True)
            ensure_private_directory(os.path.join(temp_dir, 'state'), reject_reparse=True)
            gate = supervisor.DependencyGate(ready=False)
            item = supervisor.ManagedKeychecks(
                config_path,
                str(APP_DIR),
                temp_dir,
                {'log_dir': os.path.join(temp_dir, 'logs'), 'state_dir': os.path.join(temp_dir, 'state')},
                {'enabled': True, 'repeat': True, 'summary_tsv': os.path.join(temp_dir, 'summary.tsv')},
                dependency_gate=gate,
            )

            ok, message = item.start_recheck('all', ['--retry-network'])
            self.assertTrue(ok)
            self.assertIn('blocked', message)
            item.poll()
            self.assertIsNotNone(item.command_override)
            owned.assert_not_called()
            gate.set_ready(True)
            self.assertEqual(owned.call_count, 1)
            item.process.returncode = 0
            item.poll()
            self.assertIsNone(item.command_override)
            self.assertEqual(item.desired_state, 'running')
            self.assertEqual(item.status, 'waiting')
            item.stop(final=True)

    def test_source_and_keycheck_env_overrides_cannot_replace_managed_dsn(self):
        canonical = 'postgresql://truf:secret@127.0.0.1:5432/truf'
        with tempfile.TemporaryDirectory() as temp_dir:
            gate = supervisor.DependencyGate(ready=True, database_url=canonical)
            source = source_fixture(temp_dir, gate)
            source.env_overrides = {
                'SCANNER_DB_URL': 'postgresql://attacker/other',
                'DATABASE_URL': 'postgresql://attacker/other',
                'TRUF_MANAGED_POSTGRES_DSN': 'postgresql://attacker/other',
                'PGSERVICE': 'attacker',
                'PGHOSTADDR': '203.0.113.10',
            }
            source_env = source.build_env()
            self.assertEqual(source_env['SCANNER_DB_URL'], canonical)
            self.assertEqual(source_env['DATABASE_URL'], canonical)
            self.assertEqual(source_env['TRUF_MANAGED_POSTGRES_DSN'], canonical)
            self.assertNotIn('PGSERVICE', source_env)
            self.assertNotIn('PGHOSTADDR', source_env)

            item = supervisor.ManagedKeychecks(
                os.path.join(temp_dir, 'config.yaml'),
                str(APP_DIR),
                temp_dir,
                {'log_dir': os.path.join(temp_dir, 'logs'), 'state_dir': os.path.join(temp_dir, 'state')},
                {'enabled': True, 'env': {'KEYCHECK_DB_URL': 'postgresql://attacker/other'}},
                dependency_gate=gate,
            )
            self.assertEqual(item.build_env()['KEYCHECK_DB_URL'], canonical)


class ManagedConfigurationAuthorityTests(unittest.TestCase):
    def test_managed_dsn_overrides_config_database_urls(self):
        canonical = 'postgresql://truf:secret@127.0.0.1:5432/truf'
        with tempfile.TemporaryDirectory() as temp_dir, mock.patch.dict(
            os.environ,
            {'TRUF_MANAGED_POSTGRES_DSN': canonical},
            clear=False,
        ):
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            config = apply_path_config({
                'global': {
                    'root_dir': temp_dir,
                    'project_dir': temp_dir,
                    'database_url': 'postgresql://attacker/other',
                    'dashboard_db_url': 'postgresql://attacker/dashboard',
                },
            }, config_path)
        self.assertEqual(config['global']['database_url'], canonical)
        self.assertEqual(config['global']['dashboard_db_url'], canonical)
        self.assertEqual(
            os.path.dirname(config['supervisor']['instance_file']),
            config['global']['control_dir'],
        )
        self.assertEqual(
            os.path.dirname(config['supervisor']['lock_file']),
            config['global']['control_dir'],
        )


class SourceRunFinalizationTests(unittest.TestCase):
    def test_finalize_source_runs_closes_cycles_and_runs_transactionally(self):
        connection = mock.Mock()
        connection.execute.side_effect = [
            SimpleNamespace(rowcount=2),
            SimpleNamespace(rowcount=1),
        ]
        with mock.patch.object(supervisor, 'connect_postgres', return_value=connection):
            counts = supervisor.finalize_source_runs(
                'postgresql://truf:fixture@127.0.0.1:5432/truf',
                'github',
                'stopped',
                'controlled stop',
            )
        self.assertEqual(counts, (2, 1))
        connection.commit.assert_called_once_with()
        connection.rollback.assert_not_called()
        connection.close.assert_called_once_with()

    def test_source_start_does_not_run_database_finalization_before_child_handshake(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.desired_state = 'running'
            source.manual_stop = False
            with mock.patch.object(source, 'finalize_database_runs', side_effect=AssertionError('control blocked')) as finalize, \
                    mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess):
                self.assertTrue(source.start())
            finalize.assert_not_called()
            self.assertEqual(source.status, 'running')

    def test_scanner_child_finalizes_stale_rows_before_inserting_its_new_run(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            database = os.path.join(temp_dir, 'scanner.db')
            db = ScannerDB(db_path=database, db_url='', initialize=True)
            try:
                old_run = db.start_run('config', selected_source='github')
                old_cycle = db.start_source_cycle(old_run, 'github', 'github', 'recent', 'fixture')
                new_run = db.start_run('config', selected_source='github')
                old_run_row = db.conn.execute('SELECT status FROM runs WHERE id = ?', (old_run,)).fetchone()
                old_cycle_row = db.conn.execute('SELECT status FROM source_cycles WHERE id = ?', (old_cycle,)).fetchone()
                new_run_row = db.conn.execute('SELECT status FROM runs WHERE id = ?', (new_run,)).fetchone()
                self.assertEqual(old_run_row['status'], 'interrupted')
                self.assertEqual(old_cycle_row['status'], 'interrupted')
                self.assertEqual(new_run_row['status'], 'running')
                indexes = db.conn.table_indexes('runs')
                self.assertEqual(
                    indexes['idx_runs_selected_source_status']['columns'],
                    ['selected_source', 'status', 'id'],
                )
            finally:
                db.close()


class AuthenticatedControlTests(unittest.TestCase):
    @staticmethod
    def structured_source_state(source='sample', role='scanner', **overrides):
        value = {
            'id': source,
            'source': source,
            'role': role,
            'lifecycle_state': 'running',
            'desired_state': 'running',
            'process_state': 'running',
            'pid': 1234,
            'enabled': True,
            'dependency_blocked': False,
            'startup_cleanup_pending': False,
            'mode': 'loop',
            'interval_seconds': 60,
            'restart_enabled': True,
            'restart_delay_seconds': 5,
            'restart_count': 0,
            'restart_streak': 0,
            'last_exit_code': None,
            'last_exit_at': None,
            'next_scheduled_run_at': None,
            'safe_error_category': '',
            'auth_summary': {},
            'allowed_actions': ['start', 'stop'],
        }
        value.update(overrides)
        return value

    def setUp(self):
        self.shutdown_event = threading.Event()
        self.server = supervisor.SupervisorControlServer(
            ('127.0.0.1', 0),
            [],
            {'shutdown_event': self.shutdown_event, 'with_postgres': True},
            threading.RLock(),
            'instance-test',
            't' * 48,
        )
        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
        self.thread.start()
        self.metadata = {
            'instance_id': 'instance-test',
            'token': 't' * 48,
            'control': {'host': '127.0.0.1', 'port': self.server.server_address[1]},
        }

    def tearDown(self):
        self.server.shutdown()
        self.server.server_close()

    def test_requests_require_instance_and_token_and_shutdown_returns_promptly(self):
        self.assertFalse(supervisor.SupervisorControlServer.allow_reuse_address)
        wrong = dict(self.metadata)
        wrong['token'] = 'x' * 48
        with self.assertRaises(RuntimeError):
            supervisor.send_control_request(wrong, 'handshake')

        handshake = supervisor.send_control_request(self.metadata, 'handshake')
        self.assertEqual(handshake['instance_id'], 'instance-test')
        result = supervisor.send_control_request(self.metadata, 'shutdown', with_postgres=True)
        self.assertIn('shutdown requested', result)
        self.assertTrue(self.shutdown_event.is_set())

    def test_control_request_timeout_is_one_total_deadline(self):
        sock = mock.MagicMock()
        sock.__enter__.return_value = sock
        sock.recv.side_effect = [b'{', b'']
        with mock.patch.object(
            supervisor.socket, 'create_connection', return_value=sock,
        ) as connect, mock.patch.object(
            supervisor.time, 'monotonic', side_effect=[0, 0, 1, 2, 6],
        ):
            with self.assertRaisesRegex(TimeoutError, '^control request timed out$'):
                supervisor.send_control_request(self.metadata, 'handshake', timeout=5)

        connect.assert_called_once_with(
            ('127.0.0.1', self.metadata['control']['port']), timeout=5,
        )
        self.assertEqual(
            [call.args[0] for call in sock.settimeout.call_args_list], [4, 3],
        )
        self.assertEqual(sock.recv.call_count, 1)

    def test_structured_runtime_snapshot_covers_all_managed_process_state(self):
        source = mock.Mock()
        source.structured_state.return_value = self.structured_source_state(
            'result-ingester', 'result-ingester',
            mode='singleton', interval_seconds=0,
            allowed_actions=['start', 'stop', 'restart', 'pause', 'resume'],
        )
        controller = mock.Mock()
        controller.snapshot.return_value = {
            'state': 'READY', 'ready': True, 'failures': 0,
            'detail': 'postgres-secret-must-not-leak',
        }
        dashboard = mock.Mock()
        dashboard.snapshot.return_value = {
            'status': 'healthy', 'desired': 'running', 'healthy': True,
            'pid': 4321, 'failures': 0,
            'detail': 'dashboard-secret-must-not-leak',
            'url': 'https://dashboard-secret-must-not-leak',
        }
        self.server.managed_sources[:] = [source]
        self.server.context.update({
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
            'postgres_controller': controller,
            'dashboard_manager': dashboard,
        })

        snapshot = supervisor.get_runtime_snapshot(self.metadata)

        self.assertEqual(snapshot['snapshot_schema'], supervisor.RUNTIME_SNAPSHOT_SCHEMA)
        self.assertEqual(snapshot['runtime']['phase'], PHASE_ACTIVE)
        self.assertEqual(snapshot['sources'], [source.structured_state.return_value])
        self.assertEqual(snapshot['postgres']['state'], 'READY')
        self.assertEqual(snapshot['dashboard']['status'], 'healthy')
        serialized = json.dumps(snapshot, sort_keys=True)
        self.assertNotIn('must-not-leak', serialized)
        for phase in (PHASE_STOPPING, PHASE_FAILED_HOLD):
            with self.subTest(phase=phase):
                self.server.context['lifecycle_phase'] = phase
                self.server.context['activation_state'] = phase
                held = supervisor.get_runtime_snapshot(self.metadata)
                self.assertEqual(held['runtime']['phase'], phase)
        self.server.context['lifecycle_phase'] = PHASE_ACTIVE
        self.server.context['activation_state'] = PHASE_ACTIVE
        with self.assertRaisesRegex(RuntimeError, 'fields are invalid'):
            supervisor.send_control_request(
                self.metadata, 'runtime-snapshot', command='status',
            )
        source.structured_state.side_effect = RuntimeError('snapshot-secret-must-not-leak')
        with self.assertRaisesRegex(RuntimeError, '^control request failed$') as raised:
            supervisor.get_runtime_snapshot(self.metadata)
        self.assertNotIn('must-not-leak', str(raised.exception))

    def test_typed_actions_cover_managed_source_and_dashboard_functionality(self):
        source = mock.Mock()
        source.source = 'sample'
        source.manual_only = False
        source.runtime_blocked = False
        source.desired_state = 'running'
        source.is_running.return_value = False
        source.start.return_value = True
        source.stop.return_value = True
        source.restart_now.return_value = True
        source.pause.return_value = True
        source.resume.return_value = True
        source.structured_state.return_value = self.structured_source_state(
            allowed_actions=list(
                supervisor.MANAGED_SOURCE_LIFECYCLE_ACTIONS
                + supervisor.MANAGED_SOURCE_SETTING_ACTIONS
            ),
        )
        self.server.managed_sources[:] = [source]
        self.server.context.update({
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
        })

        with mock.patch.object(supervisor, 'run_control_command') as generic:
            for action, method in (
                ('start', source.start), ('stop', source.stop),
                ('restart', source.restart_now), ('pause', source.pause),
                ('resume', source.resume),
            ):
                with self.subTest(action=action):
                    result = supervisor.send_managed_source_action(
                        self.metadata, 'sample', action,
                    )
                    self.assertEqual(result['source_action'], action)
                    self.assertEqual(result['outcome'], 'completed')
                    method.assert_called()
            source.is_running.return_value = True
            supervisor.send_managed_source_action(self.metadata, 'sample', 'once')
            source.stop.assert_called_with(timeout=10)
            source.start.assert_called_with(force=True)
            self.assertTrue(source.once)
            self.assertFalse(source.repeat)
            source.is_running.return_value = False
            supervisor.send_managed_source_action(
                self.metadata, 'sample', 'set-mode', mode='repeat',
            )
            self.assertTrue(source.once)
            self.assertTrue(source.repeat)
            supervisor.send_managed_source_action(
                self.metadata, 'sample', 'set-interval', interval_seconds=90,
            )
            self.assertEqual(source.interval, 90)
            with self.assertRaisesRegex(RuntimeError, 'interval is invalid'):
                supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'set-interval',
                    interval_seconds=supervisor.MAX_MANAGED_SOURCE_DELAY_SECONDS + 1,
                )
            with self.assertRaisesRegex(RuntimeError, 'interval is invalid'):
                supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'set-interval', interval_seconds=0,
                )
            supervisor.send_managed_source_action(
                self.metadata, 'sample', 'set-restart', restart_enabled=False,
            )
            self.assertFalse(source.restart)
            supervisor.send_managed_source_action(
                self.metadata, 'sample', 'set-restart-delay', restart_delay_seconds=12,
            )
            self.assertEqual(source.restart_delay, 12)
            with self.assertRaisesRegex(RuntimeError, 'restart delay is invalid'):
                supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'set-restart-delay',
                    restart_delay_seconds=supervisor.MAX_MANAGED_SOURCE_DELAY_SECONDS + 1,
                )
            with self.assertRaisesRegex(RuntimeError, 'restart delay is invalid'):
                supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'set-restart-delay', restart_delay_seconds=0,
                )
            self.server.context['status_file'] = 'status-secret-must-not-leak'
            with mock.patch.object(
                supervisor, 'write_status_file', side_effect=OSError('path-secret-must-not-leak'),
            ) as write_status:
                result = supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'start',
                )
            self.assertEqual(result['outcome'], 'completed')
            write_status.assert_called_once_with(
                self.server.context['status_file'], [source], self.server.context,
            )
            self.server.context.pop('status_file')
            generic.assert_not_called()

        dashboard = mock.Mock()
        dashboard.start.return_value = True
        dashboard.stop.return_value = True
        dashboard.snapshot.return_value = {
            'status': 'healthy', 'desired': 'running', 'healthy': True,
            'pid': 1234, 'failures': 0, 'detail': '', 'url': '',
        }
        self.server.context['dashboard_manager'] = dashboard
        with mock.patch.object(supervisor, 'run_control_command') as generic:
            for action in ('start', 'stop', 'restart'):
                with self.subTest(dashboard_action=action):
                    result = supervisor.send_dashboard_action(self.metadata, action)
                    self.assertEqual(result['dashboard_action'], action)
                    self.assertEqual(result['outcome'], 'completed')
            generic.assert_not_called()

    def test_typed_log_tail_is_exact_allowlisted_and_bounded(self):
        source = mock.Mock()
        source.source = 'sample'
        source.tail_log_lines.return_value = ['first', 'second']
        self.server.managed_sources[:] = [source]
        self.server.context.update({
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
        })

        with mock.patch.object(supervisor, 'run_control_command') as generic:
            result = supervisor.send_managed_source_log_tail(
                self.metadata, 'sample', 2,
            )
            self.assertEqual(result, {
                'source_id': 'sample',
                'line_count': 2,
                'lines': ['first', 'second'],
                'response_truncated': False,
            })
            source.tail_log_lines.assert_called_once_with(2)
            generic.assert_not_called()

            for source_id in ('all', 'docker', '../sample', 'sample; status'):
                with self.subTest(source_id=source_id), self.assertRaisesRegex(
                    RuntimeError, 'managed source ID is unknown',
                ):
                    supervisor.send_managed_source_log_tail(
                        self.metadata, source_id, 2,
                    )
            for line_count in (0, supervisor.MAX_LOG_TAIL_LINES + 1, True, '2'):
                with self.subTest(line_count=line_count), self.assertRaisesRegex(
                    RuntimeError, 'line count is invalid',
                ):
                    supervisor.send_managed_source_log_tail(
                        self.metadata, 'sample', line_count,
                    )
            with self.assertRaisesRegex(RuntimeError, 'fields are invalid'):
                supervisor.send_control_request(
                    self.metadata,
                    'managed-source-log-tail',
                    source_id='sample',
                    line_count=2,
                    path='log-secret-must-not-be-accepted',
                )
            generic.assert_not_called()

        source.tail_log_lines.return_value = ['a' * 20, 'b' * 20]
        with mock.patch.object(supervisor, 'MAX_LOG_TAIL_BYTES', 32):
            bounded = supervisor.send_managed_source_log_tail(
                self.metadata, 'sample', 2,
            )
        self.assertEqual(bounded['lines'], ['b' * 20])
        self.assertTrue(bounded['response_truncated'])
        self.assertLessEqual(
            len(json.dumps(bounded['lines'], ensure_ascii=True).encode('utf-8')),
            32,
        )

        escaped_lines = ['emoji:\U0001f600\n', 'control:\u0000\t']
        source.tail_log_lines.side_effect = None
        source.tail_log_lines.return_value = escaped_lines
        escaped_limit = len(json.dumps(
            [escaped_lines[-1]], ensure_ascii=True, separators=(',', ':'),
        ).encode('utf-8'))
        with mock.patch.object(supervisor, 'MAX_LOG_TAIL_BYTES', escaped_limit):
            escaped = supervisor.send_managed_source_log_tail(
                self.metadata, 'sample', 2,
            )
        self.assertEqual(escaped['lines'], [escaped_lines[-1]])
        self.assertTrue(escaped['response_truncated'])
        self.assertLessEqual(len(json.dumps(
            escaped['lines'], ensure_ascii=True, separators=(',', ':'),
        ).encode('utf-8')), escaped_limit)

        compact_lines = ['x', 'y']
        source.tail_log_lines.return_value = compact_lines
        compact_limit = len(json.dumps(
            compact_lines, ensure_ascii=True, separators=(',', ':'),
        ).encode('utf-8'))
        self.assertGreater(len(json.dumps(
            compact_lines, ensure_ascii=True,
        ).encode('utf-8')), compact_limit)
        with mock.patch.object(supervisor, 'MAX_LOG_TAIL_BYTES', compact_limit):
            compact = supervisor.send_managed_source_log_tail(
                self.metadata, 'sample', 2,
            )
        self.assertEqual(compact['lines'], compact_lines)
        self.assertFalse(compact['response_truncated'])

        source.tail_log_lines.return_value = ['a' * 20, 'b' * 20]
        for phase in (PHASE_STOPPING, PHASE_FAILED_HOLD):
            with self.subTest(phase=phase):
                self.server.context['lifecycle_phase'] = phase
                self.server.context['activation_state'] = phase
                held = supervisor.send_managed_source_log_tail(
                    self.metadata, 'sample', 1,
                )
                self.assertEqual(held['lines'], ['b' * 20])
        self.server.context['lifecycle_phase'] = PHASE_ACTIVE
        self.server.context['activation_state'] = PHASE_ACTIVE
        source.tail_log_lines.side_effect = RuntimeError(
            'log-path-secret-must-not-leak',
        )
        with self.assertRaisesRegex(RuntimeError, '^managed source log tail failed$'):
            supervisor.send_managed_source_log_tail(
                self.metadata, 'sample', 2,
            )

    def test_pipeline_workers_expose_only_meaningful_typed_settings(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            log_dir = os.path.join(temp_dir, 'logs')
            state_dir = os.path.join(temp_dir, 'state')
            ensure_private_directory(log_dir, reject_reparse=True)
            ensure_private_directory(state_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            worker = supervisor.ManagedPipelineWorker(
                'result-ingester', config_path, str(APP_DIR), temp_dir,
                {
                    'log_dir': log_dir,
                    'state_dir': state_dir,
                    'defaults': {'enabled': True, 'restart': True},
                },
                {'enabled': True},
                dependency_gate=supervisor.DependencyGate(ready=True),
            )
            self.assertEqual(worker.structured_state()['allowed_actions'], [
                'start', 'stop', 'restart', 'pause', 'resume',
                'set-restart', 'set-restart-delay',
            ])
            self.server.managed_sources[:] = [worker]
            self.server.context.update({
                'lifecycle_phase': PHASE_ACTIVE,
                'activation_state': PHASE_ACTIVE,
                'start_gate_open': True,
            })
            with self.assertRaisesRegex(RuntimeError, 'unavailable'):
                supervisor.send_managed_source_action(
                    self.metadata, 'result-ingester', 'set-mode', mode='once',
                )

    def test_keychecks_reject_loop_mode_but_retain_periodic_controls(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            log_dir = os.path.join(temp_dir, 'logs')
            state_dir = os.path.join(temp_dir, 'state')
            ensure_private_directory(log_dir, reject_reparse=True)
            ensure_private_directory(state_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            source = supervisor.ManagedKeychecks(
                config_path, str(APP_DIR), temp_dir,
                {'log_dir': log_dir, 'state_dir': state_dir},
                {
                    'enabled': True,
                    'repeat': True,
                    'interval': 60,
                    'summary_tsv': os.path.join(temp_dir, 'summary.tsv'),
                },
                dependency_gate=supervisor.DependencyGate(ready=True),
            )
            self.server.managed_sources[:] = [source]
            self.server.context.update({
                'lifecycle_phase': PHASE_ACTIVE,
                'activation_state': PHASE_ACTIVE,
                'start_gate_open': True,
            })
            with self.assertRaisesRegex(RuntimeError, 'mode is unavailable'):
                supervisor.send_managed_source_action(
                    self.metadata, 'keychecks', 'set-mode', mode='loop',
                )
            supervisor.send_managed_source_action(
                self.metadata, 'keychecks', 'set-mode', mode='repeat',
            )
            self.assertTrue(source.once)
            self.assertTrue(source.repeat)

    def test_typed_mutation_rechecks_phase_after_acquiring_control_lock(self):
        source = mock.Mock(
            source='sample', manual_only=False, runtime_blocked=False,
            desired_state='running',
        )
        source.start.return_value = True
        self.server.managed_sources[:] = [source]
        self.server.context.update({
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
        })

        class StopOnEnter:
            def __enter__(lock_self):
                self.server.context['lifecycle_phase'] = PHASE_STOPPING
                self.server.context['activation_state'] = PHASE_STOPPING
                return lock_self

            def __exit__(lock_self, exc_type, exc, traceback):
                return False

        original_lock = self.server.lock
        self.server.lock = StopOnEnter()
        try:
            response = self.server.run_request({
                'schema': supervisor.CONTROL_SCHEMA,
                'instance_id': self.metadata['instance_id'],
                'token': self.metadata['token'],
                'action': 'managed-source-action',
                'source_id': 'sample',
                'source_action': 'start',
            })
        finally:
            self.server.lock = original_lock
        self.assertFalse(response['ok'])
        self.assertIn(PHASE_STOPPING, response['error'])
        source.start.assert_not_called()

    def test_typed_actions_require_exact_ids_fields_and_role_capabilities(self):
        source = mock.Mock()
        source.source = 'manual'
        source.manual_only = True
        source.structured_state.return_value = self.structured_source_state(
            'manual', 'docker-shadow', lifecycle_state='stopped',
            desired_state='stopped', process_state='stopped', pid=None,
            mode='manual', interval_seconds=0,
        )
        self.server.managed_sources[:] = [source]
        self.server.context.update({
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
        })

        with self.assertRaisesRegex(RuntimeError, 'unknown'):
            supervisor.send_managed_source_action(self.metadata, 'all', 'start')
        with self.assertRaisesRegex(RuntimeError, 'unavailable'):
            supervisor.send_managed_source_action(self.metadata, 'manual', 'restart')
        with self.assertRaisesRegex(RuntimeError, 'fields are invalid'):
            supervisor.send_managed_source_action(
                self.metadata, 'manual', 'start', interval_seconds=1,
            )
        for source_id in ('discovery-producer:github', 'discovery-producer:unknown'):
            with self.subTest(source_id=source_id), self.assertRaisesRegex(
                RuntimeError, 'unknown',
            ):
                supervisor.send_managed_source_action(
                    self.metadata, source_id, 'start',
                )
        for source_action in (None, True, 1, [], {}, 'start; status'):
            with self.subTest(source_action=source_action), self.assertRaisesRegex(
                RuntimeError, 'unsupported',
            ):
                supervisor.send_managed_source_action(
                    self.metadata, 'manual', source_action,
                )
        source.start.assert_not_called()

    def test_typed_clients_reject_malformed_authenticated_responses(self):
        class ResponseSocket:
            def __init__(self, response):
                self.response = json.dumps(response).encode('utf-8')

            def __enter__(self):
                return self

            def __exit__(self, exc_type, exc, traceback):
                return False

            def settimeout(self, _timeout):
                return None

            def sendall(self, _payload):
                return None

            def shutdown(self, _direction):
                return None

            def recv(self, _limit):
                response, self.response = self.response, b''
                return response

        envelope = {
            'schema': supervisor.CONTROL_SCHEMA,
            'instance_id': self.metadata['instance_id'],
            'ok': True,
            'result': {},
        }
        for response in (
            {**envelope, 'ok': 1},
            {**envelope, 'extra': True},
            {
                'schema': supervisor.CONTROL_SCHEMA,
                'instance_id': self.metadata['instance_id'],
                'ok': False,
                'error': {'unsafe': True},
            },
        ):
            with self.subTest(envelope=response), mock.patch.object(
                supervisor.socket, 'create_connection',
                return_value=ResponseSocket(response),
            ), self.assertRaisesRegex(ValueError, 'control response'):
                supervisor.send_control_request(self.metadata, 'handshake')

        valid_source = {
            'source_action': 'start', 'outcome': 'completed',
            'source': self.structured_source_state(),
        }
        malformed_source = (
            None,
            {},
            {**valid_source, 'source_action': 'stop'},
            {**valid_source, 'outcome': 'unknown'},
            {**valid_source, 'source': {'id': 'other'}},
            {**valid_source, 'extra': True},
        )
        for result in malformed_source:
            with self.subTest(source_result=result), mock.patch.object(
                supervisor, 'send_control_request', return_value=result,
            ), self.assertRaisesRegex(ValueError, 'managed source action response'):
                supervisor.send_managed_source_action(
                    self.metadata, 'sample', 'start',
                )

        valid_tail = {
            'source_id': 'sample', 'line_count': 1,
            'lines': ['line'], 'response_truncated': False,
        }
        malformed_tail = (
            {**valid_tail, 'line_count': 2},
            {**valid_tail, 'response_truncated': 0},
            {**valid_tail, 'extra': True},
        )
        for result in malformed_tail:
            with self.subTest(tail_result=result), mock.patch.object(
                supervisor, 'send_control_request', return_value=result,
            ), self.assertRaisesRegex(ValueError, 'managed source log tail'):
                supervisor.send_managed_source_log_tail(
                    self.metadata, 'sample', 1,
                )

        valid_dashboard = {
            'dashboard_action': 'start', 'outcome': 'completed',
            'dashboard': {
                'id': 'dashboard', 'status': 'healthy',
                'desired_state': 'running', 'process_state': 'running',
                'healthy': True, 'pid': 1234, 'restart_count': 0,
                'safe_error_category': '',
                'allowed_actions': ['start', 'stop', 'restart'],
            },
        }
        for result in (
            {**valid_dashboard, 'dashboard_action': 'stop'},
            {**valid_dashboard, 'outcome': 'unknown'},
            {**valid_dashboard, 'extra': True},
        ):
            with self.subTest(dashboard_result=result), mock.patch.object(
                supervisor, 'send_control_request', return_value=result,
            ), self.assertRaisesRegex(ValueError, 'dashboard action response'):
                supervisor.send_dashboard_action(self.metadata, 'start')

        runtime = {
            'pid': 1234, 'phase': PHASE_ACTIVE, 'manages_postgres': True,
            'start_gate_open': True, 'shutdown_requested': False,
            'runtime_failed': False,
        }
        valid_snapshot = {
            'snapshot_schema': supervisor.RUNTIME_SNAPSHOT_SCHEMA,
            'runtime': runtime,
            'postgres': {
                'state': 'READY', 'ready': True, 'failures': 0,
                'safe_error_category': '',
            },
            'dashboard': valid_dashboard['dashboard'],
            'sources': [self.structured_source_state()],
            'pipeline': {
                'ingester_ready': True, 'projector_ready': True,
                'cutover_ready': True,
                'ingester_state': 'active', 'projector_state': 'active',
                'bundle_items': 0, 'bundle_bytes': 0,
                'projection_items': 0, 'projection_bytes': 0,
                'keycheck_items': 0, 'keycheck_bytes': 0,
                'quarantine_items': 0, 'quarantine_bytes': 0,
            },
            'scan_workers': {
                'active': 0, 'limit': 1, 'base_active': 0, 'base_limit': 1,
                'bonus_active': 0, 'bonus_limit': 0, 'trufflehog': 0,
                'sources': {},
            },
        }
        for result in (
            {**valid_snapshot, 'extra': True},
            {**valid_snapshot, 'runtime': {**runtime, 'pid': True}},
            {**valid_snapshot, 'sources': [
                {**self.structured_source_state(), 'unknown': 'value'},
            ]},
        ):
            with self.subTest(snapshot=result), mock.patch.object(
                supervisor, 'send_control_request', return_value=result,
            ), self.assertRaisesRegex(ValueError, 'structured supervisor snapshot'):
                supervisor.get_runtime_snapshot(self.metadata)

    def test_many_idle_clients_use_bounded_resources_and_authenticated_control_stays_responsive(self):
        idle = []
        try:
            for _ in range(supervisor.MAX_CONTROL_PENDING_SOCKETS * 2):
                client = socket.create_connection(self.server.server_address, timeout=2)
                idle.append(client)
            deadline = time.monotonic() + 2
            while time.monotonic() < deadline and self.server.pending_control_connections == 0:
                time.sleep(0.01)
            started = time.monotonic()
            result = supervisor.send_control_request(self.metadata, 'handshake', timeout=3)
            elapsed = time.monotonic() - started
            self.assertEqual(result['instance_id'], self.metadata['instance_id'])
            self.assertLess(elapsed, 2)
            self.assertLessEqual(self.server.pending_control_connections, supervisor.MAX_CONTROL_PENDING_SOCKETS)
            self.assertLessEqual(self.server.active_control_workers, supervisor.MAX_CONTROL_WORKERS)
            control_threads = [
                thread for thread in threading.enumerate()
                if thread.name.startswith('supervisor-control')
            ]
            self.assertLessEqual(len(control_threads), supervisor.MAX_CONTROL_WORKERS + 1)
        finally:
            for client in idle:
                try:
                    client.close()
                except OSError:
                    pass

    def test_inactive_control_plane_requires_activation_before_runtime_requests(self):
        source = mock.Mock()
        activation_event = threading.Event()
        context = {
            'shutdown_event': threading.Event(),
            'with_postgres': True,
            'activation_state': PHASE_ACTIVATING,
            'lifecycle_phase': PHASE_ACTIVATING,
            'start_gate_open': False,
            'activation_event': activation_event,
            'activation_lock': threading.RLock(),
        }
        server = supervisor.SupervisorControlServer(
            ('127.0.0.1', 0), [source], context, threading.RLock(), 'inactive-instance', 'a' * 48,
        )
        thread = threading.Thread(target=server.serve_forever, daemon=True)
        thread.start()
        metadata = {
            'instance_id': 'inactive-instance',
            'token': 'a' * 48,
            'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
        }
        try:
            handshake = supervisor.send_control_request(metadata, 'handshake')
            self.assertEqual(handshake['activation_state'], PHASE_ACTIVATING)
            with self.assertRaisesRegex(RuntimeError, 'ACTIVATING'):
                supervisor.send_control_request(metadata, 'snapshot')
            source.poll.assert_not_called()

            activated = supervisor.send_control_request(metadata, 'activate')
            self.assertEqual(activated['activation_state'], PHASE_ACTIVE)
            self.assertTrue(activation_event.is_set())
            self.assertEqual(
                supervisor.send_control_request(metadata, 'handshake')['activation_state'],
                PHASE_ACTIVE,
            )
        finally:
            server.shutdown()
            server.server_close()

    def test_activating_phase_never_ticks_lifecycle_objects(self):
        controller = mock.Mock()
        dashboard = mock.Mock()
        gate = mock.Mock()
        context = {
            'lifecycle_phase': PHASE_ACTIVATING,
            'activation_state': PHASE_ACTIVATING,
            'start_gate_open': False,
            'postgres_controller': controller,
            'dashboard_manager': dashboard,
            'dependency_gate': gate,
        }
        supervisor.tick_supervisor_runtime(context)
        controller.tick.assert_not_called()
        dashboard.poll.assert_not_called()
        gate.set_ready.assert_not_called()

    def test_runtime_tick_honors_low_overhead_status_intervals(self):
        source_gate = mock.Mock(ready=False)
        context = {
            'lifecycle_phase': PHASE_ACTIVE,
            'start_gate_open': True,
            'source_dependency_gate': source_gate,
            'supervisor_config': {
                'authority_check_interval_sec': 5,
                'pipeline_status_refresh_sec': 2,
            },
        }
        with (
            mock.patch.object(supervisor.time, 'monotonic', return_value=100),
            mock.patch.object(supervisor, 'check_runtime_authority', return_value=True),
        ):
            supervisor.tick_supervisor_runtime(context)

        self.assertEqual(context['next_authority_check_at'], 105)
        self.assertEqual(context['_pipeline_status_refresh_at'], 102)
        source_gate.set_ready.assert_not_called()

    def test_database_gate_transition_defers_source_admission_one_tick(self):
        controller = mock.Mock(ready=True)
        controller.snapshot.return_value = {
            'state': 'READY', 'ready': True, 'failures': 0, 'detail': '',
        }
        dependency_gate = supervisor.DependencyGate(ready=False)
        source_gate = supervisor.DependencyGate(ready=False)
        context = {
            'lifecycle_phase': PHASE_ACTIVE,
            'start_gate_open': True,
            'postgres_controller': controller,
            'dependency_gate': dependency_gate,
            'source_dependency_gate': source_gate,
            'next_authority_check_at': 101,
        }
        ready = {
            'ingester_ready': True,
        }
        with mock.patch.object(supervisor.time, 'monotonic', return_value=100), \
                mock.patch.object(supervisor, 'pipeline_status_snapshot',
                                  return_value=ready) as pipeline:
            supervisor.tick_supervisor_runtime(context)
            pipeline.assert_not_called()
            self.assertTrue(dependency_gate.ready)
            self.assertFalse(source_gate.ready)
            self.assertTrue(context['_defer_pipeline_status_until_next_tick'])

            supervisor.tick_supervisor_runtime(context, pipeline_snapshot=ready)

        pipeline.assert_not_called()
        self.assertTrue(source_gate.ready)
        self.assertNotIn('_defer_pipeline_status_until_next_tick', context)

    def test_deferred_pipeline_status_never_opens_database(self):
        context = {'_defer_pipeline_status_until_next_tick': True}
        with mock.patch.object(supervisor, 'connect_postgres') as connect:
            snapshot = supervisor.pipeline_status_snapshot(context)
        self.assertFalse(snapshot['ingester_ready'])
        self.assertEqual(snapshot['ingester_state'], 'starting')
        connect.assert_not_called()

    def test_cached_pipeline_status_never_opens_database(self):
        context = {}
        with mock.patch.object(supervisor, 'connect_postgres') as connect:
            snapshot = supervisor.pipeline_status_snapshot(
                context, allow_refresh=False,
            )
        self.assertFalse(snapshot['ingester_ready'])
        self.assertEqual(snapshot['ingester_state'], 'starting')
        connect.assert_not_called()

    def test_command_failure_is_not_reported_as_transport_success(self):
        source = mock.Mock(source='sample', runtime_blocked=False, last_action_error='owned process remained live')
        source.stop.return_value = False
        server = supervisor.SupervisorControlServer(
            ('127.0.0.1', 0), [source], {'shutdown_event': threading.Event(), 'with_postgres': False},
            threading.RLock(), 'failure-instance', 'f' * 48,
        )
        thread = threading.Thread(target=server.serve_forever, daemon=True)
        thread.start()
        metadata = {
            'instance_id': 'failure-instance',
            'token': 'f' * 48,
            'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
        }
        try:
            with self.assertRaisesRegex(RuntimeError, 'stop failed'):
                supervisor.send_control_command(metadata, 'stop sample')
        finally:
            server.shutdown()
            server.server_close()

    def test_control_handler_returns_error_when_start_raises(self):
        source = mock.Mock(source='sample', runtime_blocked=False, last_action_error='')
        source.start.side_effect = RuntimeError('owned launch failed')
        server = supervisor.SupervisorControlServer(
            ('127.0.0.1', 0), [source], {'shutdown_event': threading.Event(), 'with_postgres': False},
            threading.RLock(), 'raising-instance', 'e' * 48,
        )
        threading.Thread(target=server.serve_forever, daemon=True).start()
        metadata = {
            'instance_id': 'raising-instance',
            'token': 'e' * 48,
            'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
        }
        try:
            with self.assertRaisesRegex(RuntimeError, 'owned launch failed'):
                supervisor.send_control_command(metadata, 'start sample')
        finally:
            server.shutdown()
            server.server_close()

    def test_shutdown_phase_closes_start_and_snapshot_gates_before_ack(self):
        entered = threading.Event()
        release = threading.Event()

        class BarrierEvent(threading.Event):
            def set(self):
                super().set()
                entered.set()
                release.wait(5)

        source = mock.Mock(source='sample', runtime_blocked=False, last_action_error='')
        context = {
            'shutdown_event': BarrierEvent(),
            'with_postgres': False,
            'lifecycle_phase': PHASE_ACTIVE,
            'activation_state': PHASE_ACTIVE,
            'start_gate_open': True,
        }
        server = supervisor.SupervisorControlServer(
            ('127.0.0.1', 0), [source], context, threading.RLock(), 'race-instance', 'r' * 48,
        )
        threading.Thread(target=server.serve_forever, daemon=True).start()
        metadata = {
            'instance_id': 'race-instance',
            'token': 'r' * 48,
            'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
        }
        results = {}

        def shutdown_request():
            results['shutdown'] = supervisor.send_control_request(metadata, 'shutdown')

        def late_start():
            try:
                supervisor.send_control_command(metadata, 'start sample')
            except RuntimeError as exc:
                results['start_error'] = str(exc)

        shutdown_thread = threading.Thread(target=shutdown_request)
        shutdown_thread.start()
        self.assertTrue(entered.wait(2))
        start_thread = threading.Thread(target=late_start)
        start_thread.start()
        release.set()
        shutdown_thread.join(5)
        start_thread.join(5)
        try:
            self.assertIn('shutdown requested', results['shutdown'])
            self.assertIn('STOPPING', results['start_error'])
            self.assertEqual(context['lifecycle_phase'], PHASE_STOPPING)
            source.start.assert_not_called()
            source.row.return_value = ['sample', 'failed', '-', 'loop', '-', '-', '-', '0/0', '-', 'log', 'live', 'stopped']
            snapshot = supervisor.send_control_request(metadata, 'snapshot')
            self.assertEqual(snapshot['activation_state'], PHASE_STOPPING)
            source.poll.assert_not_called()
        finally:
            server.shutdown()
            server.server_close()


class InstanceAuthorityTests(unittest.TestCase):
    def test_private_metadata_tracks_two_phase_activation_state(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            instance_path = os.path.join(temp_dir, 'supervisor.instance.json')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            metadata = build_instance_metadata(
                'nonce', supervisor.__file__, config_path, '127.0.0.1', 12345, False,
            )
            self.assertEqual(metadata['activation_state'], PHASE_ACTIVATING)
            write_instance_metadata(instance_path, metadata)
            updated = supervisor_instance.update_instance_activation(
                instance_path, metadata['instance_id'], PHASE_ACTIVE,
            )
            self.assertEqual(updated['activation_state'], PHASE_ACTIVE)
            self.assertEqual(
                supervisor_instance.load_instance_metadata(instance_path)['activation_state'],
                PHASE_ACTIVE,
            )

    def test_reused_pid_creation_identity_is_refused(self):
        identity = current_process_identity()
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir)
            supervisor_path = os.path.join(temp_dir, 'supervisor.py')
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(supervisor_path).write_text('pass\n', encoding='ascii')
            Path(config_path).write_text('{}\n', encoding='ascii')
            metadata = build_instance_metadata(
                'nonce', supervisor_path, config_path, '127.0.0.1', 12345, False,
                identity=identity,
            )
            metadata['process_creation_time'] += '-reused'
            with self.assertRaises(ProcessIdentityError):
                verify_instance_process(metadata)

    def test_duplicate_verified_launch_metadata_is_detected(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir)
            config_path = os.path.join(temp_dir, 'config.yaml')
            instance_path = os.path.join(temp_dir, 'supervisor.instance.json')
            Path(config_path).write_text('{}\n', encoding='ascii')
            server = supervisor.SupervisorControlServer(
                ('127.0.0.1', 0), [], {'shutdown_event': threading.Event(), 'with_postgres': False},
                threading.RLock(), 'duplicate-instance', 'z' * 48,
            )
            thread = threading.Thread(target=server.serve_forever, daemon=True)
            thread.start()
            try:
                metadata = build_instance_metadata(
                    'launch', supervisor.__file__, config_path, '127.0.0.1', server.server_address[1], False,
                    instance_id='duplicate-instance', token='z' * 48,
                )
                write_instance_metadata(instance_path, metadata)
                retained_process = mock.Mock()
                with mock.patch.object(supervisor, 'verify_instance_process', return_value=retained_process):
                    state, found, retained = supervisor.inspect_existing_instance(instance_path, config_path)
                self.assertEqual(state, 'verified')
                self.assertEqual(found['instance_id'], 'duplicate-instance')
                retained.close()
            finally:
                server.shutdown()
                server.server_close()

    def test_coordinated_stop_uses_shutdown_request_and_retained_handle(self):
        metadata = {
            'instance_id': 'coordinated',
            'manages_postgres': True,
        }

        class Retained:
            def __init__(self):
                self.waited = None
                self.closed = False

            def wait(self, timeout):
                self.waited = timeout
                return True

            def close(self):
                self.closed = True

            def exit_code(self):
                return 0

        retained = Retained()
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor, 'load_instance_metadata', return_value=metadata), \
                mock.patch.object(supervisor, 'verify_instance_process', return_value=retained), \
                mock.patch.object(supervisor, 'send_control_request', return_value='coordinated shutdown requested') as send, \
                mock.patch.object(supervisor, 'remove_instance_if_matches') as remove:
            result = supervisor.stop_background(
                os.path.join(temp_dir, 'config.yaml'), temp_dir,
                private_background_config(temp_dir, background_shutdown_timeout_sec=7),
                with_postgres=True,
            )

        self.assertTrue(result)
        self.assertEqual(send.call_args.args[1], 'shutdown')
        self.assertTrue(send.call_args.kwargs['with_postgres'])
        self.assertEqual(retained.waited, 7)
        self.assertTrue(retained.closed)
        remove.assert_called_once()
        self.assertEqual(remove.call_args.args[1], 'coordinated')
        self.assertTrue(remove.call_args.kwargs['lock_path'].endswith('supervisor.lock'))

    def test_identity_mismatch_never_sends_shutdown(self):
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor, 'load_instance_metadata', return_value={'pid': 123}), \
                mock.patch.object(supervisor, 'verify_instance_process', side_effect=ProcessIdentityError('reused PID')), \
                mock.patch.object(supervisor, 'send_control_request') as send:
            result = supervisor.stop_background(
                os.path.join(temp_dir, 'config.yaml'), temp_dir, private_background_config(temp_dir),
            )
        self.assertFalse(result)
        send.assert_not_called()

    def test_nonzero_child_exit_retains_metadata_and_fails_stop(self):
        metadata = {'instance_id': 'failed-stop', 'manages_postgres': True}
        retained = mock.Mock()
        retained.wait.return_value = True
        retained.exit_code.return_value = 7
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor, 'load_instance_metadata', return_value=metadata), \
                mock.patch.object(supervisor, 'verify_instance_process', return_value=retained), \
                mock.patch.object(supervisor, 'send_control_request', return_value='shutdown requested'), \
                mock.patch.object(supervisor, 'remove_instance_if_matches') as remove:
            result = supervisor.stop_background(
                os.path.join(temp_dir, 'config.yaml'),
                temp_dir,
                private_background_config(temp_dir),
                with_postgres=True,
            )
        self.assertFalse(result)
        remove.assert_not_called()

    def test_config_drift_allows_only_command_bound_authenticated_shutdown_verification(self):
        class Retained:
            def __init__(self, args):
                self.args = args
                self.closed = False

            def command_line(self):
                return self.args

            def close(self):
                self.closed = True

        identity = current_process_identity()
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir)
            supervisor_path = os.path.join(temp_dir, 'supervisor.py')
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(supervisor_path).write_text('pass\n', encoding='ascii')
            Path(config_path).write_text('version: 1\n', encoding='ascii')
            metadata = build_instance_metadata(
                'nonce', supervisor_path, config_path, '127.0.0.1', 12345, False,
                identity=identity,
            )
            Path(config_path).write_text('version: 2\n', encoding='ascii')
            args = [sys.executable, supervisor_path, '--config', config_path, '--background-child']
            with mock.patch.object(supervisor_instance, 'verify_retained_process', return_value=Retained(args)):
                with self.assertRaisesRegex(InstanceMetadataError, 'config hash drifted'):
                    verify_instance_process(metadata, supervisor_path, config_path)
                retained = verify_instance_process(metadata, supervisor_path, config_path, allow_config_drift=True)
            self.assertFalse(retained.closed)
            retained.close()
            unrelated = Retained([sys.executable, '-c', 'print(1)'])
            with mock.patch.object(supervisor_instance, 'verify_retained_process', return_value=unrelated):
                with self.assertRaisesRegex(InstanceMetadataError, 'not a background supervisor'):
                    verify_instance_process(metadata, supervisor_path, config_path, allow_config_drift=True)
            self.assertTrue(unrelated.closed)

    def test_metadata_publication_never_replaces_existing_instance(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            path = os.path.join(temp_dir, 'supervisor.instance.json')
            first = build_instance_metadata('one', supervisor.__file__, config_path, '127.0.0.1', 12345, False)
            second = build_instance_metadata('two', supervisor.__file__, config_path, '127.0.0.1', 12346, False)
            write_instance_metadata(path, first)
            with self.assertRaises(FileExistsError):
                write_instance_metadata(path, second)

    def test_supervisor_script_hash_drift_never_grants_control(self):
        identity = current_process_identity()
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir)
            supervisor_path = os.path.join(temp_dir, 'supervisor.py')
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(supervisor_path).write_text('version = 1\n', encoding='ascii')
            Path(config_path).write_text('{}\n', encoding='ascii')
            metadata = build_instance_metadata(
                'nonce', supervisor_path, config_path, '127.0.0.1', 12345, False,
                identity=identity,
            )
            Path(supervisor_path).write_text('version = 2\n', encoding='ascii')
            with self.assertRaisesRegex(InstanceMetadataError, 'script hash'):
                verify_instance_process(
                    metadata,
                    supervisor_path,
                    config_path,
                    allow_config_drift=True,
                )

    def test_lifetime_lock_excludes_a_second_process(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            control_dir = os.path.join(temp_dir, 'control')
            ensure_private_directory(control_dir, reject_reparse=True)
            instance_path = os.path.join(control_dir, 'supervisor.instance.json')
            second_instance_path = os.path.join(temp_dir, 'custom', 'other.instance.json')
            singleton_path = os.path.join(control_dir, 'supervisor.lock')
            lock = SupervisorInstanceLock(instance_path, lock_path=singleton_path).acquire()
            code = (
                'import sys; sys.path.insert(0, sys.argv[1]); '
                'from supervisor_instance import SupervisorInstanceLock, InstanceLockError; '
                'lock=SupervisorInstanceLock(sys.argv[2], lock_path=sys.argv[3]); '
                '\ntry: lock.acquire()\nexcept InstanceLockError: raise SystemExit(3)\n'
                'lock.release(); raise SystemExit(0)'
            )
            try:
                blocked = subprocess.run(
                    [sys.executable, '-c', code, str(APP_DIR), second_instance_path, singleton_path],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    timeout=15,
                    check=False,
                )
                self.assertEqual(blocked.returncode, 3, blocked.stderr.decode(errors='replace'))
            finally:
                lock.release()
            admitted = subprocess.run(
                [sys.executable, '-c', code, str(APP_DIR), second_instance_path, singleton_path],
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                timeout=15,
                check=False,
            )
            self.assertEqual(admitted.returncode, 0, admitted.stderr.decode(errors='replace'))

    def test_concurrent_unmanaged_background_starts_are_both_rejected(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            config_path, paths = prepare_supervisor_fixture(temp_dir)
            runtime_dir = paths['runtime_dir']
            instance_paths = [
                os.path.join(runtime_dir, 'control', 'one.instance.json'),
                os.path.join(runtime_dir, 'control', 'two.instance.json'),
            ]
            commands = [
                [
                    sys.executable,
                    str(APP_DIR / 'supervisor.py'),
                    '--config', config_path,
                    '--background',
                    '--instance-file', instance_path,
                ]
                for instance_path in instance_paths
            ]
            environment = os.environ.copy()
            environment['SCANNER_DB_URL'] = 'postgresql://truf:fixture@127.0.0.1:5432/truf'
            environment['DATABASE_URL'] = environment['SCANNER_DB_URL']
            environment.pop('TRUF_MANAGED_POSTGRES_DSN', None)
            parents = [
                subprocess.Popen(
                    command,
                    cwd=temp_dir,
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                    env=environment,
                )
                for command in commands
            ]
            results = []
            try:
                for parent in parents:
                    output, _ = parent.communicate(timeout=20)
                    results.append((parent.returncode, output.decode(errors='replace')))
                self.assertTrue(all(code != 0 for code, _ in results), results)
                self.assertTrue(all('unmanaged PostgreSQL mutation is retired' in output for _, output in results))
                self.assertFalse(any(os.path.exists(path) for path in instance_paths))
            finally:
                for parent in parents:
                    if parent.poll() is None:
                        parent.terminate()
                        parent.wait(timeout=5)

    def test_background_custom_port_still_rejects_unmanaged_mutation(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            config_path, paths = prepare_supervisor_fixture(temp_dir, control_port=0)
            instance_path = os.path.join(paths['control_dir'], 'custom.instance.json')
            environment = os.environ.copy()
            environment['SCANNER_DB_URL'] = 'postgresql://truf:fixture@127.0.0.1:5432/truf'
            environment['DATABASE_URL'] = environment['SCANNER_DB_URL']
            environment.pop('TRUF_MANAGED_POSTGRES_DSN', None)
            common = [sys.executable, str(APP_DIR / 'supervisor.py'), '--config', config_path]
            started = subprocess.run(
                [*common, '--background', '--instance-file', instance_path],
                cwd=temp_dir,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                env=environment,
                timeout=20,
                check=False,
            )
            output = started.stdout.decode(errors='replace')
            self.assertNotEqual(started.returncode, 0, output)
            self.assertIn('unmanaged PostgreSQL mutation is retired', output)
            self.assertFalse(os.path.exists(instance_path))

    def test_startup_rollback_terminates_only_the_exact_child_when_auth_is_unavailable(self):
        child = FakeOwnedProcess()
        child.pid = 50001
        candidate = {'pid': 99999, 'launch_nonce': 'other', 'instance_id': 'other'}
        with mock.patch.object(supervisor, 'send_control_request') as send, \
                mock.patch.object(supervisor, 'remove_instance_if_matches') as remove:
            outcome = supervisor._rollback_background_start(
                child, candidate, 'unused.json', 'expected', False,
            )
        self.assertIn('unpublished child', outcome)
        self.assertTrue(child.terminated)
        send.assert_not_called()
        remove.assert_not_called()

    def test_lost_activation_response_uses_coordinated_shutdown_without_force(self):
        child = FakeOwnedProcess()
        child.pid = 50002
        candidate = {
            'pid': child.pid,
            'launch_nonce': 'expected',
            'instance_id': 'activated',
            'activation_state': PHASE_ACTIVATING,
        }

        def request(_metadata, action, **_kwargs):
            if action == 'activation':
                return {'activation_state': PHASE_ACTIVE}
            if action == 'shutdown':
                child.returncode = 0
                return 'coordinated shutdown requested'
            raise AssertionError(action)

        with mock.patch.object(supervisor, 'send_control_request', side_effect=request), \
                mock.patch.object(supervisor, 'remove_instance_if_matches', return_value=True):
            outcome = supervisor._rollback_background_start(
                child, candidate, 'unused.json', 'expected', True,
                activation_attempted=True, shutdown_timeout=37,
            )
        self.assertIn('authenticated coordinated shutdown', outcome)
        self.assertFalse(child.terminated)

    def test_exact_candidate_never_force_terminates_when_parent_flag_says_not_activated(self):
        child = FakeOwnedProcess()
        child.pid = 50004
        candidate = {
            'pid': child.pid,
            'launch_nonce': 'expected',
            'instance_id': 'exact',
            'activation_state': PHASE_ACTIVATING,
        }

        def request(_metadata, action, **_kwargs):
            if action == 'activation':
                return {'activation_state': PHASE_ACTIVE}
            if action == 'shutdown':
                child.returncode = 0
                return 'shutdown requested'
            raise AssertionError(action)

        with mock.patch.object(supervisor, 'send_control_request', side_effect=request), \
                mock.patch.object(supervisor, 'remove_instance_if_matches', return_value=True):
            outcome = supervisor._rollback_background_start(
                child,
                candidate,
                'unused.json',
                'expected',
                True,
                activation_attempted=False,
                shutdown_timeout=29,
            )
        self.assertIn('authenticated coordinated shutdown', outcome)
        self.assertFalse(child.terminated)

    def test_uncertain_activation_timeout_never_force_terminates_postgres_root(self):
        class TimedOutChild(FakeOwnedProcess):
            def __init__(self):
                super().__init__()
                self.wait_timeouts = []

            def wait(self, timeout=None):
                self.wait_timeouts.append(timeout)
                raise subprocess.TimeoutExpired('supervisor', timeout)

        child = TimedOutChild()
        child.pid = 50003
        candidate = {
            'pid': child.pid,
            'launch_nonce': 'expected',
            'instance_id': 'uncertain',
            'activation_state': PHASE_ACTIVATING,
        }
        with mock.patch.object(supervisor, 'send_control_request', side_effect=TimeoutError('response lost')):
            outcome = supervisor._rollback_background_start(
                child, candidate, 'unused.json', 'expected', True,
                activation_attempted=True, shutdown_timeout=41,
            )
        self.assertIn('left running', outcome)
        self.assertFalse(child.terminated)
        self.assertEqual(child.wait_timeouts, [41.0])


class ScanWorkerStatusTests(unittest.TestCase):
    def test_table_signature_does_not_parse_source_logs_or_state(self):
        process = mock.Mock()
        process.pid = 123
        process.poll.return_value = None
        source = SimpleNamespace(
            source='github',
            status='running',
            desired_state='running',
            process=process,
            last_exit_code=None,
            restarts=0,
            restart_streak=0,
            runtime_blocked=False,
            last_action_error='',
            row=mock.Mock(side_effect=AssertionError('row must not be rendered for signatures')),
        )
        signature = supervisor.table_signature([source])
        self.assertEqual(signature[0][:4], ('github', 'running', 'running', 123))
        source.row.assert_not_called()

    def test_snapshot_reports_authoritative_scan_slot_leases_by_source(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            db_path = os.path.join(temp_dir, 'scan_limiter.db')
            connection = sqlite3.connect(db_path)
            try:
                connection.execute('CREATE TABLE scan_slots (owner_source TEXT, child_executable TEXT)')
                connection.executemany(
                    'INSERT INTO scan_slots(owner_source, child_executable) VALUES (?, ?)',
                    [('github', r'C:\Tools\trufflehog.exe'), ('github', None), ('npm', r'C:\Tools\trufflehog.exe')],
                )
                connection.commit()
            finally:
                connection.close()

            snapshot = supervisor.scan_worker_snapshot({'global': {
                'max_active_scans': 6,
                'opportunistic_scan_slots': 1,
                'scan_limiter_db': db_path,
            }})

        self.assertEqual(snapshot['active'], 3)
        self.assertEqual(snapshot['limit'], 7)
        self.assertEqual(snapshot['base_active'], 3)
        self.assertEqual(snapshot['bonus_active'], 0)
        self.assertEqual(snapshot['trufflehog'], 2)
        self.assertEqual(snapshot['sources'], {'github': 2, 'npm': 1})
        self.assertEqual(snapshot['detail'], '')

    def test_runtime_table_includes_scan_worker_activity(self):
        context = {'config': {'global': {'max_active_scans': 6}}}
        with mock.patch.object(supervisor, 'scan_worker_snapshot', return_value={
            'active': 4,
            'limit': 6,
            'base_active': 3,
            'base_limit': 5,
            'bonus_active': 1,
            'bonus_limit': 1,
            'trufflehog': 3,
            'sources': {'github': 1, 'npm': 3},
            'detail': '',
        }):
            lines = supervisor.build_runtime_table_lines([], context=context)
        self.assertEqual(
            lines[0],
            'Scan workers: active=4/6 base=3/5 bonus=1/1 trufflehog=3 staging=1 sources=github:1, npm:3',
        )


class DependencyFailureTests(unittest.TestCase):
    def test_failed_manual_source_stop_remains_unsafe_during_dependency_loss(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            source.process = UnkillableProcess()
            source.desired_state = 'running'
            source.status = 'running'
            self.assertFalse(source.stop())
            self.assertEqual(source.desired_state, 'stopped')
            with self.assertRaises(supervisor.DependencyStopError):
                gate.set_ready(False)
            self.assertTrue(source.is_running())

    def test_uncancellable_source_outage_escalates_and_stays_unsafe(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            source.process = UnkillableProcess()
            source.desired_state = 'running'
            source.status = 'running'
            with self.assertRaises(supervisor.DependencyStopError):
                gate.set_ready(False)
            self.assertTrue(source.is_running())
            self.assertEqual(source.status, 'failed')
            self.assertTrue(gate.stop_failures)

    def test_uncancellable_dashboard_outage_escalates(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            logs = os.path.join(temp_dir, 'logs')
            ensure_private_directory(logs, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            gate = supervisor.DependencyGate(ready=True)
            manager = supervisor.ManagedDashboard(
                config_path,
                temp_dir,
                {'log_dir': logs, 'dashboard': {'enabled': True}},
                temp_dir,
                temp_dir,
                dependency_gate=gate,
                process_factory=FakeOwnedProcess,
            )
            manager.process = UnkillableProcess()
            manager.desired_state = 'running'
            with self.assertRaises(supervisor.DependencyStopError):
                gate.set_ready(False)
            self.assertTrue(manager.fatal_stop_failure)
            self.assertIsNotNone(manager.process)
            manager.close()


class AuthorityAndManualCommandTests(unittest.TestCase):
    def test_config_barrier_during_activating_fails_without_lifecycle_publication(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            script_path = os.path.join(temp_dir, 'supervisor.py')
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(script_path).write_text('version = 1\n', encoding='ascii')
            Path(config_path).write_text('version: 1\n', encoding='ascii')
            harden_private_file(script_path)
            harden_private_file(config_path)
            callback = mock.Mock()
            context = {
                'authority': supervisor.runtime_authority(config_path, script_path),
                'shutdown_event': threading.Event(),
                'lifecycle_phase': PHASE_ACTIVATING,
                'activation_state': PHASE_ACTIVATING,
                'start_gate_open': False,
                'activation_callback': callback,
            }
            server = supervisor.SupervisorControlServer(
                ('127.0.0.1', 0), [], context, threading.RLock(), 'barrier-instance', 'b' * 48,
            )
            threading.Thread(target=server.serve_forever, daemon=True).start()
            metadata = {
                'instance_id': 'barrier-instance',
                'token': 'b' * 48,
                'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
            }
            Path(config_path).write_text('version: 2\n', encoding='ascii')
            try:
                with self.assertRaisesRegex(RuntimeError, 'config authority hash drifted'):
                    supervisor.send_control_request(metadata, 'activate')
                callback.assert_not_called()
                self.assertEqual(context['lifecycle_phase'], PHASE_ACTIVATING)
                self.assertFalse(context['start_gate_open'])
            finally:
                server.shutdown()
                server.server_close()

    def test_automatic_source_start_rejects_config_drift_before_launch(self):
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            gate = supervisor.DependencyGate(ready=True)
            source = source_fixture(temp_dir, gate)
            authority = supervisor.runtime_authority(source.config_path)
            context = {
                'authority': authority,
                'dependency_gate': gate,
                'shutdown_event': threading.Event(),
                'supervisor_config': {},
            }
            source.authority_check = lambda: supervisor.check_runtime_authority(context)
            Path(source.config_path).write_text('changed: true\n', encoding='ascii')
            self.assertFalse(source.start(force=True))
            self.assertIn('authority', source.last_action_error)
            self.assertIn('config authority hash drifted', context['authority_drift'])
            owned.assert_not_called()

    def test_restart_does_not_launch_when_stop_failed(self):
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(supervisor, 'OwnedProcess', side_effect=FakeOwnedProcess) as owned:
            source = source_fixture(temp_dir, supervisor.DependencyGate(ready=True))
            source.process = UnkillableProcess()
            source.desired_state = 'running'
            self.assertFalse(source.restart_now())
            self.assertTrue(source.process.poll() is None)
            owned.assert_not_called()

    def test_drift_rejects_mutation_but_authenticated_shutdown_remains_available(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            script_path = os.path.join(temp_dir, 'supervisor.py')
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(script_path).write_text('pass\n', encoding='ascii')
            Path(config_path).write_text('version: 1\n', encoding='ascii')
            harden_private_file(script_path)
            harden_private_file(config_path)
            shutdown_event = threading.Event()
            context = {
                'authority': supervisor.runtime_authority(config_path, script_path),
                'shutdown_event': shutdown_event,
                'with_postgres': False,
                'supervisor_config': {'authority_drift_shutdown_grace_sec': 30},
            }
            Path(config_path).write_text('version: 2\n', encoding='ascii')
            server = supervisor.SupervisorControlServer(
                ('127.0.0.1', 0), [], context, threading.RLock(), 'drift-instance', 'd' * 48,
            )
            threading.Thread(target=server.serve_forever, daemon=True).start()
            metadata = {
                'instance_id': 'drift-instance',
                'token': 'd' * 48,
                'control': {'host': '127.0.0.1', 'port': server.server_address[1]},
            }
            try:
                with self.assertRaisesRegex(RuntimeError, 'config authority hash drifted'):
                    supervisor.send_control_command(metadata, 'set all interval 10')
                self.assertFalse(shutdown_event.is_set())
                self.assertIn('shutdown requested', supervisor.send_control_request(metadata, 'shutdown'))
                self.assertTrue(shutdown_event.wait(1))
            finally:
                server.shutdown()
                server.server_close()

    def test_live_reload_explicitly_requires_supervisor_restart(self):
        outcome = supervisor.run_control_command('reload', [], {}, threading.RLock())
        self.assertFalse(outcome['success'])
        self.assertIn('Live reload is disabled', outcome['output'])


class CoordinatedShutdownTests(unittest.TestCase):
    def test_authority_drift_ticks_late_controller_results_and_reaches_shutdown_grace(self):
        shutdown_event = threading.Event()
        controller = mock.Mock()
        context = {
            'lifecycle_phase': PHASE_ACTIVE,
            'start_gate_open': True,
            'postgres_controller': controller,
            'shutdown_event': shutdown_event,
            'supervisor_config': {'authority_drift_shutdown_grace_sec': 5},
        }
        with mock.patch.object(supervisor.time, 'monotonic', side_effect=[0, 0, 6]):
            supervisor.inhibit_for_authority_drift(context, 'authority drift')
            supervisor.tick_supervisor_runtime(context)
        controller.inhibit_lifecycle.assert_called_once()
        controller.tick.assert_called_once()
        self.assertTrue(shutdown_event.is_set())

    def test_child_failure_prevents_postgres_stop_request(self):
        source = mock.Mock(source='source')
        source.stop.return_value = False
        controller = mock.Mock()
        context = {'postgres_controller': controller, 'supervisor_config': {}}
        self.assertFalse(supervisor.coordinated_shutdown([source], context))
        controller.request_stop.assert_not_called()

    def test_live_dashboard_prevents_postgres_stop_even_if_stop_claims_success(self):
        source = mock.Mock(source='source')
        source.stop.return_value = True
        source.is_running.return_value = False
        dashboard = mock.Mock()
        dashboard.stop.return_value = True
        dashboard.process = UnkillableProcess()
        controller = mock.Mock()
        context = {
            'postgres_controller': controller,
            'dashboard_manager': dashboard,
            'supervisor_config': {},
        }
        self.assertFalse(supervisor.coordinated_shutdown([source], context))
        controller.request_stop.assert_not_called()

    def test_main_propagates_failed_background_stop(self):
        args = SimpleNamespace(
            config='config.yaml', sources=None, dashboard=False, no_dashboard=False,
            background=False, stop_background=True, background_status=False,
            attach=False, cmd=None, instance_file=None, pid_file=None,
            with_postgres=True,
        )
        runtime = ({'global': {}}, str(APP_DIR), 'results', {}, [])
        with mock.patch.object(supervisor, 'parse_args', return_value=args), \
                mock.patch.dict(os.environ, {supervisor.RUNTIME_BOOTSTRAP_ENV: supervisor.RUNTIME_BOOTSTRAP_VALUE}, clear=False), \
                mock.patch.object(supervisor, 'sha256_file', return_value='a' * 64), \
                mock.patch.object(supervisor, 'preflight_lifecycle_paths'), \
                mock.patch.object(supervisor, 'load_supervisor_runtime', return_value=runtime), \
                mock.patch.object(supervisor, 'stop_background', return_value=False):
            self.assertEqual(supervisor.main(), 1)

    def test_preflight_failure_happens_before_secret_load_or_background_log_open(self):
        args = SimpleNamespace(
            config='config.yaml', sources=None, dashboard=False, no_dashboard=False,
            background=True, stop_background=False, background_status=False,
            attach=False, cmd=None, instance_file=None, pid_file=None,
            with_postgres=True,
        )
        runtime = ({'global': {}}, str(APP_DIR), 'results', {}, [])
        with mock.patch.object(supervisor, 'parse_args', return_value=args), \
                mock.patch.dict(os.environ, {supervisor.RUNTIME_BOOTSTRAP_ENV: supervisor.RUNTIME_BOOTSTRAP_VALUE}, clear=False), \
                mock.patch.object(supervisor, 'sha256_file', return_value='a' * 64), \
                mock.patch.object(supervisor, 'load_supervisor_runtime', return_value=runtime), \
                mock.patch.object(supervisor, 'preflight_lifecycle_paths', side_effect=OSError('foreign owner')), \
                mock.patch.object(supervisor, 'load_postgres_env') as load_env, \
                mock.patch.object(supervisor, 'start_background') as start:
            with self.assertRaisesRegex(SystemExit, 'foreign owner'):
                supervisor.main()
        load_env.assert_not_called()
        start.assert_not_called()

    def test_duplicate_direct_runtime_exits_before_any_lifecycle_construction(self):
        args = SimpleNamespace(
            config='config.yaml', sources=None, once=False, dry_run=False,
            status_interval=None, no_clear=True, autostart=False, non_interactive=True,
            background_child=False, background=False, stop_background=False,
            background_status=False, attach=False, cmd=None, instance_file=None,
            pid_file=None, dashboard=False, no_dashboard=False, with_postgres=True,
        )
        config = {
            'global': {'queue_dir': 'queues'},
            'sources': {'fixture': {'enabled': True}},
            'keychecks': {'enabled': False},
        }
        runtime = (config, str(APP_DIR), 'results', {'dashboard': {'enabled': False}}, ['fixture'])
        lock = mock.Mock()
        lock.acquire.side_effect = supervisor.InstanceLockError('held')
        with mock.patch.object(supervisor, 'parse_args', return_value=args), \
                mock.patch.dict(os.environ, {supervisor.RUNTIME_BOOTSTRAP_ENV: supervisor.RUNTIME_BOOTSTRAP_VALUE}, clear=False), \
                mock.patch.object(supervisor, 'sha256_file', return_value='a' * 64), \
                mock.patch.object(supervisor, 'load_supervisor_runtime', return_value=runtime), \
                mock.patch.object(supervisor, 'preflight_lifecycle_paths'), \
                mock.patch.object(supervisor, 'validate_managed_runtime_startup'), \
                mock.patch.object(supervisor, 'background_paths', return_value=('instance', 'log', 'status', 'pid')), \
                mock.patch.object(supervisor, 'background_lock_path', return_value='lock'), \
                mock.patch.object(supervisor, 'ClusterAuthorityLock') as cluster_lock, \
                mock.patch.object(supervisor, 'SupervisorInstanceLock', return_value=lock), \
                mock.patch.object(supervisor, 'ManagedSource') as source, \
                mock.patch.object(supervisor, 'controller_from_config') as controller:
            with self.assertRaisesRegex(SystemExit, 'duplicate lifecycle-owning'):
                supervisor.main()
        source.assert_not_called()
        controller.assert_not_called()
        cluster_lock.return_value.acquire.assert_called_once()

    def test_postgres_stop_failed_makes_shutdown_fail(self):
        source = mock.Mock(source='source')
        source.stop.return_value = True

        class Controller:
            terminal = True
            stop_succeeded = False
            detail = 'recovery left running'
            state = supervisor.PostgresState.STOP_FAILED

            def request_stop(self):
                pass

            def close(self, wait=False):
                pass

        context = {'postgres_controller': Controller(), 'supervisor_config': {}}
        self.assertFalse(supervisor.coordinated_shutdown([source], context))


class ImmediateExecutor:
    def submit(self, function, *args):
        from concurrent.futures import Future

        future = Future()
        try:
            future.set_result(function(*args))
        except BaseException as exc:
            future.set_exception(exc)
        return future


class DeferredExecutor:
    def __init__(self):
        self.futures = []

    def submit(self, function, *args):
        from concurrent.futures import Future

        class NonCancellingFuture(Future):
            def cancel(self):
                return False

        future = NonCancellingFuture()
        self.futures.append(future)
        return future

    def shutdown(self, wait=False, cancel_futures=False):
        pass


class Clock:
    def __init__(self):
        self.now = 0.0

    def __call__(self):
        return self.now


class DashboardManagerTests(unittest.TestCase):
    def test_non_loopback_dashboard_address_is_rejected_even_when_disabled(self):
        with self.assertRaisesRegex(ValueError, 'loopback'):
            supervisor.dashboard_settings({
                'dashboard': {'enabled': False, 'address': '0.0.0.0', 'port': 5000},
            })

    def test_launch_log_failure_retains_unstoppable_process_ownership(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            logs = os.path.join(temp_dir, 'logs')
            ensure_private_directory(logs, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            process = UnkillableProcess()
            process.stdout = object()
            manager = supervisor.ManagedDashboard(
                config_path,
                temp_dir,
                {'log_dir': logs, 'dashboard': {'enabled': True}},
                temp_dir,
                temp_dir,
                process_factory=lambda *args, **kwargs: process,
            )
            try:
                with mock.patch.object(
                    supervisor.BoundedRotatingLogPump,
                    'attach',
                    side_effect=RuntimeError('fixture attach failure'),
                ):
                    self.assertFalse(manager.start(force=True))
                self.assertIs(manager.process, process)
                self.assertIsNone(process.poll())
                self.assertEqual(manager.status, 'failed')
                self.assertTrue(manager.fatal_stop_failure)
            finally:
                if manager.log_pump is not None:
                    manager.log_pump.close()
                    manager.log_pump = None
                manager.close()

    def test_stale_health_future_cannot_mark_replacement_generation(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            logs = os.path.join(temp_dir, 'logs')
            ensure_private_directory(logs, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            executor = DeferredExecutor()
            manager = supervisor.ManagedDashboard(
                config_path,
                temp_dir,
                {
                    'log_dir': logs,
                    'dashboard': {'enabled': True, 'health_interval_sec': 1},
                },
                temp_dir,
                temp_dir,
                process_factory=FakeOwnedProcess,
                executor=executor,
            )
            self.assertTrue(manager.start())
            manager.poll()
            stale_future = executor.futures[0]
            stale_generation = manager._future_generation
            self.assertTrue(manager.stop())
            self.assertTrue(manager.start(force=True))
            replacement = manager.process
            stale_future.set_result(False)
            manager._future = stale_future
            manager._future_generation = stale_generation
            manager._consume_health(manager._clock())
            self.assertIs(manager.process, replacement)
            self.assertFalse(replacement.terminated)
            self.assertEqual(manager.status, 'pending')
            manager.stop()
            manager.close()

    def test_gate_health_backoff_recovery_and_env_authority(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            ensure_private_directory(temp_dir, reject_reparse=True)
            config_path = os.path.join(temp_dir, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            harden_private_file(config_path)
            ensure_private_directory(os.path.join(temp_dir, 'logs'), reject_reparse=True)
            ensure_private_directory(os.path.join(temp_dir, 'queues'), reject_reparse=True)
            canonical = 'postgresql://truf:secret@127.0.0.1:5432/truf'
            gate = supervisor.DependencyGate(ready=False, database_url=canonical)
            clock = Clock()
            probes = iter((False, False, True, True))
            created = []

            def factory(*args, **kwargs):
                process = FakeOwnedProcess(*args, **kwargs)
                created.append(process)
                return process

            manager = supervisor.ManagedDashboard(
                config_path,
                str(APP_DIR),
                {
                    'log_dir': os.path.join(temp_dir, 'logs'),
                    'dashboard_log': os.path.join(temp_dir, 'logs', 'dashboard.log'),
                    'dashboard': {
                        'enabled': True,
                        'startup_grace_sec': 5,
                        'health_interval_sec': 1,
                        'restart_base_sec': 2,
                        'stable_health_sec': 0,
                        'env': {'SCANNER_DB_URL': 'postgresql://attacker/other'},
                    },
                },
                temp_dir,
                os.path.join(temp_dir, 'queues'),
                dependency_gate=gate,
                process_factory=factory,
                health_probe=lambda _url, _timeout: next(probes),
                executor=ImmediateExecutor(),
                clock=clock,
            )
            self.assertFalse(manager.start())
            self.assertEqual(manager.status, 'blocked')
            self.assertEqual(created, [])

            gate.set_ready(True)
            self.assertEqual(len(created), 1)
            self.assertEqual(created[0].kwargs['env']['SCANNER_DB_URL'], canonical)
            manager.poll()
            manager.poll()
            self.assertEqual(manager.status, 'pending')

            clock.now = 6
            manager.poll()
            manager.poll()
            self.assertEqual(manager.status, 'backoff')
            self.assertEqual(manager.desired_state, 'running')
            self.assertTrue(created[0].terminated)

            clock.now = 8
            manager.poll()
            self.assertEqual(len(created), 2)
            manager.poll()
            manager.poll()
            self.assertTrue(manager.healthy)
            self.assertEqual(manager.failures, 0)
            gate.set_ready(False)
            self.assertEqual(manager.desired_state, 'running')
            self.assertEqual(manager.status, 'blocked')
            self.assertTrue(created[1].terminated)
            gate.set_ready(True)
            self.assertEqual(len(created), 3)
            manager.poll()
            manager.poll()
            self.assertTrue(manager.healthy)
            self.assertTrue(manager.stop())
            manager.close()


if __name__ == '__main__':
    unittest.main()