truf-server/tests/test_runtime_safety_layer.py

import copy
import json
import multiprocessing
import os
from pathlib import Path
from types import SimpleNamespace
import sys
import tempfile
import threading
import unittest
import uuid
from unittest import mock


ROOT = Path(__file__).resolve().parents[1]
APP_DIR = ROOT / 'app'
sys.path.insert(0, str(APP_DIR))

import console_runner
import migrate_runtime_safety
import scanner
import scanner_db
from migrate_runtime_safety import reconcile_todo_file
from result_spool import ResultSpool, prepare_scan_event
from scanner_db import (
    DiscoveryPausedError,
    DiscoveryRetryLeaseError,
    RuntimeSafetySchemaError,
    ScanEventConflictError,
    ScannerDB,
    migrate_runtime_safety_schema,
    normalize_target,
)


def projection_worker(mode, queue_dir, start, outcomes):
    try:
        start.wait(10)
        if mode == 'enqueue':
            console_runner.enqueue_targets_for_platform(
                queue_dir, 'github', ['https://github.com/example/new'],
            )
        else:
            console_runner.mark_checked(
                [{'target': 'https://github.com/example/original'}],
                os.path.join(queue_dir, 'todo_github.txt'),
                os.path.join(queue_dir, 'checked_github.txt'),
                'github',
            )
        outcomes.put('ok')
    except Exception as exc:
        outcomes.put(type(exc).__name__ + ':' + str(exc))


class SQLiteSafetyCase(unittest.TestCase):
    def setUp(self):
        self.environment = mock.patch.dict(os.environ, {'SCANNER_DB_URL': '', 'DATABASE_URL': ''})
        self.environment.start()
        self.temp = tempfile.TemporaryDirectory()
        self.db = ScannerDB(db_path=os.path.join(self.temp.name, 'scanner.db'))
        self.run_id = self.db.start_run('test', ['test'])
        self.cycle_id = self.db.start_source_cycle(
            self.run_id, 'source', 'github', 'search', 'q', 1, 1, None, {}, {},
        )

    def tearDown(self):
        self.db.close()
        self.temp.cleanup()
        self.environment.stop()

    def claim(self, target, owner='owner-a', max_attempts=3):
        self.db.enqueue_targets('source', 'github', 'q', [target])
        rows = self.db.claim_targets(
            'source', 'github', 1, owner, 60,
            max_attempts=max_attempts, return_rows=True,
        )
        self.assertEqual(len(rows), 1)
        return dict(rows[0])

    def event(self, claim, event_id=None, finding_uid=None, derived=None):
        event_id = event_id or str(uuid.uuid4())
        finding_uid = finding_uid or f'finding-{event_id}'
        result = {
            'scan_event_id': event_id,
            'target': claim['target'],
            'scan_type': 'github',
            'timestamp': '2026-07-18T12:00:00+00:00',
            'findings': [{
                'DetectorName': 'Example',
                'Raw': f'secret-{finding_uid}',
                'finding_uid': finding_uid,
            }],
            'errors': [],
        }
        return prepare_scan_event({
            'version': 1,
            'scan_event_id': event_id,
            'run_id': self.run_id,
            'cycle_id': self.cycle_id,
            'source': 'source',
            'query': 'q',
            'target': claim['target'],
            'result': result,
            'scan_options': {},
            'queue_id': claim['id'],
            'claim_lease_token': claim['lease_token'],
            'claim_lease_owner': claim['lease_owner'],
            'queue_status': 'done',
            'derived_postman_targets': list(derived or []),
        })


class PostgresCatalogFixture:
    def __init__(self, sqlite_connection):
        self.base = sqlite_connection
        self.is_postgres = True
        self.is_sqlite = False
        self.application_schema = 'public'
        self.missing_generation = None
        self.invalid_index = None
        self.missing_foreign_key = None

    def table_exists(self, table):
        return self.base.table_exists(table)

    def table_columns(self, table):
        return self.base.table_columns(table)

    def table_column_details(self, table):
        details = {name: dict(value) for name, value in self.base.table_column_details(table).items()}
        specs = scanner_db.RUNTIME_TABLE_SPECS.get(table, {})
        for name, (expected_type, _) in specs.items():
            if expected_type in ('id', 'id_ref'):
                details[name]['type'] = 'bigint'
        generated_id = scanner_db.GENERATED_ID_COLUMNS.get(table)
        if generated_id:
            details[generated_id]['identity'] = 'd'
            details[generated_id]['sequence'] = f'public.{table}_{generated_id}_seq'
        if self.missing_generation == table:
            details[generated_id].update(identity='', sequence='', default='', has_default=False)
        return details

    def table_indexes(self, table):
        indexes = {
            name: dict(value, columns=list(value['columns']))
            for name, value in self.base.table_indexes(table).items()
        }
        if (
            table in scanner_db.RUNTIME_PRIMARY_KEYS
            and not any(index.get('primary') for index in indexes.values())
        ):
            indexes[f'{table}_pkey'] = {
                'unique': True, 'primary': True, 'valid': True, 'ready': True,
                'live': True, 'predicate': '',
                'columns': list(scanner_db.RUNTIME_PRIMARY_KEYS[table]), 'sql': '',
            }
        if self.invalid_index in indexes:
            indexes[self.invalid_index]['valid'] = False
        return indexes

    def table_foreign_keys(self, table):
        foreign_keys = {
            name: dict(value, columns=list(value['columns']), referenced_columns=list(value['referenced_columns']))
            for name, value in self.base.table_foreign_keys(table).items()
        }
        for foreign_key in foreign_keys.values():
            foreign_key['referenced_schema'] = 'public'
        if self.missing_foreign_key:
            missing_table, missing_columns = self.missing_foreign_key
            if table == missing_table:
                foreign_keys = {
                    name: value for name, value in foreign_keys.items()
                    if tuple(value['columns']) != tuple(missing_columns)
                }
        return foreign_keys

    def table_check_constraints(self, table):
        return {
            name: dict(value)
            for name, value in self.base.table_check_constraints(table).items()
        }

    def table_triggers(self, table):
        if table != 'runtime_audit_events':
            return self.base.table_triggers(table)
        function_sql = """
            CREATE FUNCTION reject_runtime_audit_event_mutation() RETURNS trigger AS $$
            BEGIN
                RAISE EXCEPTION 'runtime_audit_events is append-only';
            END;
            $$ LANGUAGE plpgsql;
        """
        return {
            'runtime_audit_events_reject_mutation': {
                'enabled': True,
                'sql': '''CREATE TRIGGER runtime_audit_events_reject_mutation
                          BEFORE UPDATE OR DELETE ON public.runtime_audit_events
                          FOR EACH ROW EXECUTE FUNCTION reject_runtime_audit_event_mutation()''',
                'function_sql': function_sql,
            },
            'runtime_audit_events_reject_truncate': {
                'enabled': True,
                'sql': '''CREATE TRIGGER runtime_audit_events_reject_truncate
                          BEFORE TRUNCATE ON public.runtime_audit_events
                          FOR EACH STATEMENT EXECUTE FUNCTION reject_runtime_audit_event_mutation()''',
                'function_sql': function_sql,
            },
        }

    def execute(self, sql, params=None):
        return self.base.execute(sql, params)

    def commit(self):
        return self.base.commit()

    def rollback(self):
        return self.base.rollback()


class LeaseAndIngestionTests(SQLiteSafetyCase):
    def test_claimable_target_hint_tracks_live_lease_fencing(self):
        target = 'https://github.com/example/dispatch-hint'
        self.db.enqueue_targets('source', 'github', 'q', [target])
        self.assertTrue(self.db.has_claimable_targets('source', 'github', max_attempts=3))
        claimed = self.db.claim_targets(
            'source', 'github', 1, 'owner-a', 60,
            max_attempts=3, return_rows=True, claim_batch='batch-a',
        )
        self.assertEqual(len(claimed), 1)
        self.assertFalse(self.db.has_claimable_targets('source', 'github', max_attempts=3))

    def test_reclaim_changes_token_and_stale_token_cannot_complete(self):
        target = 'https://github.com/example/repo'
        first = self.claim(target)
        self.db.conn.execute(
            'UPDATE target_queue SET lease_expires_at = ? WHERE id = ?',
            ('2000-01-01T00:00:00+00:00', first['id']),
        )
        self.db.conn.commit()
        second = dict(self.db.claim_targets(
            'source', 'github', 1, 'owner-b', 60,
            max_attempts=3, return_rows=True,
        )[0])

        self.assertNotEqual(first['lease_token'], second['lease_token'])
        self.assertFalse(self.db.complete_target_queue_item(
            'source', 'github', target, status='done',
            queue_id=first['id'], lease_token=first['lease_token'],
        ))
        row = self.db.target_queue_item('source', 'github', target)
        self.assertEqual(row['status'], 'in_progress')
        self.assertEqual(row['lease_token'], second['lease_token'])

    def test_stale_event_preserves_findings_outbox_and_newer_lease(self):
        target = 'https://github.com/example/stale'
        first = self.claim(target)
        self.db.conn.execute(
            'UPDATE target_queue SET lease_expires_at = ? WHERE id = ?',
            ('2000-01-01T00:00:00+00:00', first['id']),
        )
        self.db.conn.commit()
        second = dict(self.db.claim_targets(
            'source', 'github', 1, 'owner-b', 60,
            max_attempts=3, return_rows=True,
        )[0])
        derived = '{"kind":"collection","sha256":"abc","cache_path":"fixture"}'
        outcome = self.db.ingest_scan_event(self.event(first, derived=[derived]))

        self.assertTrue(outcome['stale'])
        self.assertFalse(outcome['queue_completion_applied'])
        queue_row = self.db.target_queue_item('source', 'github', target)
        self.assertEqual(queue_row['status'], 'in_progress')
        self.assertEqual(queue_row['lease_token'], second['lease_token'])
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM target_scans').fetchone()['n'], 1)
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM findings').fetchone()['n'], 1)
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM scan_publication_outbox').fetchone()['n'], 1)
        derived_row = self.db.conn.execute(
            "SELECT status FROM target_queue WHERE source = 'postman'"
        ).fetchone()
        self.assertEqual(derived_row['status'], 'pending')

    def test_commit_ambiguity_replay_is_single_event(self):
        claim = self.claim('https://github.com/example/ambiguous')
        event = self.event(claim)
        self.db.require_runtime_safety_schema()
        real_commit = self.db.conn.commit
        raised = {'value': False}

        def ambiguous_commit():
            real_commit()
            if not raised['value']:
                raised['value'] = True
                raise OSError('simulated lost commit acknowledgement')

        self.db.conn.commit = ambiguous_commit
        with self.assertRaises(OSError):
            self.db.ingest_scan_event(event)
        self.db.conn.commit = real_commit

        replay = self.db.ingest_scan_event(event)
        self.assertTrue(replay['duplicate'])
        for table in ('target_scans', 'findings', 'scan_publication_outbox'):
            count = self.db.conn.execute(f'SELECT COUNT(*) AS n FROM {table}').fetchone()['n']
            self.assertEqual(count, 1, table)

    def test_claim_commit_ambiguity_recovers_exact_reserved_batch_without_second_attempt(self):
        target = 'https://github.com/example/claim-ambiguous'
        self.db.enqueue_targets('source', 'github', 'q', [target])
        spool = ResultSpool(os.path.join(self.temp.name, 'spool'), min_free_bytes=0)
        reservation = spool.reserve_claims('owner-a', 1, 60)
        real_commit = self.db.conn.commit
        raised = {'value': False}

        def ambiguous_commit():
            real_commit()
            if not raised['value']:
                raised['value'] = True
                raise OSError('lost claim commit acknowledgement')

        self.db.conn.commit = ambiguous_commit
        with self.assertRaisesRegex(OSError, 'lost claim commit acknowledgement'):
            self.db.claim_targets(
                'source', 'github', 1, 'owner-a', 60, max_attempts=3,
                return_rows=True, claim_batch=reservation,
            )
        self.db.conn.commit = real_commit
        recovered = self.db.recover_claim_batch(reservation, 'owner-a')
        self.assertEqual(len(recovered), 1)
        self.assertEqual(recovered[0]['attempts'], 1)
        self.assertEqual(recovered[0]['claim_batch'], reservation)
        spool.bind_claims(reservation, recovered)

    def test_true_claim_rollback_raises_original_and_recovers_no_rows(self):
        target = 'https://github.com/example/claim-rollback'
        self.db.enqueue_targets('source', 'github', 'q', [target])
        real_commit = self.db.conn.commit

        def failed_commit():
            raise OSError('claim transaction rolled back')

        self.db.conn.commit = failed_commit
        with self.assertRaisesRegex(OSError, 'claim transaction rolled back'):
            self.db.claim_targets(
                'source', 'github', 1, 'owner-a', 60, max_attempts=3,
                return_rows=True, claim_batch='rollback-batch',
            )
        self.db.conn.commit = real_commit
        self.assertEqual(self.db.recover_claim_batch('rollback-batch', 'owner-a'), [])
        row = self.db.target_queue_item('source', 'github', target)
        self.assertEqual(row['status'], 'pending')
        self.assertEqual(row['attempts'], 0)

    def test_same_event_id_with_different_hash_fails_closed(self):
        claim = self.claim('https://github.com/example/conflict')
        event = self.event(claim)
        self.db.ingest_scan_event(event)
        conflicting = copy.deepcopy(event)
        conflicting.pop('scan_event_hash')
        conflicting['result']['findings'][0]['Raw'] = 'different-secret'
        conflicting = prepare_scan_event(conflicting)

        with self.assertRaises(ScanEventConflictError):
            self.db.ingest_scan_event(conflicting)
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM target_scans').fetchone()['n'], 1)
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM findings').fetchone()['n'], 1)
        self.assertEqual(self.db.conn.execute('SELECT COUNT(*) AS n FROM scan_publication_outbox').fetchone()['n'], 1)

    def test_duplicate_event_insert_is_noop_without_aborting_transaction(self):
        claim = self.claim('https://github.com/example/duplicate-insert')
        event = self.event(claim)
        outcome = self.db.ingest_scan_event(event)
        result = event['result']
        duplicate_id = self.db._insert_target_result(
            self.run_id,
            self.cycle_id,
            'source',
            'q',
            claim['target'],
            result,
            {},
            scan_event_id=event['scan_event_id'],
            scan_event_hash_value=event['scan_event_hash'],
            queue_id=claim['id'],
            claim_lease_token=claim['lease_token'],
        )
        self.assertIsNone(duplicate_id)
        self.assertEqual(self.db.conn.execute('SELECT 1 AS n').fetchone()['n'], 1)
        self.db.conn.rollback()
        self.assertEqual(
            self.db.conn.execute('SELECT COUNT(*) AS n FROM target_scans').fetchone()['n'], 1,
        )
        self.assertEqual(outcome['scan_event_id'], event['scan_event_id'])

    def test_unresolved_docker_row_uses_bounded_retryable_resolver_state(self):
        self.db.enqueue_targets('docker-source', 'docker', 'q', [], unresolved_targets=['owner/image'])
        self.db.conn.execute(
            "UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' WHERE source = 'docker-source'"
        )
        self.db.conn.commit()
        rows = self.db.claim_docker_resolutions('docker-source', 1, 'resolver', 60)
        self.assertEqual([row['target'] for row in rows], ['owner/image'])
        self.assertTrue(self.db.finish_docker_resolution(
            'docker-source', rows[0]['id'], rows[0]['resolver_token'], ['owner/image:latest'],
        ))
        bare = self.db.target_queue_item('docker-source', 'docker', 'owner/image')
        tagged = self.db.target_queue_item('docker-source', 'docker', 'owner/image:latest')
        self.assertEqual(bare['status'], 'done')
        self.assertEqual(tagged['status'], 'pending')

    def test_discarded_unissued_target_is_reactivated_without_duplicate(self):
        target = 'https://github.com/example/stale-backlog'
        self.assertEqual(self.db.enqueue_targets('source', 'github', 'old', [target]), 1)
        before = self.db.target_queue_item('source', 'github', target)
        self.db.conn.execute(
            '''UPDATE target_queue SET status = 'quarantined', attempts = 7,
                      available_after = '2099-01-01T00:00:00+00:00',
                      last_error = ? WHERE id = ?''',
            (scanner_db.ADMIN_DISCARDED_QUEUE_REASON, before['id']),
        )
        self.db.conn.commit()

        self.assertEqual(self.db.enqueue_targets('source', 'github', 'fresh', [target]), 1)
        rows = self.db.conn.execute(
            'SELECT * FROM target_queue WHERE source = ? AND normalized_target = ?',
            ('source', normalize_target(target, 'github')),
        ).fetchall()

        self.assertEqual(len(rows), 1)
        self.assertEqual(rows[0]['id'], before['id'])
        self.assertEqual(rows[0]['status'], 'pending')
        self.assertEqual(rows[0]['query'], 'fresh')
        self.assertEqual(rows[0]['attempts'], 0)
        self.assertIsNone(rows[0]['available_after'])
        self.assertIsNone(rows[0]['last_error'])

    def test_non_admin_quarantine_is_not_reactivated_by_discovery(self):
        target = 'https://github.com/example/real-quarantine'
        self.db.enqueue_targets('source', 'github', 'old', [target])
        row = self.db.target_queue_item('source', 'github', target)
        self.db.conn.execute(
            "UPDATE target_queue SET status = 'quarantined', last_error = 'bundle conflict' WHERE id = ?",
            (row['id'],),
        )
        self.db.conn.commit()

        self.db.enqueue_targets('source', 'github', 'fresh', [target])
        current = self.db.conn.execute(
            'SELECT * FROM target_queue WHERE id = ?', (row['id'],),
        ).fetchone()

        self.assertEqual(current['status'], 'quarantined')
        self.assertEqual(current['query'], 'old')
        self.assertEqual(current['last_error'], 'bundle conflict')

    def test_discarded_docker_repository_is_freshly_readmitted(self):
        repository = 'example/stale-repository'
        first = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'old-query', [repository],
        )
        self.assertEqual(first['inserted_count'], 1)
        row = self.db.target_queue_item('dockerhub', 'docker', repository)
        self.db.conn.execute(
            '''UPDATE target_queue SET status = 'quarantined',
                      resolver_state = 'resolved', resolver_due_at = NULL,
                      available_after = NULL, last_error = ? WHERE id = ?''',
            (scanner_db.ADMIN_DISCARDED_QUEUE_REASON, row['id']),
        )
        self.db.conn.commit()

        second = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'fresh-query', [repository],
        )
        current = self.db.conn.execute(
            'SELECT * FROM target_queue WHERE id = ?', (row['id'],),
        ).fetchone()

        self.assertEqual(second['inserted_count'], 1)
        self.assertEqual(second['preexisting_count'], 0)
        self.assertEqual(current['id'], row['id'])
        self.assertEqual(current['status'], 'deferred')
        self.assertEqual(current['query'], 'fresh-query')
        self.assertEqual(current['resolver_state'], 'pending')
        self.assertEqual(current['available_after'], current['resolver_due_at'])

    def test_docker_resolver_failures_keep_a_bounded_non_null_due_state(self):
        self.db.enqueue_targets('docker-source', 'docker', 'q', [], unresolved_targets=['owner/retry'])
        with mock.patch.dict(os.environ, {
            'DOCKER_RESOLVER_RETRY_SEC': '60',
            'DOCKER_RESOLVER_RETRY_MAX_SEC': '60',
        }):
            for _ in range(3):
                self.db.conn.execute(
                    "UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' WHERE source = 'docker-source'"
                )
                self.db.conn.commit()
                row = self.db.claim_docker_resolutions('docker-source', 1, 'resolver', 60)[0]
                self.assertTrue(self.db.finish_docker_resolution(
                    'docker-source', row['id'], row['resolver_token'], [], 'network unavailable',
                ))
                state = self.db.conn.execute(
                    "SELECT resolver_state, resolver_due_at, available_after FROM target_queue WHERE id = ?",
                    (row['id'],),
                ).fetchone()
                self.assertEqual(state['resolver_state'], 'retry')
                self.assertTrue(state['resolver_due_at'])
                self.assertEqual(state['resolver_due_at'], state['available_after'])


class DiscoveryGateTests(SQLiteSafetyCase):
    def set_discovery_paused(self, paused):
        state = self.db.runtime_control_state()
        return self.db.set_runtime_discovery_paused(
            paused,
            expected_revision=state['revision'],
            actor='test:discovery-gate',
            operation_id=str(uuid.uuid4()),
        )

    def test_pause_blocks_provider_admission_and_retry_claim_but_not_retry_enqueue(self):
        self.set_discovery_paused(True)
        self.assertEqual(self.db.enqueue_targets(
            'maintenance', 'github', 'q', ['https://github.com/acme/maintenance'],
        ), 1)
        with self.assertRaises(DiscoveryPausedError):
            self.db.enqueue_targets(
                'github', 'github', 'q', ['https://github.com/acme/discovered'],
                discovery_admission=True,
            )
        with self.assertRaises(DiscoveryPausedError):
            self.db.persist_dockerhub_discovery_page(
                'dockerhub', 'q', ['acme/discovered'],
            )

        retry = self.db.enqueue_discovery_retry(
            'dockerhub', 'q', 'a' * 64, 'ordinary', 'page',
        )
        self.assertEqual(self.db.claim_discovery_retries(
            'dockerhub', {'q': 'a' * 64}, 'retry-owner',
        ), [])
        row = self.db.conn.execute(
            'SELECT status, attempts, lease_token FROM discovery_retry_queue WHERE id = ?',
            (retry['id'],),
        ).fetchone()
        self.assertEqual((row['status'], row['attempts'], row['lease_token']), (
            'pending', 0, None,
        ))

        self.set_discovery_paused(False)
        claims = self.db.claim_discovery_retries(
            'dockerhub', {'q': 'a' * 64}, 'retry-owner',
        )
        self.assertEqual(len(claims), 1)

    def test_pause_blocks_resolver_claim_and_tag_admission_but_allows_settlement(self):
        repositories = ['acme/tagged', 'acme/deferred', 'acme/waiting']
        self.db.enqueue_targets(
            'dockerhub', 'docker', 'q', [], unresolved_targets=repositories,
        )
        self.db.conn.execute(
            "UPDATE target_queue SET resolver_due_at = '2000-01-01T00:00:00+00:00' "
            "WHERE source = 'dockerhub'",
        )
        self.db.conn.commit()
        claims = self.db.claim_docker_resolutions(
            'dockerhub', 2, 'resolver-owner', 60,
        )
        self.assertEqual(len(claims), 2)

        self.set_discovery_paused(True)
        with self.assertRaises(DiscoveryPausedError):
            self.db.finish_docker_resolution(
                'dockerhub', claims[0]['id'], claims[0]['resolver_token'],
                ['acme/tagged:latest'],
            )
        first = self.db.conn.execute(
            'SELECT resolver_state, resolver_token FROM target_queue WHERE id = ?',
            (claims[0]['id'],),
        ).fetchone()
        self.assertEqual((first['resolver_state'], first['resolver_token']), (
            'resolving', claims[0]['resolver_token'],
        ))
        self.assertIsNone(self.db.target_queue_item(
            'dockerhub', 'docker', 'acme/tagged:latest',
        ))

        self.assertTrue(self.db.finish_docker_resolution(
            'dockerhub', claims[1]['id'], claims[1]['resolver_token'], [],
            'provider unavailable', complete=False,
        ))
        waiting_before = self.db.target_queue_item(
            'dockerhub', 'docker', 'acme/waiting',
        )
        self.assertEqual(self.db.claim_docker_resolutions(
            'dockerhub', 1, 'paused-owner', 60,
        ), [])
        waiting_after = self.db.target_queue_item(
            'dockerhub', 'docker', 'acme/waiting',
        )
        self.assertEqual(waiting_after['attempts'], waiting_before['attempts'])
        self.assertIsNone(waiting_after['lease_token'])

        self.set_discovery_paused(False)
        self.assertTrue(self.db.finish_docker_resolution(
            'dockerhub', claims[0]['id'], claims[0]['resolver_token'],
            ['acme/tagged:latest'],
        ))
        self.assertEqual(
            self.db.target_queue_item('dockerhub', 'docker', 'acme/tagged:latest')['status'],
            'pending',
        )

    def test_ingestion_derived_work_remains_admissible_while_paused(self):
        claim = self.claim('https://github.com/acme/result-before-pause')
        self.set_discovery_paused(True)
        derived = '{"kind":"collection","sha256":"pause-safe","cache_path":"fixture"}'
        outcome = self.db.ingest_scan_event(self.event(claim, derived=[derived]))
        self.assertTrue(outcome['queue_completion_applied'])
        row = self.db.conn.execute(
            "SELECT status FROM target_queue WHERE source = 'postman'",
        ).fetchone()
        self.assertEqual(row['status'], 'pending')

    def test_drain_uses_the_effective_gate_without_changing_explicit_pause(self):
        state = self.db.runtime_control_state()
        self.db.start_runtime_drain(
            expected_revision=state['revision'],
            actor='test:discovery-gate',
            operation_id=str(uuid.uuid4()),
        )
        gated = self.db.runtime_control_state()
        self.assertFalse(gated['discovery_paused'])
        self.assertTrue(gated['effective_discovery_paused'])
        with self.assertRaises(DiscoveryPausedError):
            self.db.enqueue_targets(
                'gitlab', 'gitlab', 'q', ['https://gitlab.com/acme/drained'],
                discovery_admission=True,
            )
        retry = self.db.enqueue_discovery_retry(
            'dockerhub', 'q', 'a' * 64, 'ordinary', 'page',
        )
        self.assertEqual(self.db.claim_discovery_retries(
            'dockerhub', {'q': 'a' * 64}, 'retry-owner',
        ), [])

        self.db.cancel_runtime_drain(
            expected_revision=gated['revision'],
            actor='test:discovery-gate',
            operation_id=str(uuid.uuid4()),
        )
        claims = self.db.claim_discovery_retries(
            'dockerhub', {'q': 'a' * 64}, 'retry-owner',
        )
        self.assertEqual([claim['id'] for claim in claims], [retry['id']])


class ReconciliationTests(SQLiteSafetyCase):
    def test_offline_schema_migration_is_idempotent(self):
        self.db.conn.execute('DROP INDEX uq_target_scans_scan_event_id')
        self.db.conn.execute('DROP TABLE target_queue_reconciliation_cursors')
        self.db.conn.commit()
        migrate_runtime_safety_schema(self.db)
        migrate_runtime_safety_schema(self.db)
        self.assertTrue(self.db.runtime_safety_schema_available(), self.db.last_error)

    def test_reconciliation_is_bounded_idempotent_and_status_preserving(self):
        source = 'docker-source'
        existing = 'owner/existing:tag'
        new_target = 'owner/new:tag'
        bare = 'owner/unresolved'
        self.db.enqueue_targets(source, 'docker', 'old', [existing])
        self.db.conn.execute(
            "UPDATE target_queue SET status = 'done', completed_at = '2026-01-01T00:00:00+00:00' WHERE source = ?",
            (source,),
        )
        self.db.conn.commit()
        todo = os.path.join(self.temp.name, 'todo_docker.txt')
        original = f'{existing}\n{new_target}\n{bare}\n'
        Path(todo).write_text(original, encoding='utf-8')

        reports = [
            reconcile_todo_file(self.db, todo, source, 'docker', max_rows=1, max_bytes=1024, max_seconds=1)
            for _ in range(4)
        ]
        self.assertTrue(all(report['rows_read'] <= 1 for report in reports))
        self.assertIn('sha256:', reports[2]['unresolved_docker_rows'][0]['target'])
        self.assertEqual(reports[3]['rows_read'], 0)
        self.assertTrue(reports[3]['at_eof'])
        self.assertEqual(Path(todo).read_text(encoding='utf-8'), original)
        rows = self.db.conn.execute(
            'SELECT normalized_target, status FROM target_queue WHERE source = ? ORDER BY normalized_target',
            (source,),
        ).fetchall()
        states = {row['normalized_target']: row['status'] for row in rows}
        self.assertEqual(states[normalize_target(existing, 'docker')], 'done')
        self.assertEqual(states[normalize_target(new_target, 'docker')], 'pending')
        self.assertEqual(states[normalize_target(bare, 'docker')], 'deferred')
        resolver = self.db.conn.execute(
            'SELECT resolver_state, resolver_due_at FROM target_queue WHERE source = ? AND normalized_target = ?',
            (source, normalize_target(bare, 'docker')),
        ).fetchone()
        self.assertEqual(resolver['resolver_state'], 'pending')
        self.assertTrue(resolver['resolver_due_at'])
        issue = self.db.conn.execute(
            'SELECT reason, target_preview FROM target_queue_reconciliation_issues WHERE source_file = ?',
            (os.path.normcase(os.path.abspath(todo)),),
        ).fetchone()
        self.assertEqual(issue['reason'], 'unresolved bare Docker repository')
        self.assertNotIn(bare, issue['target_preview'])

    def test_reconciliation_rolls_back_and_retries_if_handle_snapshot_changes(self):
        todo = os.path.join(self.temp.name, 'todo_github.txt')
        Path(todo).write_text('https://github.com/example/one\n', encoding='utf-8')
        checks = iter((True, True, False, True, True, True))
        with mock.patch.object(migrate_runtime_safety, '_same_file_snapshot', side_effect=lambda *args: next(checks)):
            report = reconcile_todo_file(self.db, todo, 'source', 'github')
        self.assertTrue(report['at_eof'])
        self.assertEqual(
            self.db.conn.execute("SELECT COUNT(*) AS n FROM target_queue WHERE source = 'source'").fetchone()['n'],
            1,
        )

    def test_oversized_and_invalid_rows_advance_with_durable_redacted_issues(self):
        todo = os.path.join(self.temp.name, 'todo_npm.txt')
        with open(todo, 'wb') as handle:
            handle.write(b'{"name":"' + (b'x' * 64) + b'"}\n')
            handle.write(b'\xffbad\n')
            handle.write(b'{broken}\n')
        reports = []
        for _ in range(20):
            report = reconcile_todo_file(
                self.db, todo, 'npm-source', 'npm', max_rows=1, max_bytes=16, max_seconds=1,
            )
            reports.append(report)
            if report['at_eof']:
                break
        self.assertTrue(reports[-1]['at_eof'])
        self.assertGreaterEqual(reports[-1]['cumulative_rejected'], 2)
        issues = self.db.conn.execute(
            '''SELECT reason, target_preview FROM target_queue_reconciliation_issues
               WHERE source_file = ? ORDER BY id''',
            (os.path.normcase(os.path.abspath(todo)),),
        ).fetchall()
        self.assertGreaterEqual(len(issues), 2)
        self.assertTrue(all(str(row['target_preview']).startswith('<sha256:') for row in issues))

    def test_valid_crlf_row_that_straddles_batch_budget_is_deferred_without_cursor_advance(self):
        first = 'https://github.com/example/a'
        second = 'https://github.com/example/a-much-longer-repository-name'
        first_bytes = (first + '\r\n').encode('ascii')
        todo = os.path.join(self.temp.name, 'todo_crlf.txt')
        Path(todo).write_bytes(first_bytes + (second + '\r\n').encode('ascii'))
        report = reconcile_todo_file(
            self.db, todo, 'crlf-source', 'github', max_rows=10,
            max_bytes=len(first_bytes) + 3, max_seconds=1,
        )
        self.assertEqual(report['rows_read'], 1)
        self.assertEqual(report['byte_offset'], len(first_bytes))
        self.assertFalse(report['at_eof'])
        followup = reconcile_todo_file(
            self.db, todo, 'crlf-source', 'github', max_rows=10,
            max_bytes=8, max_seconds=1,
        )
        self.assertEqual(followup['rows_read'], 1)
        self.assertTrue(followup['at_eof'])
        rows = self.db.conn.execute(
            "SELECT target FROM target_queue WHERE source = 'crlf-source' ORDER BY id"
        ).fetchall()
        self.assertEqual([row['target'] for row in rows], [first, second])

    def test_malformed_complete_rows_obey_batch_budget_and_first_row_progresses(self):
        cases = {
            'malformed-json': b'{broken}\r\n',
            'invalid-utf8': b'\xffbad\r\n',
        }
        for name, malformed in cases.items():
            with self.subTest(name=name):
                todo = os.path.join(self.temp.name, f'todo_{name}.txt')
                first = b'{"name":"first"}\r\n'
                final = b'{"name":"final"}\r\n'
                Path(todo).write_bytes(first + malformed + final)
                source = f'source-{name}'
                report = reconcile_todo_file(
                    self.db, todo, source, 'npm', max_rows=10,
                    max_bytes=len(first) + 1, max_seconds=1,
                )
                self.assertEqual(report['rows_read'], 1)
                self.assertEqual(report['byte_offset'], len(first))
                self.assertEqual(report['rejected_rows'], 0)

                malformed_report = reconcile_todo_file(
                    self.db, todo, source, 'npm', max_rows=10,
                    max_bytes=1, max_seconds=1,
                )
                self.assertEqual(malformed_report['rows_read'], 1)
                self.assertEqual(malformed_report['rejected_rows'], 1)
                self.assertEqual(malformed_report['byte_offset'], len(first) + len(malformed))
                self.assertFalse(malformed_report['at_eof'])

                final_report = reconcile_todo_file(
                    self.db, todo, source, 'npm', max_rows=10,
                    max_bytes=1, max_seconds=1,
                )
                self.assertEqual(final_report['rows_read'], 1)
                self.assertTrue(final_report['at_eof'])

    def test_truly_oversized_row_is_consumed_and_following_crlf_row_progresses(self):
        todo = os.path.join(self.temp.name, 'todo_oversized.txt')
        Path(todo).write_bytes((b'x' * 40) + b'\r\nok\r\n')
        reports = []
        with mock.patch.object(migrate_runtime_safety, 'RECONCILIATION_ABSOLUTE_ROW_BYTES', 16):
            for _ in range(20):
                report = reconcile_todo_file(
                    self.db, todo, 'oversized-source', 'github',
                    max_rows=2, max_bytes=8, max_seconds=1,
                )
                reports.append(report)
                if report['at_eof']:
                    break
        self.assertTrue(reports[-1]['at_eof'])
        issue = self.db.conn.execute(
            "SELECT reason FROM target_queue_reconciliation_issues WHERE source_file = ?",
            (os.path.normcase(os.path.abspath(todo)),),
        ).fetchone()
        self.assertIn('absolute 16-byte', issue['reason'])
        self.assertIsNotNone(self.db.target_queue_item(
            'oversized-source', 'github', 'ok',
        ))

    def test_reconciliation_cursor_cannot_be_rebound_to_another_mapping(self):
        todo = os.path.join(self.temp.name, 'todo.txt')
        Path(todo).write_text('https://github.com/example/one\n', encoding='utf-8')
        reconcile_todo_file(self.db, todo, 'source-a', 'github')
        with self.assertRaisesRegex(ValueError, 'cannot be reused'):
            reconcile_todo_file(self.db, todo, 'source-b', 'github')

    def test_issue_resolution_validates_the_complete_set_before_commit(self):
        now = '2026-07-18T00:00:00+00:00'
        ids = []
        for line in (1, 2):
            cursor = self.db.conn.execute(
                '''INSERT INTO target_queue_reconciliation_issues (
                    source_file, file_identity, source, platform, line_number,
                    byte_offset, reason, target_preview, created_at
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)''',
                ('fixture', 'identity', 'source', 'github', line, line, 'review', '<redacted>', now),
            )
            ids.append(cursor.lastrowid)
        self.db.conn.commit()

        with self.assertRaisesRegex(RuntimeError, 'absent or already resolved'):
            migrate_runtime_safety.resolve_reconciliation_issues(self.db, [ids[0], 99999999])
        row = self.db.conn.execute(
            'SELECT resolved_at FROM target_queue_reconciliation_issues WHERE id = ?',
            (ids[0],),
        ).fetchone()
        self.assertIsNone(row['resolved_at'])

        self.assertEqual(migrate_runtime_safety.resolve_reconciliation_issues(self.db, ids), 2)

    def test_schema_validator_rejects_and_migration_repairs_wrong_event_index(self):
        self.db.conn.execute('DROP INDEX uq_target_scans_scan_event_id')
        self.db.conn.execute('CREATE INDEX uq_target_scans_scan_event_id ON target_scans(scan_event_id)')
        self.db.conn.commit()
        self.assertFalse(self.db.runtime_safety_schema_available())
        self.assertIn('unique partial scan-event index', self.db.last_error)
        migrate_runtime_safety_schema(self.db)
        self.assertTrue(self.db.runtime_safety_schema_available(), self.db.last_error)

    def test_schema_validator_checks_keycheck_types_and_outbox_nullability(self):
        original = self.db.conn.table_column_details

        def malformed(table):
            details = original(table)
            if table == 'keycheck_results':
                details['service'] = dict(details['service'], type='integer')
            if table == 'scan_publication_outbox':
                details['payload_json'] = dict(details['payload_json'], not_null=False)
            if table == 'target_queue':
                details['attempts'] = dict(details['attempts'], default='99')
            return details

        with mock.patch.object(self.db.conn, 'table_column_details', side_effect=malformed):
            self.assertFalse(self.db.runtime_safety_schema_available())
        self.assertIn('keycheck_results.service type', self.db.last_error)
        self.assertIn('scan_publication_outbox.payload_json nullability', self.db.last_error)
        self.assertIn('target_queue.attempts default', self.db.last_error)

    def test_postgres_catalog_fixture_rejects_missing_generation_invalid_index_and_fk(self):
        original = self.db.conn
        fixture = PostgresCatalogFixture(original)
        self.db.conn = fixture
        try:
            self.assertTrue(self.db.runtime_safety_schema_available(), self.db.last_error)

            fixture.missing_generation = 'target_queue'
            self.assertFalse(self.db.runtime_safety_schema_available())
            self.assertIn('target_queue.id generated ID identity/sequence', self.db.last_error)

            fixture.missing_generation = None
            fixture.invalid_index = 'idx_target_queue_claim'
            self.assertFalse(self.db.runtime_safety_schema_available())
            self.assertIn('index idx_target_queue_claim', self.db.last_error)

            fixture.invalid_index = None
            fixture.missing_foreign_key = ('keycheck_results', ('run_id',))
            self.assertFalse(self.db.runtime_safety_schema_available())
            self.assertIn('foreign key keycheck_results(run_id) -> runs(id)', self.db.last_error)
        finally:
            self.db.conn = original

    def test_sqlite_migration_rejects_non_additively_repairable_base_shape(self):
        self.db.conn.execute('DROP TABLE target_queue')
        self.db.conn.execute('CREATE TABLE target_queue (id TEXT PRIMARY KEY, source TEXT)')
        self.db.conn.commit()
        with self.assertRaisesRegex(RuntimeSafetySchemaError, 'manually rebuild'):
            migrate_runtime_safety_schema(self.db)

    def test_discovery_retry_schema_and_page_admission_preserve_every_existing_status(self):
        details = self.db.conn.table_column_details('discovery_retry_queue')
        self.assertEqual(set(details), set(scanner_db.DISCOVERY_RETRY_QUEUE_COLUMN_SPECS))
        self.assertEqual(details['id']['type'], 'integer')
        self.assertTrue(details['id']['primary_key'])
        self.assertEqual(scanner_db._normalized_default(details['status']['default']), 'pending')
        self.assertEqual(scanner_db._normalized_default(details['attempts']['default']), '0')
        indexes = self.db.conn.table_indexes('discovery_retry_queue')
        self.assertEqual(indexes['uq_discovery_retry_queue_work_key']['columns'], ['work_key'])
        self.assertTrue(indexes['uq_discovery_retry_queue_work_key']['unique'])
        self.assertEqual(
            indexes['idx_discovery_retry_queue_due']['columns'],
            ['source', 'available_after', 'id'],
        )
        self.assertEqual(
            scanner_db._normalized_predicate(
                indexes['idx_discovery_retry_queue_due']['predicate']
            ),
            scanner_db._normalized_predicate(scanner_db.DISCOVERY_RETRY_DUE_INDEX_PREDICATE),
        )

        statuses = ('pending', 'deferred', 'in_progress', 'done', 'failed', 'cold')
        now = scanner_db.utc_now_iso()
        for status in statuses:
            repository = f'fixture/{status.replace("_", "-")}'
            self.db.conn.execute(
                '''INSERT INTO target_queue(
                       source, platform, query, target, normalized_target, status,
                       resolver_state, created_at, updated_at
                   ) VALUES ('dockerhub', 'docker', 'old-query', ?, ?, ?,
                       'resolved', ?, ?)''',
                (repository, repository, status, now, now),
            )
        self.db.conn.commit()
        before = {
            row['normalized_target']: (
                row['status'], row['query'], row['target'], row['resolver_state'],
            )
            for row in self.db.conn.execute(
                '''SELECT normalized_target, status, query, target, resolver_state
                   FROM target_queue WHERE source = 'dockerhub' '''
            ).fetchall()
        }

        repositories = [
            {'repo_name': f'fixture/{status.replace("_", "-")}'} for status in statuses
        ] + [{'repo_name': 'fixture/new'}, {'repo_name': 'fixture/new'}]
        report = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'exact-query', repositories,
        )
        self.assertEqual(report['attempted_count'], 8)
        self.assertEqual(report['normalized_count'], 7)
        self.assertEqual(report['duplicate_count'], 1)
        self.assertEqual(report['preexisting_count'], 6)
        self.assertEqual(report['inserted_count'], 1)
        self.assertEqual(
            report['preexisting_repositories'], frozenset(before),
        )
        after = {
            row['normalized_target']: (
                row['status'], row['query'], row['target'], row['resolver_state'],
            )
            for row in self.db.conn.execute(
                '''SELECT normalized_target, status, query, target, resolver_state
                   FROM target_queue WHERE source = 'dockerhub'
                     AND normalized_target <> 'fixture/new' '''
            ).fetchall()
        }
        self.assertEqual(after, before)
        inserted = self.db.conn.execute(
            '''SELECT status, query, resolver_state, resolver_due_at, available_after
               FROM target_queue WHERE source = 'dockerhub'
                 AND normalized_target = 'fixture/new' '''
        ).fetchone()
        self.assertEqual(inserted['status'], 'deferred')
        self.assertEqual(inserted['query'], 'exact-query')
        self.assertEqual(inserted['resolver_state'], 'pending')
        self.assertEqual(inserted['resolver_due_at'], inserted['available_after'])
        replay = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'exact-query', repositories,
        )
        self.assertEqual(replay['inserted_count'], 0)
        self.assertEqual(replay['preexisting_count'], 7)
        with self.assertRaisesRegex(ValueError, 'bare repository anchor'):
            self.db.persist_dockerhub_discovery_page(
                'dockerhub', 'exact-query', ['fixture/tagged:latest'],
            )

    def test_discovery_retry_schema_validator_repairs_required_index_shape(self):
        self.db.conn.execute('DROP INDEX idx_discovery_retry_queue_due')
        self.db.conn.execute(
            'CREATE INDEX idx_discovery_retry_queue_due ON discovery_retry_queue(source)'
        )
        self.db.conn.commit()
        self.assertFalse(self.db.runtime_safety_schema_available())
        self.assertIn('index idx_discovery_retry_queue_due', self.db.last_error)
        migrate_runtime_safety_schema(self.db)
        self.assertTrue(self.db.runtime_safety_schema_available(), self.db.last_error)
        index = self.db.conn.table_indexes('discovery_retry_queue')[
            'idx_discovery_retry_queue_due'
        ]
        self.assertEqual(index['columns'], ['source', 'available_after', 'id'])
        self.assertEqual(
            scanner_db._normalized_predicate(index['predicate']),
            scanner_db._normalized_predicate(scanner_db.DISCOVERY_RETRY_DUE_INDEX_PREDICATE),
        )

    def test_discovery_retry_coalescing_refund_hold_and_stale_fences(self):
        policy = 'a' * 64
        first = self.db.enqueue_discovery_retry(
            'dockerhub', 'exact-query', policy, 'deep', 'query',
            error_category='page_unavailable',
        )
        second = self.db.enqueue_discovery_retry(
            'dockerhub', 'exact-query', policy, 'deep', 'query',
            error_category='rate_limit',
        )
        self.assertEqual(first['id'], second['id'])
        self.assertEqual(first['work_key'], second['work_key'])
        self.assertEqual(first['inserted_count'], 1)
        self.assertEqual(second['coalesced_count'], 1)
        self.assertEqual(first['page_start'], 1)
        self.assertEqual(first['page_end'], scanner_db.DISCOVERY_RETRY_MAX_PAGE)

        claim = self.db.claim_discovery_retries(
            'dockerhub', {'exact-query': policy}, 'owner-a', lease_seconds=60,
        )[0]
        self.assertEqual(claim['attempts'], 1)
        renewed = self.db.renew_discovery_retry_lease(
            claim['id'], claim['lease_owner'], claim['lease_token'], lease_seconds=120,
        )
        self.assertEqual(renewed['status'], 'leased')
        self.assertGreater(renewed['lease_expires_at'], claim['lease_expires_at'])
        retry_at = (
            scanner_db.datetime.now(scanner_db.timezone.utc)
            + scanner_db.timedelta(seconds=30)
        ).isoformat(timespec='seconds')
        deferred = self.db.update_discovery_retry(
            claim['id'], claim['lease_owner'], claim['lease_token'], 'provider_cooldown',
            retry_at=retry_at, refund_attempt=True,
        )
        self.assertEqual(deferred['status'], 'pending')
        self.assertEqual(deferred['attempts'], 0)
        self.db.conn.execute(
            'UPDATE discovery_retry_queue SET available_after = NULL WHERE id = ?',
            (claim['id'],),
        )
        self.db.conn.commit()

        reclaimed = self.db.claim_discovery_retries(
            'dockerhub', [('exact-query', policy)], 'owner-b', lease_seconds=60,
        )[0]
        self.assertNotEqual(reclaimed['lease_token'], claim['lease_token'])
        with self.assertRaises(DiscoveryRetryLeaseError):
            self.db.finish_discovery_retry(
                claim['id'], claim['lease_owner'], claim['lease_token'],
            )
        with self.assertRaises(DiscoveryRetryLeaseError):
            self.db.update_discovery_retry(
                claim['id'], claim['lease_owner'], claim['lease_token'], 'network',
            )
        with self.assertRaises(DiscoveryRetryLeaseError):
            self.db.hold_discovery_retry(
                claim['id'], claim['lease_owner'], claim['lease_token'],
            )
        with self.assertRaises(DiscoveryRetryLeaseError):
            self.db.renew_discovery_retry_lease(
                claim['id'], claim['lease_owner'], claim['lease_token'],
            )
        held = self.db.hold_discovery_retry(
            reclaimed['id'], reclaimed['lease_owner'], reclaimed['lease_token'],
            'policy_mismatch',
        )
        self.assertEqual(held['status'], 'held')
        reactivated = self.db.enqueue_discovery_retry(
            'dockerhub', 'exact-query', policy, 'deep', 'query',
        )
        self.assertEqual(reactivated['status'], 'pending')
        final_claim = self.db.claim_discovery_retries(
            'dockerhub', {'exact-query': policy}, 'owner-c', lease_seconds=60,
        )[0]
        self.assertEqual(
            self.db.finish_discovery_retry(
                final_claim['id'], final_claim['lease_owner'], final_claim['lease_token'],
            )['deleted_count'],
            1,
        )
        self.assertIsNone(self.db.conn.execute(
            'SELECT id FROM discovery_retry_queue WHERE id = ?', (first['id'],),
        ).fetchone())
        with self.assertRaisesRegex(ValueError, 'error category'):
            self.db.enqueue_discovery_retry(
                'dockerhub', 'another-query', policy, 'ordinary', 'page', 1, 1,
                error_category='raw credential detail',
            )
        mismatched = self.db.enqueue_discovery_retry(
            'dockerhub', 'policy-query', 'c' * 64, 'ordinary', 'page', 1, 1,
        )
        self.assertEqual(
            self.db.claim_discovery_retries(
                'dockerhub', {'policy-query': 'd' * 64}, 'policy-owner',
            ),
            [],
        )
        mismatch_row = self.db.conn.execute(
            'SELECT status, last_error_category FROM discovery_retry_queue WHERE id = ?',
            (mismatched['id'],),
        ).fetchone()
        self.assertEqual(
            (mismatch_row['status'], mismatch_row['last_error_category']),
            ('held', 'policy_mismatch'),
        )

    def test_discovery_retry_page_progress_and_completion_are_atomic(self):
        policy = 'b' * 64
        retry = self.db.enqueue_discovery_retry(
            'dockerhub', 'range-query', policy, 'ordinary', 'range', 2, 3,
        )
        claim = self.db.claim_discovery_retries(
            'dockerhub', {'range-query': policy}, 'range-owner', lease_seconds=60,
        )[0]
        page_two = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'range-query', [{'repo_name': 'fixture/range-two'}],
            claim['id'], claim['lease_owner'], claim['lease_token'], next_page=3,
        )
        self.assertEqual(page_two['retry_progress_count'], 1)
        row = self.db.conn.execute(
            'SELECT status, next_page FROM discovery_retry_queue WHERE id = ?',
            (retry['id'],),
        ).fetchone()
        self.assertEqual((row['status'], row['next_page']), ('leased', 3))
        page_three = self.db.persist_dockerhub_discovery_page(
            'dockerhub', 'range-query', [{'repo_name': 'fixture/range-three'}],
            claim['id'], claim['lease_owner'], claim['lease_token'], complete=True,
        )
        self.assertEqual(page_three['retry_completed_count'], 1)
        self.assertIsNone(self.db.conn.execute(
            'SELECT id FROM discovery_retry_queue WHERE id = ?', (retry['id'],),
        ).fetchone())
        with self.assertRaises(DiscoveryRetryLeaseError):
            self.db.persist_dockerhub_discovery_page(
                'dockerhub', 'range-query', [{'repo_name': 'fixture/stale-page'}],
                claim['id'], claim['lease_owner'], claim['lease_token'], complete=True,
            )
        self.assertIsNone(self.db.conn.execute(
            "SELECT id FROM target_queue WHERE normalized_target = 'fixture/stale-page'"
        ).fetchone())

    def test_infrastructure_refund_does_not_consume_target_attempt(self):
        claim = self.claim('https://github.com/example/refund', max_attempts=1)
        self.assertEqual(claim['attempts'], 1)
        self.assertTrue(self.db.refund_target_claim(claim['id'], claim['lease_token'], 'disk fsync failed'))
        row = self.db.target_queue_item('source', 'github', claim['target'])
        self.assertEqual(row['status'], 'pending')
        self.assertEqual(row['attempts'], 0)
        reclaimed = self.db.claim_targets(
            'source', 'github', 1, 'owner-b', 60, max_attempts=1, return_rows=True,
        )
        self.assertEqual(len(reclaimed), 1)

    def test_atomic_refund_rolls_back_if_one_claim_has_a_replacement_token(self):
        targets = [
            'https://github.com/example/refund-one',
            'https://github.com/example/refund-two',
        ]
        self.db.enqueue_targets('source', 'github', 'q', targets)
        claims = [dict(row) for row in self.db.claim_targets(
            'source', 'github', 2, 'owner-a', 60, max_attempts=3, return_rows=True,
        )]
        self.db.conn.execute(
            "UPDATE target_queue SET lease_token = 'replacement-token' WHERE id = ?",
            (claims[1]['id'],),
        )
        self.db.conn.commit()
        self.assertFalse(self.db.refund_target_claims(claims, 'setup failed'))
        rows = self.db.conn.execute(
            'SELECT id, status, attempts, lease_token FROM target_queue ORDER BY id DESC LIMIT 2'
        ).fetchall()
        by_id = {row['id']: row for row in rows}
        self.assertEqual(by_id[claims[0]['id']]['status'], 'in_progress')
        self.assertEqual(by_id[claims[0]['id']]['attempts'], 1)
        self.assertEqual(by_id[claims[0]['id']]['lease_token'], claims[0]['lease_token'])
        self.assertEqual(by_id[claims[1]['id']]['lease_token'], 'replacement-token')


class FakePostgresDB:
    def __init__(self, enqueue_ok=True, sequence=None):
        self.conn = SimpleNamespace(is_postgres=True)
        self.postgres_required = True
        self.enqueue_ok = enqueue_ok
        self.sequence = sequence if sequence is not None else []
        self.targets = []
        self.refunded = []
        self.observations = []
        self.enqueue_discovery_admission = []

    def require_runtime_safety_schema(self):
        return True

    def known_target_normalizations(self, source, platform):
        return set()

    def enqueue_targets(
        self, source, platform, query, targets, requeue_done=False,
        unresolved_targets=None, *, discovery_admission=False,
    ):
        values = [*targets, *(unresolved_targets or [])]
        self.sequence.append('enqueue')
        self.enqueue_discovery_admission.append(discovery_admission)
        if not self.enqueue_ok:
            return 0
        self.targets.extend(values)
        return len(values)

    def observe_discovered_targets(
        self, source, platform, query, discoveries, rescan_limit=0, cooldown_seconds=0,
    ):
        records = [dict(item) for item in discoveries]
        self.sequence.append('observe')
        self.observations.append({
            'source': source, 'platform': platform, 'query': query,
            'discoveries': records, 'rescan_limit': rescan_limit,
            'cooldown_seconds': cooldown_seconds,
        })
        self.targets.extend(item['target'] for item in records)
        return {
            'attempted_count': len(records),
            'queued_new_count': max(0, len(records) - 1),
            'queued_updated_count': min(1, len(records)),
        }

    def claim_targets(
        self, source, platform, limit, owner, lease_seconds,
        max_attempts=0, return_rows=False, claim_batch=None,
    ):
        self.sequence.append('claim')
        return [
            {
                'id': index + 1,
                'target': target,
                'normalized_target': normalize_target(target, platform),
                'attempts': 1,
                'lease_owner': owner,
                'lease_token': f'token-{index + 1}',
                'claim_batch': claim_batch,
            }
            for index, target in enumerate(self.targets[:limit])
        ]

    def recover_claim_batch(self, claim_batch, owner):
        self.sequence.append('recover')
        return []

    def refund_target_claims(self, claims, error):
        self.sequence.append('refund')
        self.refunded.extend((dict(claim), error) for claim in claims)
        return True

    def target_queue_counts(self, source):
        return {'pending': len(self.targets)}


def prepare_args(temp_dir):
    return SimpleNamespace(
        save_dir=temp_dir,
        queue_dir=os.path.join(temp_dir, 'queues'),
        platform='github',
        query='q',
        workers=1,
        target_claim_batch_size=1,
        max_targets=0,
        timeout=60,
        target_retry_max_attempts=3,
        runtime_dir=os.path.join(temp_dir, 'runtime'),
        result_spool_dir=os.path.join(temp_dir, 'runtime', 'spool'),
        result_spool_max_event_bytes=1024,
        result_spool_max_events=10,
        result_spool_max_total_bytes=1024 * 20,
        result_spool_min_free_bytes=0,
    )


class DiscoveryAuthorityTests(unittest.TestCase):
    def test_prepare_targets_marks_provider_enqueue_as_discovery_admission(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.sync_file_queues = False
            db = FakePostgresDB()
            console_runner.prepare_targets(
                args, ['https://github.com/acme/repository'], db, 1, 1,
                'github', enqueue_only=True,
            )
        self.assertEqual(db.enqueue_discovery_admission, [True])

    def test_updated_discovery_deduplicates_timestamps_and_reports_promotions(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.sync_file_queues = False
            args.updated_target_rescan_enabled = True
            args.updated_target_rescan_max_per_cycle = 1
            args.updated_target_rescan_cooldown_hours = 24
            db = FakePostgresDB()
            _, _, _, queue_info = console_runner.prepare_targets(
                args,
                [
                    {'target': 'https://github.com/acme/one.git', 'remote_modified_at': '2026-08-24T00:00:00Z'},
                    {'target': 'https://github.com/acme/one', 'remote_modified_at': '2026-08-25T00:00:00Z'},
                    {'target': 'https://github.com/acme/two', 'remote_modified_at': 'not-a-time'},
                ],
                db, 1, 1, 'github', enqueue_only=True,
            )
        self.assertEqual(db.sequence, ['observe'])
        observation = db.observations[0]
        self.assertEqual(observation['rescan_limit'], 1)
        self.assertEqual(observation['cooldown_seconds'], 24 * 3600)
        self.assertEqual(observation['discoveries'], [
            {
                'target': 'https://github.com/acme/one',
                'remote_modified_at': '2026-08-25T00:00:00+00:00',
            },
            {'target': 'https://github.com/acme/two', 'remote_modified_at': None},
        ])
        self.assertEqual(queue_info['queued_new_count'], 1)
        self.assertEqual(queue_info['queued_updated_count'], 1)

    def test_updated_discovery_preserves_metrics_before_queue_diagnostics(self):
        class DB(FakePostgresDB):
            def target_queue_counts(self, source):
                raise RuntimeError('queue diagnostics unavailable')

        partial_metrics = {}
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.sync_file_queues = False
            args.updated_target_rescan_enabled = True
            args.updated_target_rescan_max_per_cycle = 1
            args.updated_target_rescan_cooldown_hours = 24
            with self.assertRaisesRegex(RuntimeError, 'queue diagnostics unavailable'):
                console_runner.prepare_targets(
                    args,
                    [
                        {
                            'target': 'https://github.com/acme/changed',
                            'remote_modified_at': '2026-08-25T00:00:00Z',
                        },
                        {
                            'target': 'https://github.com/acme/new',
                            'remote_modified_at': '2026-08-25T01:00:00Z',
                        },
                    ],
                    DB(), 1, 1, 'github', enqueue_only=True,
                    partial_metrics=partial_metrics,
                )

        self.assertEqual(partial_metrics, {
            'fetched_count': 2,
            'queued_new_count': 1,
            'queued_updated_count': 1,
        })

    def test_slot_first_claim_releases_capacity_not_backed_by_queue_rows(self):
        class Lease:
            heartbeat_thread = None

            def __init__(self):
                self.release_calls = 0

            def release(self):
                self.release_calls += 1

        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.workers = 3
            db = FakePostgresDB()
            leases = [Lease(), Lease(), Lease()]
            targets, _, _, queue_info = console_runner.prepare_targets(
                args,
                ['https://github.com/example/only-one'],
                db,
                1,
                1,
                'source',
                claim_limit_override=3,
                dispatch_leases=leases,
            )
        self.assertEqual(targets, ['https://github.com/example/only-one'])
        self.assertEqual(queue_info['scan_slot_leases'], leases[:1])
        self.assertEqual(leases[0].release_calls, 0)
        self.assertEqual([lease.release_calls for lease in leases[1:]], [1, 1])

    def test_cross_process_projection_append_and_rewrite_preserve_both_updates(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            queue_dir = os.path.join(temp_dir, 'queues')
            console_runner.ensure_private_directory(queue_dir, reject_reparse=True)
            todo = os.path.join(queue_dir, 'todo_github.txt')
            console_runner.write_lines(todo, ['https://github.com/example/original'])
            context = multiprocessing.get_context('spawn')
            start = context.Event()
            outcomes = context.Queue()
            processes = [
                context.Process(target=projection_worker, args=(mode, queue_dir, start, outcomes))
                for mode in ('enqueue', 'complete')
            ]
            for process in processes:
                process.start()
            start.set()
            for process in processes:
                process.join(15)
                self.assertFalse(process.is_alive())
                self.assertEqual(process.exitcode, 0)
            self.assertEqual([outcomes.get(timeout=5) for _ in processes], ['ok', 'ok'])
            self.assertEqual(console_runner.load_set_from_file(todo), {'https://github.com/example/new'})
            self.assertEqual(
                console_runner.load_set_from_file(os.path.join(queue_dir, 'checked_github.txt')),
                {'https://github.com/example/original'},
            )

    def test_enqueue_failure_has_no_file_projection(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            db = FakePostgresDB(enqueue_ok=False)
            with mock.patch.object(console_runner, '_append_lines_unlocked') as append:
                with self.assertRaises(RuntimeError):
                    console_runner.prepare_targets(
                        prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1, 'source',
                    )
            append.assert_not_called()
            self.assertEqual(db.sequence, ['enqueue'])

    def test_projection_failure_after_commit_leaves_target_claimable(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            sequence = []
            db = FakePostgresDB(sequence=sequence)

            def projection_failure(path, rows):
                sequence.append('projection')
                raise OSError('projection unavailable')

            with mock.patch.object(console_runner, '_append_lines_unlocked', side_effect=projection_failure):
                targets, _, _, _ = console_runner.prepare_targets(
                    prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1, 'source',
                )
            self.assertEqual(targets, ['https://github.com/a/b'])
            self.assertEqual(sequence, ['enqueue', 'projection', 'claim'])

    def test_reservation_bind_failure_refunds_claim_batch_before_release(self):
        class Spool:
            def __init__(self):
                self.released = []

            def reserve_claims(self, owner, count, lease_seconds):
                return 'reservation-id'

            def bind_claims(self, reservation_id, claims):
                raise OSError('reservation fsync failed')

            def release_reservation(self, reservation_id):
                self.released.append(reservation_id)
                return True

        with tempfile.TemporaryDirectory() as temp_dir:
            db = FakePostgresDB()
            spool = Spool()
            with mock.patch.object(console_runner, '_append_lines_unlocked'):
                with self.assertRaisesRegex(OSError, 'fsync'):
                    console_runner.prepare_targets(
                        prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1,
                        'source', spool=spool,
                    )
            self.assertEqual(len(db.refunded), 1)
            self.assertEqual(db.refunded[0][0]['claim_batch'], 'reservation-id')
            self.assertEqual(spool.released, ['reservation-id'])

    def test_empty_claim_recovery_raises_original_and_releases_reservation(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('claim rolled back before commit')

            def recover_claim_batch(self, claim_batch, owner):
                return []

        class Spool:
            def __init__(self):
                self.released = []

            def reserve_claims(self, owner, count, lease_seconds):
                return 'ambiguous-reservation'

            def release_reservation(self, reservation_id):
                self.released.append(reservation_id)
                return True

        with tempfile.TemporaryDirectory() as temp_dir:
            db = DB()
            spool = Spool()
            with mock.patch.object(console_runner, '_append_lines_unlocked'):
                with self.assertRaisesRegex(OSError, 'claim rolled back before commit'):
                    console_runner.prepare_targets(
                        prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1,
                        'source', spool=spool,
                    )
            self.assertEqual(spool.released, ['ambiguous-reservation'])

    def test_lost_connection_recovery_reports_both_errors_and_keeps_reservation(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('claim commit acknowledgement lost')

            def recover_claim_batch(self, claim_batch, owner):
                raise ConnectionError('recovery connection unavailable')

        class Spool:
            def __init__(self):
                self.released = []

            def reserve_claims(self, owner, count, lease_seconds):
                return 'lost-connection-reservation'

            def release_reservation(self, reservation_id):
                self.released.append(reservation_id)

        with tempfile.TemporaryDirectory() as temp_dir:
            db = DB()
            spool = Spool()
            with mock.patch.object(console_runner, '_append_lines_unlocked'):
                with self.assertRaisesRegex(RuntimeError, 'claim commit acknowledgement lost.*recovery connection unavailable'):
                    console_runner.prepare_targets(
                        prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1,
                        'source', spool=spool,
                    )
            self.assertEqual(spool.released, [])

    def test_zero_row_recovery_release_failure_is_combined_fatal_error(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('claim acknowledgement lost')

            def recover_claim_batch(self, claim_batch, owner):
                return []

        class Spool:
            def reserve_claims(self, owner, count, lease_seconds):
                return 'rollback-failure-reservation'

            def release_reservation(self, reservation_id):
                raise OSError('reservation fsync unavailable')

        with tempfile.TemporaryDirectory() as temp_dir:
            with mock.patch.object(console_runner, '_append_lines_unlocked'), \
                    self.assertRaisesRegex(
                        RuntimeError,
                        'FATAL durability error.*claim acknowledgement lost.*reservation fsync unavailable',
                    ):
                console_runner.prepare_targets(
                    prepare_args(temp_dir), ['https://github.com/a/b'], DB(), 1, 1,
                    'source', spool=Spool(),
                )

    def test_zero_row_recovery_with_nonempty_expectation_keeps_reservation(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('ambiguous claim')

            def recover_claim_batch(self, claim_batch, owner):
                return []

            def claim_recovery_expectation(self, claim_batch, owner):
                return [{'id': 1, 'lease_token': 'expected-token'}]

        class Spool:
            def __init__(self):
                self.released = []

            def reserve_claims(self, owner, count, lease_seconds):
                return 'expected-row-reservation'

            def release_reservation(self, reservation_id):
                self.released.append(reservation_id)
                return True

        with tempfile.TemporaryDirectory() as temp_dir:
            spool = Spool()
            with mock.patch.object(console_runner, '_append_lines_unlocked'), \
                    self.assertRaisesRegex(RuntimeError, 'exact pre-commit fenced row set'):
                console_runner.prepare_targets(
                    prepare_args(temp_dir), ['https://github.com/a/b'], DB(), 1, 1,
                    'source', spool=spool,
                )
            self.assertEqual(spool.released, [])

    def test_zero_row_recovery_restores_real_spool_claim_capacity(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('claim rolled back before commit')

            def recover_claim_batch(self, claim_batch, owner):
                return []

        with tempfile.TemporaryDirectory() as temp_dir:
            spool = ResultSpool(
                os.path.join(temp_dir, 'spool'),
                max_event_bytes=1024,
                max_events=2,
                max_total_bytes=4096,
                min_free_bytes=0,
            )
            with mock.patch.object(console_runner, '_append_lines_unlocked'), \
                    self.assertRaisesRegex(OSError, 'claim rolled back before commit'):
                console_runner.prepare_targets(
                    prepare_args(temp_dir), ['https://github.com/a/b'], DB(), 1, 1,
                    'source', spool=spool,
                )
            retry_reservation = spool.reserve_claims('retry-owner', 1, 60, bytes_per_event=1024)
            self.assertTrue(spool.release_reservation(retry_reservation))

    def test_inconsistent_claim_recovery_fails_without_releasing_reservation(self):
        class DB(FakePostgresDB):
            def claim_targets(self, *args, **kwargs):
                raise OSError('ambiguous claim')

            def recover_claim_batch(self, claim_batch, owner):
                return [{
                    'id': 1, 'target': 'https://github.com/a/b', 'attempts': 1,
                    'lease_owner': 'wrong-owner', 'lease_token': 'token-1',
                    'claim_batch': claim_batch,
                }]

        class Spool:
            def __init__(self):
                self.released = []

            def reserve_claims(self, owner, count, lease_seconds):
                return 'inconsistent-reservation'

            def release_reservation(self, reservation_id):
                self.released.append(reservation_id)

        with tempfile.TemporaryDirectory() as temp_dir:
            db = DB()
            spool = Spool()
            with mock.patch.object(console_runner, '_append_lines_unlocked'):
                with self.assertRaisesRegex(RuntimeError, 'inconsistent fenced row metadata'):
                    console_runner.prepare_targets(
                        prepare_args(temp_dir), ['https://github.com/a/b'], db, 1, 1,
                        'source', spool=spool,
                    )
            self.assertEqual(spool.released, [])

    def test_pending_spool_ingest_failure_prevents_fetch_and_claim(self):
        record = SimpleNamespace(envelope={'event': 1}, event_id='event-id', event_hash='hash')

        class FailingDB:
            postgres_required = True
            conn = SimpleNamespace(is_postgres=True)

            def require_runtime_safety_schema(self):
                return True

            def ingest_scan_event(self, envelope):
                raise RuntimeError('database unavailable')

        spool = SimpleNamespace(pending_events=lambda: [record])
        args = SimpleNamespace(platform='github')
        with mock.patch.object(console_runner, 'result_spool_for_args', return_value=spool), \
                mock.patch.object(console_runner, 'fetch_targets') as fetch, \
                mock.patch.object(console_runner, 'prepare_targets') as prepare:
            with self.assertRaisesRegex(RuntimeError, 'database unavailable'):
                console_runner.run_cycle(args, FailingDB(), 1, 1, 'source')
        fetch.assert_not_called()
        prepare.assert_not_called()

    def test_failed_ingestion_does_not_delete_spooled_event(self):
        class FailingDB:
            conn = SimpleNamespace(is_postgres=True)

            def require_runtime_safety_schema(self):
                return True

            def ingest_scan_event(self, envelope):
                raise RuntimeError('database unavailable')

        with tempfile.TemporaryDirectory() as temp_dir:
            spool = ResultSpool(os.path.join(temp_dir, 'spool'), min_free_bytes=0)
            record = spool.write_event({
                'version': 1,
                'scan_event_id': str(uuid.uuid4()),
                'result': {},
            })
            with self.assertRaisesRegex(RuntimeError, 'database unavailable'):
                console_runner.drain_result_spool(spool, FailingDB())
            self.assertTrue(os.path.exists(record.path))
            self.assertEqual(len(spool.pending_events()), 1)

    def test_mismatched_ingestion_acknowledgement_keeps_event_pending(self):
        class WrongAckDB:
            conn = SimpleNamespace(is_postgres=True)

            def require_runtime_safety_schema(self):
                return True

            def ingest_scan_event(self, envelope):
                return {
                    'ingested': True,
                    'scan_event_id': envelope['scan_event_id'],
                    'scan_event_hash': '0' * 64,
                }

        with tempfile.TemporaryDirectory() as temp_dir:
            spool = ResultSpool(os.path.join(temp_dir, 'spool'), min_free_bytes=0)
            record = spool.write_event({
                'version': 1,
                'scan_event_id': str(uuid.uuid4()),
                'result': {},
            })
            with self.assertRaisesRegex(RuntimeError, 'did not confirm'):
                console_runner.drain_result_spool(spool, WrongAckDB())
            self.assertTrue(os.path.exists(record.path))

    def test_heartbeat_recheck_tolerates_token_removed_during_query(self):
        active = {'token-a'}
        lock = __import__('threading').Lock()

        class DB:
            def active_target_lease_tokens(self, owner, tokens):
                with lock:
                    active.clear()
                return set()

        self.assertTrue(console_runner.recheck_active_lease_ownership(
            DB(), 'owner', ['token-a'], active, lock,
        ))

    def test_outbox_delivery_requires_fenced_finish_acknowledgement(self):
        class DB:
            conn = SimpleNamespace(is_postgres=True)

            def claim_scan_publications(self, owner, limit):
                return [{'id': 7, 'payload_json': '{}'}]

            def finish_scan_publication(self, outbox_id, owner, delivered, error):
                return False

        with mock.patch.object(console_runner, 'publish_scan_payload', return_value=(True, '')):
            with self.assertRaisesRegex(RuntimeError, 'acknowledgement failed'):
                console_runner.drain_scan_publication_outbox(DB(), 1)

    def test_outbox_postman_replay_revalidates_containment_size_and_hash_before_candidates(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            cache_dir = os.path.join(temp_dir, 'cache')
            scanner.ensure_private_directory(cache_dir, reject_reparse=True)
            runtime_patch = mock.patch.object(scanner.scan_config, 'runtime_dir', temp_dir)
            cache_patch = mock.patch.object(scanner.scan_config, 'postman_cache_dir', cache_dir)
            runtime_patch.start()
            cache_patch.start()
            self.addCleanup(runtime_patch.stop)
            self.addCleanup(cache_patch.stop)
            cache_path, digest, size = scanner.write_postman_cache(
                b'{"token":"fixture"}', cache_dir=cache_dir,
            )
            postman = {
                'source': 'cache', 'cache_path': cache_path,
                'sha256': digest, 'size': size,
            }
            result = {
                'structured_keycheck_pending': True,
                'postman': postman,
                'bytes': size,
                'postman_max_artifact_size_mb': 1,
            }
            with mock.patch.object(scanner.scan_config, 'postman_cache_dir', cache_dir), \
                    mock.patch.object(console_runner, 'write_structured_keycheck_candidates', return_value={'gemini': 1}) as derive, \
                    mock.patch.object(console_runner, 'save_scan_result', return_value=True) as publish:
                self.assertEqual(console_runner.publish_scan_payload(copy.deepcopy(result)), (True, ''))
                derive.assert_called_once()
                publish.assert_called_once()

                derive.reset_mock()
                publish.reset_mock()
                mismatched_size = copy.deepcopy(result)
                mismatched_size['bytes'] = size + 1
                ok, error = console_runner.publish_scan_payload(mismatched_size)
                self.assertFalse(ok)
                self.assertIn('size mismatch', error)
                derive.assert_not_called()
                publish.assert_not_called()

                outside = os.path.join(temp_dir, 'outside.json')
                Path(outside).write_bytes(Path(cache_path).read_bytes())
                scanner.harden_private_file(outside)
                escaped = copy.deepcopy(result)
                escaped['postman']['cache_path'] = outside
                ok, error = console_runner.publish_scan_payload(escaped)
                self.assertFalse(ok)
                self.assertIn('escapes', error)
                derive.assert_not_called()
                publish.assert_not_called()

                Path(cache_path).write_bytes(b'{"token":"tampered"}')
                ok, error = console_runner.publish_scan_payload(copy.deepcopy(result))
                self.assertFalse(ok)
                self.assertTrue('size mismatch' in error or 'SHA-256 mismatch' in error)
                derive.assert_not_called()
                publish.assert_not_called()

    def test_candidate_retry_count_cannot_change_projected_payload(self):
        scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
        with tempfile.TemporaryDirectory() as temp_dir:
            results_dir = os.path.join(temp_dir, 'results')
            scanner.ensure_private_directory(temp_dir, reject_reparse=True)
            scanner.ensure_private_directory(results_dir, reject_reparse=True)
            event_id = str(uuid.uuid4())
            authoritative = {
                'scan_event_id': event_id,
                'target': 'postman:sha256:' + ('a' * 64),
                'scan_type': 'postman',
                'timestamp': '2026-07-19T00:00:00+00:00',
                'findings': [{'DetectorName': 'OpenAI', 'Raw': 'fixture-secret'}],
                'errors': [],
                'structured_keycheck_pending': True,
                'postman': {'cache_path': 'fixture.json', 'sha256': 'a' * 64, 'size': 10},
                'bytes': 10,
                'postman_max_artifact_size_mb': 1,
            }
            original = copy.deepcopy(authoritative)
            inserted_counts = iter(({'gemini': 1}, {}))
            candidate_outcomes = []

            def candidates(_path, postman):
                postman['candidate_writer_mutation'] = True
                outcome = next(inserted_counts)
                candidate_outcomes.append(outcome)
                return outcome

            real_append = scanner.append_rotating_jsonl_once
            ambiguous_finding = {'pending': True}

            def append_then_fail_once(path, *args, **kwargs):
                result = real_append(path, *args, **kwargs)
                if os.path.basename(path) == 'found_secrets.jsonl' and ambiguous_finding['pending']:
                    ambiguous_finding['pending'] = False
                    return False
                return result

            with mock.patch.object(scanner.scan_config, 'results_dir', results_dir), \
                    mock.patch.object(scanner.scan_config, 'jsonl_rotation_enabled', False), \
                    mock.patch.object(console_runner, 'validate_postman_cache_artifact', return_value=('fixture.json', 10)), \
                    mock.patch.object(console_runner, 'write_structured_keycheck_candidates', side_effect=candidates), \
                    mock.patch.object(scanner, 'write_foundry_keycheck_candidates_from_findings', return_value=0), \
                    mock.patch.object(scanner, 'append_rotating_jsonl_once', side_effect=append_then_fail_once):
                first_ok, first_error = console_runner.publish_scan_payload(copy.deepcopy(authoritative))
                second_ok, second_error = console_runner.publish_scan_payload(copy.deepcopy(authoritative))

            self.assertFalse(first_ok)
            self.assertIn('publication', first_error)
            self.assertTrue(second_ok, second_error)
            self.assertEqual(candidate_outcomes, [{'gemini': 1}, {}])
            self.assertEqual(authoritative, original)

            scan_rows = [
                json.loads(line)
                for line in Path(os.path.join(results_dir, 'scan_results.jsonl')).read_text(encoding='utf-8').splitlines()
            ]
            finding_rows = [
                json.loads(line)
                for line in Path(os.path.join(results_dir, 'found_secrets.jsonl')).read_text(encoding='utf-8').splitlines()
            ]
            self.assertEqual(len(scan_rows), 1)
            self.assertEqual(len(finding_rows), 1)
            self.assertEqual(scan_rows[0]['scan_event_id'], event_id)
            self.assertNotIn('structured_keycheck_candidates', scan_rows[0])
            self.assertNotIn('candidate_writer_mutation', scan_rows[0]['postman'])

    def test_postman_cache_tiny_aggregate_quotas_fail_without_evicting_artifacts(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            cache_dir = os.path.join(temp_dir, 'cache')
            scanner.ensure_private_directory(cache_dir, reject_reparse=True)
            with mock.patch.object(scanner.scan_config, 'runtime_dir', temp_dir):
                first_path, _, _ = scanner.write_postman_cache(
                    b'{"first":"value"}',
                    cache_dir=cache_dir,
                    cache_max_items=2,
                    cache_max_bytes=1024 * 1024,
                    cache_min_free_bytes=0,
                )
                scanner.write_postman_cache(
                    b'{"second":"value"}',
                    cache_dir=cache_dir,
                    cache_max_items=2,
                    cache_max_bytes=1024 * 1024,
                    cache_min_free_bytes=0,
                )
                failures = 0
                for index in range(10):
                    with self.assertRaises(scanner.PostmanCacheCapacityError):
                        scanner.write_postman_cache(
                            f'{{"extra":{index}}}'.encode('ascii'),
                            cache_dir=cache_dir,
                            cache_max_items=2,
                            cache_max_bytes=1024 * 1024,
                            cache_min_free_bytes=0,
                        )
                    failures += 1
            usage = scanner.postman_cache_usage(cache_dir)
            self.assertEqual(failures, 10)
            self.assertEqual(usage['items'], 2)
            self.assertTrue(os.path.exists(first_path))
            self.assertTrue(os.path.exists(first_path + '.meta.json'))

    def test_postman_cache_quota_reserves_metadata_and_leaves_no_partial_artifact(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            cache_dir = os.path.join(temp_dir, 'cache')
            scanner.ensure_private_directory(cache_dir, reject_reparse=True)
            with mock.patch.object(scanner.scan_config, 'runtime_dir', temp_dir), \
                    self.assertRaises(scanner.PostmanCacheCapacityError):
                    scanner.write_postman_cache(
                        b'{"payload":"value"}',
                        cache_dir=cache_dir,
                        cache_max_items=1,
                        cache_max_bytes=24,
                        cache_min_free_bytes=0,
                    )
            self.assertEqual(scanner.postman_cache_usage(cache_dir), {'items': 0, 'files': 0, 'bytes': 0})

    def test_postman_cache_concurrent_writers_cannot_overcommit_one_item_quota(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            cache_dir = os.path.join(temp_dir, 'cache')
            scanner.ensure_private_directory(cache_dir, reject_reparse=True)
            barrier = threading.Barrier(2)
            outcomes = []
            outcome_lock = threading.Lock()

            def publish(index):
                barrier.wait(5)
                try:
                    scanner.write_postman_cache(
                        f'{{"writer":{index}}}'.encode('ascii'),
                        cache_dir=cache_dir,
                        cache_max_items=1,
                        cache_max_bytes=1024 * 1024,
                        cache_min_free_bytes=0,
                    )
                    result = 'written'
                except scanner.PostmanCacheCapacityError:
                    result = 'capacity'
                with outcome_lock:
                    outcomes.append(result)

            threads = [threading.Thread(target=publish, args=(index,)) for index in range(2)]
            with mock.patch.object(scanner.scan_config, 'runtime_dir', temp_dir):
                for thread in threads:
                    thread.start()
                for thread in threads:
                    thread.join(10)
            self.assertEqual(sorted(outcomes), ['capacity', 'written'])
            self.assertEqual(scanner.postman_cache_usage(cache_dir)['items'], 1)

    def test_postman_cache_outside_configured_runtime_is_rejected(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            runtime_dir = os.path.join(temp_dir, 'runtime')
            outside = os.path.join(temp_dir, 'outside-cache')
            scanner.ensure_private_directory(runtime_dir, reject_reparse=True)
            scanner.ensure_private_directory(outside, reject_reparse=True)
            with mock.patch.object(scanner.scan_config, 'runtime_dir', runtime_dir):
                with self.assertRaisesRegex(scanner.PostmanCacheValidationError, 'runtime directory'):
                    scanner.write_postman_cache(
                        b'{"outside":true}',
                        cache_dir=outside,
                        cache_max_items=1,
                        cache_max_bytes=1024,
                        cache_min_free_bytes=0,
                    )

    def test_heartbeat_setup_failure_refunds_all_claims_before_scanning(self):
        refunded = []
        released = []

        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True
            path = None
            url = 'postgresql://truf:secret@127.0.0.1:5432/truf'

            def require_runtime_safety_schema(self):
                return True

            def claim_scan_publications(self, owner, limit):
                return []

            def refund_target_claims(self, claims, error):
                refunded.extend(dict(claim) for claim in claims)
                return True

        class Spool:
            def next_pending_event(self):
                return None

            def assert_claims_allowed(self):
                return True

            def release_reservation(self, reservation_id):
                released.append(reservation_id)
                return True

        disabled_heartbeat = SimpleNamespace(enabled=False, close=mock.Mock())
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.detectors = ''
            args.exclude_detectors = ''
            args.no_verification = True
            args.trufflehog_config = ''
            args.drop_detectors = []
            args.token = None
            target = 'https://github.com/example/heartbeat-setup'
            claim = {
                'id': 9, 'attempts': 1, 'lease_owner': 'owner',
                'lease_token': 'lease-token', 'claim_batch': 'reservation',
            }
            queue_info = {
                'fetched_count': 1, 'queued_new_count': 0, 'scan_requested_count': 1,
                'lease_owner': 'owner', 'lease_tokens': ['lease-token'], 'lease_seconds': 1800,
                'queue_claims': {target: claim}, 'spool_reservation_id': 'reservation',
            }
            with mock.patch.object(console_runner, 'result_spool_for_args', return_value=Spool()), \
                    mock.patch.object(console_runner, 'fetch_targets', return_value=[target]), \
                    mock.patch.object(console_runner, 'prepare_targets', return_value=(
                        [target], os.path.join(args.queue_dir, 'todo.txt'),
                        os.path.join(args.queue_dir, 'checked.txt'), queue_info,
                    )), \
                    mock.patch.object(console_runner, 'ScannerDB', return_value=disabled_heartbeat), \
                    mock.patch.object(console_runner, 'scan_targets_batch') as scan:
                with self.assertRaisesRegex(RuntimeError, 'heartbeat DB'):
                    console_runner.run_cycle(args, DB(), 1, 1, 'source')
            scan.assert_not_called()
            self.assertEqual([claim['id'] for claim in refunded], [9])
            self.assertEqual(released, ['reservation'])

    def test_scan_option_helper_failure_uses_common_refund_guard(self):
        refunded = []
        released = []
        partial_metrics = {}

        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True

            def require_runtime_safety_schema(self):
                return True

            def claim_scan_publications(self, owner, limit):
                return []

            def refund_target_claims(self, claims, error):
                refunded.extend((dict(claim), error) for claim in claims)
                return True

        class Spool:
            def next_pending_event(self):
                return None

            def assert_claims_allowed(self):
                return True

            def release_reservation(self, reservation_id):
                released.append(reservation_id)
                return True

        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.drop_detectors = []
            target = 'https://github.com/example/options'
            claim = {'id': 11, 'lease_owner': 'owner', 'lease_token': 'token-11'}
            queue_info = {
                'fetched_count': 3, 'queued_new_count': 1, 'queued_updated_count': 2,
                'scan_requested_count': 1,
                'lease_owner': None, 'lease_seconds': 0,
                'queue_claims': {target: claim}, 'spool_reservation_id': 'reservation-11',
            }
            with mock.patch.object(console_runner, 'result_spool_for_args', return_value=Spool()), \
                    mock.patch.object(console_runner, 'fetch_targets', return_value=[target]), \
                    mock.patch.object(console_runner, 'prepare_targets', return_value=(
                        [target], os.path.join(args.queue_dir, 'todo.txt'),
                        os.path.join(args.queue_dir, 'checked.txt'), queue_info,
                    )), \
                    mock.patch.object(console_runner, 'csv_items', side_effect=ValueError('bad detector CSV')), \
                    mock.patch.object(console_runner, 'scan_targets_batch') as scan:
                with self.assertRaisesRegex(ValueError, 'bad detector CSV'):
                    console_runner.run_cycle(
                        args, DB(), 1, 1, 'source', partial_metrics=partial_metrics,
                    )
            scan.assert_not_called()
        self.assertEqual([claim['id'] for claim, _ in refunded], [11])
        self.assertEqual(released, ['reservation-11'])
        self.assertEqual(partial_metrics['fetched_count'], 3)
        self.assertEqual(partial_metrics['queued_new_count'], 1)
        self.assertEqual(partial_metrics['queued_updated_count'], 2)

    def test_v2_failure_after_discovery_preserves_admission_metrics(self):
        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True
            last_error = None

            def require_runtime_safety_schema(self):
                return True

            def require_final_cutover(self):
                return True

            def has_claimable_targets_v2(self, source, platform, max_attempts=0):
                return False

            def reserve_and_claim_target(self, *args, **kwargs):
                raise AssertionError('failure must occur before target admission')

        partial_metrics = {}
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.result_bundle_dir = temp_dir
            discovery_info = {
                'fetched_count': 5,
                'queued_new_count': 3,
                'queued_updated_count': 2,
            }
            with mock.patch.dict(os.environ, {'TRUF_SUPERVISOR_INSTANCE_ID': 'fixture'}), \
                    mock.patch.object(
                        console_runner, 'require_private_directory', return_value=temp_dir,
                    ), \
                    mock.patch.object(console_runner, 'fetch_targets', return_value=['target']), \
                    mock.patch.object(console_runner, 'prepare_targets', return_value=(
                        [], None, None, discovery_info,
                    )), \
                    mock.patch.object(
                        console_runner, 'prepare_scan_options',
                        side_effect=RuntimeError('post-discovery failure'),
                    ):
                with self.assertRaisesRegex(RuntimeError, 'post-discovery failure'):
                    console_runner.run_cycle(
                        args, DB(), 1, 1, 'source', partial_metrics=partial_metrics,
                    )
        self.assertEqual(partial_metrics, discovery_info)

    def test_heartbeat_thread_start_failure_uses_common_refund_guard(self):
        refunded = []
        released = []

        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True
            path = None
            url = 'postgresql://truf:secret@127.0.0.1:5432/truf'

            def require_runtime_safety_schema(self):
                return True

            def claim_scan_publications(self, owner, limit):
                return []

            def refund_target_claims(self, claims, error):
                refunded.extend((dict(claim), error) for claim in claims)
                return True

        class Spool:
            def next_pending_event(self):
                return None

            def assert_claims_allowed(self):
                return True

            def release_reservation(self, reservation_id):
                released.append(reservation_id)
                return True

        heartbeat = SimpleNamespace(
            enabled=True,
            conn=SimpleNamespace(is_postgres=True, execute=mock.Mock(), commit=mock.Mock()),
            require_runtime_safety_schema=mock.Mock(return_value=True),
            close=mock.Mock(),
        )
        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.detectors = args.exclude_detectors = args.trufflehog_config = ''
            args.no_verification = True
            args.drop_detectors = []
            args.token = None
            target = 'https://github.com/example/thread-start'
            claim = {'id': 12, 'lease_owner': 'owner', 'lease_token': 'token-12'}
            queue_info = {
                'fetched_count': 1, 'queued_new_count': 0, 'scan_requested_count': 1,
                'lease_owner': 'owner', 'lease_seconds': 1800,
                'queue_claims': {target: claim}, 'spool_reservation_id': 'reservation-12',
            }
            with mock.patch.object(console_runner, 'result_spool_for_args', return_value=Spool()), \
                    mock.patch.object(console_runner, 'fetch_targets', return_value=[target]), \
                    mock.patch.object(console_runner, 'prepare_targets', return_value=(
                        [target], os.path.join(args.queue_dir, 'todo.txt'),
                        os.path.join(args.queue_dir, 'checked.txt'), queue_info,
                    )), \
                    mock.patch.object(console_runner, 'ScannerDB', return_value=heartbeat), \
                    mock.patch.object(console_runner.threading.Thread, 'start', side_effect=RuntimeError('thread start failed')), \
                    mock.patch.object(console_runner, 'scan_targets_batch') as scan:
                with self.assertRaisesRegex(RuntimeError, 'thread start failed'):
                    console_runner.run_cycle(args, DB(), 1, 1, 'source')
            scan.assert_not_called()
        self.assertEqual([claim['id'] for claim, _ in refunded], [12])
        self.assertEqual(released, ['reservation-12'])

    def test_scan_submission_failure_refunds_only_claims_without_durable_events(self):
        refunded = []

        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True
            path = None
            url = None

            def require_runtime_safety_schema(self):
                return True

            def claim_scan_publications(self, owner, limit):
                return []

            def ingest_scan_event(self, envelope):
                return {
                    'ingested': True,
                    'scan_event_id': envelope['scan_event_id'],
                    'scan_event_hash': envelope['scan_event_hash'],
                    'stale': True,
                }

            def refund_target_claims(self, claims, error):
                refunded.extend((dict(claim), error) for claim in claims)
                return True

        with tempfile.TemporaryDirectory() as temp_dir:
            args = prepare_args(temp_dir)
            args.detectors = args.exclude_detectors = args.trufflehog_config = ''
            args.no_verification = True
            args.drop_detectors = []
            args.token = None
            targets = [
                'https://github.com/example/durable',
                'https://github.com/example/not-submitted',
            ]
            spool = ResultSpool(os.path.join(temp_dir, 'spool'), min_free_bytes=0)
            reservation = spool.reserve_claims('owner', 2, 60)
            claims = [
                {
                    'id': index + 1, 'target': target, 'attempts': 1,
                    'lease_owner': 'owner', 'lease_token': f'token-{index + 1}',
                    'claim_batch': reservation,
                }
                for index, target in enumerate(targets)
            ]
            spool.bind_claims(reservation, claims)
            queue_info = {
                'fetched_count': 2, 'queued_new_count': 0, 'scan_requested_count': 2,
                'lease_owner': None, 'lease_seconds': 0,
                'queue_claims': {claim['target']: claim for claim in claims},
                'spool_reservation_id': reservation,
            }

            def submit_then_fail(*scan_args, result_sink=None, **scan_kwargs):
                result_sink({
                    'target': targets[0], 'scan_type': 'github',
                    'scan_event_id': str(uuid.uuid4()),
                    'timestamp': '2026-07-18T00:00:00+00:00',
                    'findings': [], 'errors': [],
                })
                raise RuntimeError('second submission failed')

            with mock.patch.object(console_runner, 'result_spool_for_args', return_value=spool), \
                    mock.patch.object(console_runner, 'fetch_targets', return_value=targets), \
                    mock.patch.object(console_runner, 'prepare_targets', return_value=(
                        targets, os.path.join(args.queue_dir, 'todo.txt'),
                        os.path.join(args.queue_dir, 'checked.txt'), queue_info,
                    )), \
                    mock.patch.object(console_runner, 'scan_targets_batch', side_effect=submit_then_fail):
                with self.assertRaisesRegex(RuntimeError, 'second submission failed'):
                    console_runner.run_cycle(args, DB(), 1, 1, 'source')

            self.assertEqual([claim['id'] for claim, _ in refunded], [2])
            self.assertFalse(os.path.exists(spool._reservation_path(reservation)))

    def test_spool_persistence_failure_refunds_claim_before_propagating(self):
        refunded = []

        class DB:
            conn = SimpleNamespace(is_postgres=True)
            postgres_required = True
            path = None
            url = None

            def require_runtime_safety_schema(self):
                return True

            def refund_target_claim(self, queue_id, lease_token, error):
                refunded.append((queue_id, lease_token, error))
                return True

            def claim_scan_publications(self, owner, limit):
                return []

        class Spool:
            def next_pending_event(self):
                return None

            def assert_claims_allowed(self):
                return True

            def write_event(self, *args, **kwargs):
                raise OSError('disk full')

            def release_reserved_claim(self, *args):
                return True

        args = prepare_args(tempfile.gettempdir())
        args.detectors = ''
        args.exclude_detectors = ''
        args.no_verification = True
        args.trufflehog_config = ''
        args.drop_detectors = []
        args.token = None
        target = 'https://github.com/example/refund-on-spool-failure'
        claim = {'id': 42, 'attempts': 1, 'lease_owner': 'owner', 'lease_token': 'token'}
        queue_info = {
            'fetched_count': 1, 'queued_new_count': 0, 'scan_requested_count': 1,
            'lease_owner': None, 'lease_seconds': 0,
            'queue_claims': {target: claim}, 'spool_reservation_id': 'reservation',
        }

        def complete(*scan_args, result_sink=None, **scan_kwargs):
            result_sink({
                'target': target, 'scan_type': 'github', 'scan_event_id': str(uuid.uuid4()),
                'timestamp': '2026-07-18T00:00:00+00:00', 'findings': [], 'errors': [],
            })

        with mock.patch.object(console_runner, 'result_spool_for_args', return_value=Spool()), \
                mock.patch.object(console_runner, 'fetch_targets', return_value=[target]), \
                mock.patch.object(console_runner, 'prepare_targets', return_value=(
                    [target], os.path.join(args.queue_dir, 'todo.txt'),
                    os.path.join(args.queue_dir, 'checked.txt'), queue_info,
                )), \
                mock.patch.object(console_runner, 'scan_targets_batch', side_effect=complete):
            with self.assertRaisesRegex(RuntimeError, 'infrastructure result persistence failure'):
                console_runner.run_cycle(args, DB(), 1, 1, 'source')
        self.assertEqual(refunded[0][0:2], (42, 'token'))
        self.assertIn('disk full', refunded[0][2])


if __name__ == '__main__':
    unittest.main()