"""Real PG16 regression; run only in the trusted, offline disposable test image. Main provisions a fresh truf-projection-loss-test-<32 hex> volume separately. Run python -u -I -S -B /opt/truf/tests/container_projection_recovery_e2e.py --budget-seconds 300. The budget is cooperative: it NEVER kills PostgreSQL or releases uncertain lifecycle authority. A FAILED_HOLD may outlive that budget. The actual schema/migration helpers, queries, transactions and locks are used. Only the recovery module's manifest pin is substituted in memory for this fresh fixture. Fault adapters raise only AFTER real SQL execution or real commit. No accepted journal is deleted, no old output is rebuilt, and no application initialized marker, supervisor, scanner, ingester or provider is started. """ import argparse import contextlib import hashlib import json import os from pathlib import Path import re import runpy import signal import socket import subprocess import sys import time from types import SimpleNamespace OUTPUT = sys.stdout STOPPED = False IMAGE_PATH = Path('/opt/truf/tests/container_projection_recovery_e2e.py') STAMP = '2026-09-16T00:00:00+00:00' APPENDS = 38024 OLD_OFFSET = 97783145 def require(value, check): if not value: raise AssertionError(check) def emit(**values): try: print(json.dumps(values, sort_keys=True), file=OUTPUT, flush=True) except BaseException: pass def safe_error(error): try: line, state, current = 0, None, error for _ in range(8): tb = current.__traceback__ while tb is not None: if tb.tb_frame.f_code.co_filename == str(IMAGE_PATH): line = tb.tb_lineno tb = tb.tb_next candidate = getattr(current, 'sqlstate', None) if state is None and isinstance(candidate, str) and re.fullmatch(r'[A-Z0-9]{5}', candidate): state = candidate current = current.__cause__ or current.__context__ if current is None: break name = type(error).__name__ if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,63}', name or ''): name = 'other' return {'error_type': next((i for i, kind in enumerate( (TimeoutError, OSError, ValueError, RuntimeError, KeyboardInterrupt, AssertionError, TypeError, KeyError, AttributeError, ImportError, LookupError), 1) if isinstance(error, kind)), 0), 'line': line, 'sqlstate': state, 'error_class': name} except BaseException: return {'error_type': 0, 'line': 0, 'sqlstate': None, 'error_class': 'other'} def hold_pause(): try: time.sleep(2) except BaseException: pass def stop_confirmed(backend, pg): """Retain this exact backend and the caller's two locks until positive stop.""" global STOPPED attempts = 0 while True: try: attempts += 1 result = backend.stop() require(result.completed is True and result.stopped is True, 'stop_result') require(backend.probe().kind == pg.ProbeKind.STOPPED, 'stopped_probe') backend.close() STOPPED = True emit(stage='stop', stopped=True, attempts=attempts) return except BaseException as error: emit(stage='stop', failed_hold=True, attempts=attempts, **safe_error(error)) hold_pause() def initialize_empty(runtime, config_path): """The real lifecycle child owns initialization compensation; never kill it.""" try: child = subprocess.Popen(runtime._bootstrap_command( 'postgres-runtime', 'initialize-empty', '--config', str(config_path)), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except BaseException as error: emit(stage='initialize_empty', failed_hold=True, **safe_error(error)) return None attempts = 0 while True: try: code = child.wait(timeout=10) emit(stage='initialize_empty', completed=code == 0, exit_code=code) return code except BaseException as error: attempts += 1 emit(stage='initialize_empty', failed_hold=True, attempts=attempts, **safe_error(error)) hold_pause() def checkpoint(runtime, deadline): require(runtime._shutdown_requested is False, 'shutdown_requested') if time.monotonic() >= deadline: raise TimeoutError('driver_budget') def identifier(name): require(isinstance(name, str) and re.fullmatch(r'[a-z_][a-z0-9_]*', name), 'fixture_identifier') return '"' + name + '"' def digest(payload): return hashlib.sha256(payload).hexdigest() def metadata(connection): row = connection.execute("""SELECT pg_catalog.row_to_json(s) AS stream, pg_catalog.row_to_json(c) AS cursor FROM public.projection_streams s JOIN public.projection_cursors c USING (stream_name) WHERE s.stream_name = 'found_secrets'""").fetchone() parsed = {} for key in ('stream', 'cursor'): value = row[key] if isinstance(value, str): value = json.loads(value) require(isinstance(value, dict), 'metadata_object') parsed[key] = value return parsed def proof(connection, check): """Independent, bounded-fixture whole-row digests; never return raw rows.""" tables = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relkind = 'r' ORDER BY c.relname""").fetchall() result = {'tables': {}, 'sequences': {'public': {}}} for table in tables: check() name = table['name'] where = " WHERE t.stream_name <> 'found_secrets'" if name in ('projection_streams', 'projection_cursors') else '' row = connection.execute("""SELECT count(*) AS rows, pg_catalog.encode(pg_catalog.sha256( pg_catalog.convert_to(COALESCE(string_agg(h, '' ORDER BY h), ''), 'UTF8')), 'hex') AS sha256 FROM (SELECT pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to( pg_catalog.row_to_json(t)::text, 'UTF8')), 'hex') COLLATE "C" AS h FROM public.""" + identifier(name) + ' AS t' + where + ') AS hashes').fetchone() require(0 <= row['rows'] <= 100000, 'bounded_fixture') result['tables'][name] = row sequences = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relkind = 'S' ORDER BY c.relname""").fetchall() for row in sequences: result['sequences']['public'][row['name']] = connection.execute( 'SELECT last_value, is_called FROM public.' + identifier(row['name'])).fetchone() return result def journal_snapshot(runtime, recovery): from container_import import _input, _json path = runtime.DATA / 'config' / recovery.JOURNAL_NAME with _input(path, runtime) as (handle, before): require(0 < before[4] <= recovery.MAX_JOURNAL, 'journal_bound') raw = handle.read(recovery.MAX_JOURNAL + 1) value = _json(raw) require(raw == recovery._encoded(value) and value['record']['state'] == 'PREPARED' and value['sha256'] == digest(recovery._encoded(value['record'])), 'durable_journal') return raw, before, value class ObservedConnection: """Delegate everything to psycopg; inject only after a real operation.""" def __init__(self, connection, runtime, recovery, check, *, after_update=False, lost_ack=False): self.connection, self.runtime, self.recovery, self.check = connection, runtime, recovery, check self.after_update, self.lost_ack = after_update, lost_ack self.updates, self.commits, self.injected = 0, 0, False def __getattr__(self, name): return getattr(self.connection, name) def execute(self, query, *args, **kwargs): result = self.connection.execute(query, *args, **kwargs) sql = ' '.join(query.split()).upper() if isinstance(query, str) else '' if sql.startswith('UPDATE '): self.updates += 1 if self.after_update and sql.startswith('UPDATE PUBLIC.PROJECTION_STREAMS '): require(result.rowcount == 1, 'real_first_update') journal_snapshot(self.runtime, self.recovery) self.after_update, self.injected = False, True raise OSError('synthetic_transport_after_real_update') self.check() return result @contextlib.contextmanager def transaction(self, *args, **kwargs): with self.connection.transaction(*args, **kwargs) as transaction: yield transaction self.commits += 1 if self.lost_ack: self.lost_ack, self.injected = False, True raise OSError('synthetic_ack_loss_after_real_commit') def seed_history(connection, runtime, importer): """Seed real production tables; historical byte proofs need no old files.""" with connection.transaction(): connection.execute("""INSERT INTO public.target_scans( id, scan_event_id, scan_event_hash, source, target, normalized_target, scan_type, status, ended_at, findings_count, raw_result_storage, created_at) SELECT n, lpad(to_hex(n), 32, '0'), encode(sha256(convert_to('event-' || n, 'UTF8')), 'hex'), 'fixture', 'fixture:' || n, 'fixture:' || n, 'fixture', 'found', %s, 1, 'normalized_v2', %s FROM generate_series(1, 38024) AS g(n)""", (STAMP, STAMP)) connection.execute("""INSERT INTO public.findings( id, target_scan_id, source, target, detector_name, detector_type, verified, raw_secret, redacted_secret, secret_hash, finding_uid, created_at) SELECT id, id, 'fixture', target, 'Fixture', 'fixture', 0, 'synthetic-only-' || id, 'fixture', encode(sha256(convert_to('synthetic-only-' || id, 'UTF8')), 'hex'), encode(sha256(convert_to('finding-' || id, 'UTF8')), 'hex'), %s FROM public.target_scans""", (STAMP,)) connection.execute("""INSERT INTO public.scan_result_compat( target_scan_id, schema_version, metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at) SELECT id, 2, '{}', encode(sha256(convert_to('{}', 'UTF8')), 'hex'), 2, 'exact', %s FROM public.target_scans""", (STAMP,)) connection.execute("""INSERT INTO public.finding_compat_payloads( finding_id, raw_value, extension_json, payload_sha256, payload_bytes, payload_omitted, created_at) SELECT id, raw_secret, '{}', encode(sha256(convert_to(raw_secret, 'UTF8')), 'hex'), octet_length(raw_secret), 0, %s FROM public.findings""", (STAMP,)) connection.execute("""INSERT INTO public.projection_jobs( id, job_kind, event_id, event_hash, target_scan_id, status, required_stream_mask, capacity_items, capacity_bytes, capacity_released, attempts, created_at, updated_at, completed_at) SELECT id, 'scan_event', scan_event_id, scan_event_hash, id, 'completed', 2, 1, 65536, 1, 1, %s, %s, %s FROM public.target_scans""", (STAMP, STAMP, STAMP)) connection.execute("""WITH positions AS ( SELECT id, event_id, event_hash, (id - 1) / 2716 AS generation, CASE WHEN id > 35308 THEN 97783145::bigint ELSE 134217728::bigint END AS bytes, (id - 1) %% 2716 AS ordinal FROM public.projection_jobs) INSERT INTO public.projection_appends(id, job_id, stream_name, event_id, event_hash, generation, byte_offset, byte_length, payload_sha256, record_count, state, prepared_at, appended_at) SELECT id, id, 'found_secrets', event_id, event_hash, generation, bytes * ordinal / 2716, bytes * (ordinal + 1) / 2716 - bytes * ordinal / 2716, encode(sha256(convert_to('historical-output-' || id, 'UTF8')), 'hex'), 1, 'appended', %s, %s FROM positions""", (STAMP, STAMP)) connection.execute("""INSERT INTO public.projection_rotations( id, stream_name, from_generation, to_generation, source_bytes, segment_relative_path, state, created_at, completed_at) SELECT n + 1, 'found_secrets', n, n + 1, 134217728, 'found_secrets.g' || lpad(n::text, 6, '0') || '.jsonl', 'completed', %s, %s FROM generate_series(0, 12) AS g(n)""", (STAMP, STAMP)) for i in range(18): provider = f'p{i:02d}' for suffix, filename in (('results', 'Results.jsonl'), ('status', 'Checked.txt')): if suffix == 'status' and i >= 13: continue name = f'keycheck:{provider}:{suffix}' connection.execute("""INSERT INTO public.projection_streams( stream_name, base_relative_path, current_generation, rotation_bytes, max_generations, created_at, updated_at) VALUES (%s, %s, 0, 33554432, 16, %s, %s)""", (name, f'{provider}/{provider}{filename}', STAMP, STAMP)) connection.execute("""INSERT INTO public.projection_cursors(stream_name, generation, committed_offset, updated_at) VALUES (%s, 0, %s, %s)""", (name, 1000 + i if suffix == 'status' else 0, STAMP)) connection.execute("UPDATE public.projection_streams SET current_generation = 13 WHERE stream_name = 'found_secrets'") connection.execute("""UPDATE public.projection_cursors SET generation = 13, committed_offset = 97783145, last_append_id = 38024, last_job_id = 38024, last_event_id = (SELECT event_id FROM public.projection_jobs WHERE id = 38024), last_event_hash = (SELECT event_hash FROM public.projection_jobs WHERE id = 38024) WHERE stream_name = 'found_secrets'""") for worker in ('result_ingester', 'jsonl_projector'): connection.execute("""INSERT INTO public.pipeline_leases(worker_name, generation, lease_token, supervisor_instance_id, owner_pid, owner_creation_time, owner_executable, state, acquired_at, heartbeat_at, lease_expires_at, updated_at) VALUES (%s, 7, 'synthetic-expired', 'synthetic-expired', 999999, 'synthetic', '/synthetic/expired', 'ready', %s, %s, %s, %s)""", (worker, STAMP, STAMP, STAMP, STAMP)) for table in ('target_scans', 'findings', 'projection_jobs', 'projection_appends', 'projection_rotations'): connection.execute('SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence(%s, %s), ' + '(SELECT max(id) FROM public.' + identifier(table) + '), true)', ('public.' + table, 'id')) status_files = [] for i in range(13): provider = f'p{i:02d}' directory = runtime.DATA / 'runtime-linux/keychecks' / provider directory.mkdir(mode=0o700) path = directory / f'{provider}Checked.txt' importer._write(runtime, path, b'synthetic status snapshot\n') status_files.append(path) return status_files def fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, system_identifier): # Required source labels are inert format metadata, never opened as paths. paths = list(status_files) for name in sorted(importer.REQUIRED_FILES): path = runtime.DATA / name if not os.path.lexists(path): path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) importer._write(runtime, path, b'') paths.append(path) files = [] for path in paths: with importer._input(path, runtime) as (handle, before): raw = handle.read(4096) require(len(raw) == before[4], 'status_fixture_bound') files.append({'path': path.relative_to(runtime.DATA).as_posix(), 'size': len(raw), 'sha256': digest(raw)}) # Historical bytes are intentionally not materialized; this is not a restore test. files.append({'path': 'runtime-linux/results/found_secrets.jsonl', 'size': OLD_OFFSET, 'sha256': digest(b'intentionally-unavailable-synthetic-output')}) value = {'format': 'truf-windows-snapshot-v1', 'source': {'root': r'D:\truf', 'postgres_data_dir': r'S:\postgres-data', 'supervisor_stopped': True, 'postgres_stopped': True}, 'database': {'version_num': connection.execute("SELECT current_setting('server_version_num')::int AS version").fetchone()['version'], 'system_identifier': '1' if system_identifier != '1' else '2', 'database_name': 'synthetic_source', 'user_name': 'synthetic_source', 'port': 15432, 'data_directory': r'S:\postgres-data', 'bytes': 6, 'sha256': digest(b'PGDMPx'), 'table_counts': {name: item['rows'] + int(name in ('projection_streams', 'projection_cursors')) for name, item in baseline['tables'].items()}, 'sequence_states': baseline['sequences'], 'sequence_count': len(baseline['sequences']['public'])}, 'archive': {'bytes': sum(item['size'] for item in files) + 10240, 'sha256': digest(b'synthetic archive identity')}, 'files': files} raw = importer._encoded(value) pin = digest(raw) importer._manifest(raw, pin) importer._write(runtime, runtime.DATA / 'config/windows-import-manifest.json', raw) recovery.APPROVED_MANIFEST_SHA256 = pin return pin def exercise_projector(connection, db, runtime, config, check): from jsonl_projector import JsonlProjector from migrate_runtime_safety import initialize_projection_cursors_from_existing_files, require_legacy_cutover_clear # This is a real offline cutover after recovery, not a mocked readiness gate. db.require_runtime_safety_schema() cursors = initialize_projection_cursors_from_existing_files(db, config) legacy = require_legacy_cutover_clear(db, config) migrations = db.conn.execute('SELECT version, code_sha256 FROM runtime_schema_migrations ORDER BY version').fetchall() db.conn.commit() db.record_final_cutover({'legacy': legacy, 'projection_cursors': cursors, 'schema_migrations': [dict(row) for row in migrations]}) db.require_final_cutover() check() event_id, event_hash, finding_uid = 'f' * 32, digest(b'future-event'), digest(b'future-finding') with connection.transaction(): scan_id = connection.execute("""INSERT INTO public.target_scans( scan_event_id, scan_event_hash, target, normalized_target, scan_type, status, ended_at, findings_count, raw_result_storage, created_at) VALUES (%s, %s, 'fixture:future', 'fixture:future', 'fixture', 'found', %s, 1, 'normalized_v2', %s) RETURNING id""", (event_id, event_hash, STAMP, STAMP)).fetchone()['id'] connection.execute("""INSERT INTO public.scan_result_compat(target_scan_id, schema_version, metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at) VALUES (%s, 2, '{}', %s, 2, 'exact', %s)""", (scan_id, digest(b'{}'), STAMP)) finding_id = connection.execute("""INSERT INTO public.findings(target_scan_id, detector_name, detector_type, verified, raw_secret, redacted_secret, finding_uid, created_at) VALUES (%s, 'Fixture', 'fixture', 0, 'synthetic-future', 'fixture', %s, %s) RETURNING id""", (scan_id, finding_uid, STAMP)).fetchone()['id'] connection.execute("""INSERT INTO public.finding_compat_payloads(finding_id, raw_value, extension_json, payload_sha256, payload_bytes, payload_omitted, created_at) VALUES (%s, 'synthetic-future', '{}', %s, 16, 0, %s)""", (finding_id, digest(b'synthetic-future'), STAMP)) job_id = connection.execute("""INSERT INTO public.projection_jobs(job_kind, event_id, event_hash, target_scan_id, status, required_stream_mask, capacity_items, capacity_bytes, created_at, updated_at) VALUES ('scan_event', %s, %s, %s, 'pending', 2, 1, 65536, %s, %s) RETURNING id""", (event_id, event_hash, scan_id, STAMP, STAMP)).fetchone()['id'] connection.execute("""UPDATE public.pipeline_capacity SET projection_items = projection_items + 1, projection_bytes = projection_bytes + 65536 WHERE id = 1""") worker = JsonlProjector(db, str(runtime.DATA / 'runtime-linux/results'), 'projection-loss-e2e', keycheck_dir=str(runtime.DATA / 'runtime-linux/keychecks')) try: worker.start() require(worker.process_one() is True, 'projector_processed') require(worker.process_one() is False, 'projector_idle') row = connection.execute("""SELECT a.generation, a.byte_offset, a.byte_length, a.payload_sha256, a.state, j.status, j.capacity_released FROM public.projection_appends a JOIN public.projection_jobs j ON j.id = a.job_id WHERE a.job_id = %s AND a.stream_name = 'found_secrets'""", (job_id,)).fetchone() path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl' runtime.private_path(path) require(path.stat().st_size < 65536, 'bounded_new_output') raw = path.read_bytes() require(row and row['generation'] == 14 and row['byte_offset'] == 0 and row['byte_length'] == len(raw) and row['payload_sha256'] == digest(raw) and row['state'] == 'appended' and row['status'] == 'completed' and row['capacity_released'] == 1, 'projector_fenced_append') require(json.loads(raw)['finding_uid'] == finding_uid, 'projector_real_payload') cursor = metadata(connection)['cursor'] require(cursor['generation'] == 14 and cursor['committed_offset'] == len(raw) and cursor['last_job_id'] == job_id, 'projector_cursor') capacity = connection.execute('SELECT projection_items, projection_bytes FROM public.pipeline_capacity WHERE id = 1').fetchone() require(capacity == {'projection_items': 0, 'projection_bytes': 0}, 'projector_capacity') emit(stage='projector', passed=True, generation=14, records=1, bytes=len(raw)) finally: worker.stop() def run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check): import container_import as importer import container_projection_recovery as recovery from runtime_security import PrivateFileLock from scanner_db import ScannerDB, migrate_runtime_safety_schema schema_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False) resources.append(schema_db) require(schema_db.enabled and schema_db.conn.is_postgres, 'real_schema_connection') try: migrate_runtime_safety_schema(schema_db, initialize_base=True) schema_db.require_runtime_safety_schema() finally: schema_db.close() emit(stage='schema', passed=True) check() status_files = seed_history(connection, runtime, importer) emit(stage='seed', passed=True) baseline = proof(connection, check) emit(stage='proof', passed=True, tables=len(baseline['tables']), sequences=len(baseline['sequences']['public'])) before = metadata(connection) require(before['stream']['current_generation'] == before['cursor']['generation'] == 13 and before['cursor']['committed_offset'] == OLD_OFFSET, 'reviewed_before') require(baseline['tables']['projection_appends']['rows'] == APPENDS and baseline['tables']['projection_rotations']['rows'] == 13 and baseline['tables']['projection_append_audit']['rows'] == 0, 'reviewed_history') original_pin = recovery.APPROVED_MANIFEST_SHA256 pin = fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, identity['system_identifier']) journal_path = runtime.DATA / 'config' / recovery.JOURNAL_NAME status_before = {path: (path.stat().st_ino, path.read_bytes()) for path in status_files} emit(stage='fixture', passed=True, streams=34, appends=APPENDS, rotations=13, tables=len(baseline['tables']), sequences=len(baseline['sequences']['public'])) def recover(observed): return recovery.recover_found_secrets_projection(runtime, observed, system_identifier=identity['system_identifier'], manifest_sha256=pin, initialize_lock=initialize_lock, authority_lock=authority_lock) def refused(observed): try: recover(observed) except recovery.ProjectionRecoveryError as error: if not observed.injected and (observed.after_update or observed.lost_ack): emit(stage='fault_not_reached', passed=False, **safe_error(error)) return raise AssertionError('expected_recovery_refusal') def alone(): while True: check() connection.execute('SELECT pg_catalog.pg_stat_clear_snapshot()') count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_stat_activity WHERE backend_type = 'client backend' AND pid <> pg_backend_pid()""").fetchone()['count'] if count == 0: return time.sleep(0.05) try: alone() contender = connect() resources.append(contender) try: observed = ObservedConnection(connection, runtime, recovery, check) refused(observed) require(observed.updates == 0, 'other_client_no_updates') with contender.transaction(): contender.execute('LOCK TABLE public.projection_streams IN ROW EXCLUSIVE MODE') observed = ObservedConnection(connection, runtime, recovery, check) refused(observed) require(observed.updates == 0, 'writer_contention_no_updates') finally: contender.close() alone() with PrivateFileLock(str(runtime.DATA / 'runtime-linux/results/.jsonl-projector.lock')): observed = ObservedConnection(connection, runtime, recovery, check) refused(observed) require(observed.updates == 0, 'projector_file_lock_no_updates') require(metadata(connection) == before and proof(connection, check) == baseline and not os.path.lexists(journal_path), 'contention_unchanged') emit(stage='contention', passed=True, cases=3, updates=0) observed = ObservedConnection(connection, runtime, recovery, check, after_update=True) refused(observed) require(observed.injected and observed.updates == 1 and observed.commits == 0, 'rollback_fault_window') require(int(connection.info.transaction_status) == 0 and metadata(connection) == before and proof(connection, check) == baseline, 'both_rows_rolled_back') journal = journal_snapshot(runtime, recovery) require(journal[2]['record']['before'] == before, 'journal_before') emit(stage='rollback', passed=True, updates=1, commits=0, journal_retained=True) observed = ObservedConnection(connection, runtime, recovery, check, lost_ack=True) refused(observed) require(observed.injected and observed.updates == 2 and observed.commits == 1, 'real_commit_before_ack_loss') after = metadata(connection) require(after == journal[2]['record']['after'] and proof(connection, check) == baseline and journal_snapshot(runtime, recovery) == journal, 'committed_despite_ack_loss') emit(stage='lost_ack', passed=True, updates=2, commits=1, history_unchanged=True) observed = ObservedConnection(connection, runtime, recovery, check) result = recover(observed) require(result['status'] == 'already-committed' and observed.updates == 0 and result['journal_sha256'] == digest(journal[0]) and metadata(connection) == after and proof(connection, check) == baseline and journal_snapshot(runtime, recovery) == journal, 'idempotent_readonly_retry') require({path: (path.stat().st_ino, path.read_bytes()) for path in status_files} == status_before, 'other_output_unchanged') emit(stage='retry', passed=True, updates=0, journal_unchanged=True, history_unchanged=True, sequences_unchanged=True) writer_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False) resources.append(writer_db) require(writer_db.enabled and writer_db.conn.is_postgres, 'real_writer_connection') try: exercise_projector(connection, writer_db, runtime, config, check) finally: writer_db.close() alone() advanced, advanced_proof = metadata(connection), proof(connection, check) future_path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl' future_bytes, future_inode = future_path.read_bytes(), future_path.stat().st_ino observed = ObservedConnection(connection, runtime, recovery, check) refused(observed) require(observed.updates == 0 and advanced['cursor']['committed_offset'] > 0 and metadata(connection) == advanced and proof(connection, check) == advanced_proof and (future_path.read_bytes(), future_path.stat().st_ino) == (future_bytes, future_inode) and journal_snapshot(runtime, recovery) == journal, 'future_output_not_rewound') emit(stage='future_output', passed=True, updates=0, advanced_cursor_retained=True, journal_unchanged=True) finally: recovery.APPROVED_MANIFEST_SHA256 = original_pin def main(): global OUTPUT # Native pg_ctl also inherits fd 1/2: Python stream redirection alone is insufficient. OUTPUT = os.fdopen(os.dup(1), 'w', encoding='utf-8', buffering=1) sink = os.open(os.devnull, os.O_WRONLY) try: os.dup2(sink, 1) os.dup2(sink, 2) finally: os.close(sink) parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument('--budget-seconds', type=int, default=300) args = parser.parse_args() require(30 <= args.budget_seconds <= 3600, 'budget') started = time.monotonic() deadline = started + args.budget_seconds require(sys.platform == 'linux' and os.geteuid() == os.getuid() == 10001 and os.getegid() == os.getgid() == 10001, 'test_identity') require(Path(__file__) == IMAGE_PATH and sys.flags.isolated and sys.flags.no_site and sys.flags.dont_write_bytecode, 'test_image') require({name for _, name in socket.if_nameindex()} == {'lo'}, 'offline_test') raw_mounts = Path('/proc/self/mountinfo').read_bytes() require(len(raw_mounts) <= 1024 * 1024, 'mount_bound') mounts = [line.split() for line in raw_mounts.decode('utf-8', errors='strict').splitlines()] data = [row for row in mounts if len(row) > 6 and (row[4] == '/data' or row[4].startswith('/data/'))] require(len(data) == 1 and data[0][4] == '/data' and '-' in data[0] and re.fullmatch(r'/var/lib/docker/volumes/truf-projection-loss-test-[a-f0-9]{32}/_data', data[0][3]) and data[0][data[0].index('-') + 1] == 'ext4', 'fresh_test_volume') runtime = SimpleNamespace(**runpy.run_path('/opt/truf/app/container_runtime.py')) runtime.require_container() runtime.private_path(IMAGE_PATH.parent, directory=True) runtime.private_path(IMAGE_PATH) runtime._shutdown_requested = False for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(sig, lambda *_: setattr(runtime, '_shutdown_requested', True)) def fresh(): runtime.private_path(runtime.DATA / 'postgres-linux', directory=True) require(not any((runtime.DATA / 'postgres-linux').iterdir()), 'empty_pgdata') require(not any((runtime.DATA / 'runtime-linux/results').iterdir()), 'empty_results') for name in ('runtime-linux/postgres/cluster_identity.json', 'initialized.json', 'config/windows-import-manifest.json', 'config/found-secrets-loss-g13-g14.prepared.json'): require(not os.path.lexists(runtime.DATA / name), 'fresh_fixture') fresh() config_path = runtime.DEFAULT_CONFIG config = runtime.prepare_environment(config_path) os.environ.update(TRUF_DB_STATEMENT_TIMEOUT_MS='120000', TRUF_DB_LOCK_TIMEOUT_MS='5000', TRUF_DB_IDLE_TRANSACTION_TIMEOUT_MS='300000') import postgres_runtime as pg import psycopg from psycopg.rows import dict_row from runtime_security import ClusterAuthorityLock, PrivateFileLock def connect(): return psycopg.connect(os.environ['SCANNER_DB_URL'], autocommit=True, row_factory=dict_row, connect_timeout=5, application_name='truf-projection-loss-e2e', tcp_user_timeout=30000, options='-c search_path=public -c statement_timeout=120000 -c lock_timeout=5000 ' '-c idle_in_transaction_session_timeout=300000 -c row_security=off ' '-c log_min_error_statement=panic -c log_min_messages=panic ' '-c log_statement=none -c log_min_duration_statement=-1') resources = [] check = lambda: checkpoint(runtime, deadline) with PrivateFileLock(str(runtime.INITIALIZE_LOCK)) as initialize_lock: fresh() authority_lock = ClusterAuthorityLock(config, endpoint_dsn=os.environ['SCANNER_DB_URL']) code = initialize_empty(runtime, config_path) while True: try: authority_lock.acquire() break except BaseException as error: emit(stage='authority', failed_hold=True, **safe_error(error)) hold_pause() try: backend = None while backend is None: try: backend = pg.PostgresBackend(config, stop_timeout_sec=60) except BaseException as error: emit(stage='backend', failed_hold=True, **safe_error(error)) hold_pause() try: require(code == 0, 'initialize_empty_exit') check() require(backend.probe().kind == pg.ProbeKind.STOPPED, 'initial_stopped_probe') identity = pg.verify_cluster_identity(config) require(identity['pg_major'] == 16 and identity['data_directory'] == '/data/postgres-linux', 'bound_fixture') require(backend.start().accepted is True, 'direct_backend_start') while True: check() probe = backend.probe() if probe.kind == pg.ProbeKind.READY: break require(probe.kind == pg.ProbeKind.RECOVERING, 'authenticated_start') time.sleep(0.1) emit(stage='start', ready=True) connection = connect() resources.append(connection) count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relkind IN ('r','p','f')""").fetchone()['count'] require(count == 0, 'virgin_schema') run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check) require(not os.path.lexists(runtime.INITIALIZED), 'no_application_marker') finally: try: for resource in reversed(resources): try: resource.close() except BaseException as error: emit(stage='client_close', failed_hold=True, **safe_error(error)) finally: stop_confirmed(backend, pg) finally: authority_lock.release() emit(stage='finished', passed=True, stopped=STOPPED, application_initialized=False, elapsed_ms=round((time.monotonic() - started) * 1000)) return 0 if __name__ == '__main__': try: result = main() except BaseException as error: emit(stage='finished', passed=False, stopped=STOPPED, **safe_error(error)) result = 1 raise SystemExit(result)