sashatrask
767 lines
38 KiB
Python
767 lines
38 KiB
Python
"""Isolated host tests: python -I -S -B tests/test_container_projection_recovery.py.
|
|
|
|
Only clone code is compiled; DB, container checks and security locks are mocked.
|
|
All filesystem fixtures are under tempfile.tempdir. No real app/PG is started.
|
|
"""
|
|
|
|
import copy
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import traceback
|
|
from types import ModuleType, SimpleNamespace
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
APP = Path(__file__).resolve().parents[1] / 'app'
|
|
PRIVATE = 'private-fixture-credential-never-a-diagnostic'
|
|
|
|
|
|
def load(name, path):
|
|
module = ModuleType(name)
|
|
module.__file__ = str(path)
|
|
exec(compile(path.read_text(encoding='utf-8'), str(path), 'exec'), module.__dict__)
|
|
return module
|
|
|
|
|
|
def encoded(value):
|
|
return (json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(',', ':')) + '\n').encode('ascii')
|
|
|
|
|
|
def sha(value):
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
class Result:
|
|
def __init__(self, rows=(), rowcount=0):
|
|
self.rows, self.rowcount = rows, rowcount
|
|
|
|
def fetchone(self):
|
|
return copy.deepcopy(self.rows[0]) if self.rows else None
|
|
|
|
def fetchall(self):
|
|
return copy.deepcopy(self.rows)
|
|
|
|
|
|
class FakeConnection:
|
|
def __init__(self, module, trace):
|
|
self.m, self.trace = module, trace
|
|
self.closed, self.broken, self.autocommit = False, False, True
|
|
self.info = SimpleNamespace(transaction_status=0)
|
|
self.statements = []
|
|
self.fail_update = self.fail_commit = None
|
|
self.rowcount = 1
|
|
self.advisory_available, self.tables_available = True, True
|
|
self.other_client_call, self.online_calls = 0, 0
|
|
self.corrupt_history = False
|
|
self.after_update = None
|
|
self.rows = []
|
|
specs = [(name, path) for name, path in (
|
|
('scan_results', 'scan_results.jsonl'), ('found_secrets', 'found_secrets.jsonl'),
|
|
('scan_errors', 'scan_errors.log'))]
|
|
specs += [(f'keycheck:p{i}:results', f'p{i}/p{i}Results.jsonl') for i in range(18)]
|
|
specs += [(f'keycheck:p{i}:status', f'p{i}/p{i}Checked.txt') for i in range(13)]
|
|
for name, path in specs:
|
|
found = name == 'found_secrets'
|
|
stream = {'stream_name': name, 'base_relative_path': path,
|
|
'current_generation': 13 if found else 0, 'rotation_bytes': 134217728,
|
|
'max_generations': 16, 'created_at': '2026-09-01T00:00:00+00:00',
|
|
'updated_at': '2026-09-15T00:00:00+00:00'}
|
|
cursor = {'stream_name': name, 'generation': stream['current_generation'],
|
|
'committed_offset': 97783145 if found else 0,
|
|
'last_append_id': 38024 if found else None, 'last_job_id': 38024 if found else None,
|
|
'last_event_id': 'a' * 32 if found else None, 'last_event_hash': 'b' * 64 if found else None,
|
|
'updated_at': '2026-09-15T00:00:01+00:00'}
|
|
self.rows.append({'stream_name': name, 'cursor_stream_name': name, 'stream': stream, 'cursor': cursor})
|
|
self.last = {'id': 38024, 'job_id': 38024, 'stream_name': 'found_secrets', 'generation': 13,
|
|
'byte_offset': 97783000, 'byte_length': 145, 'event_id': 'a' * 32,
|
|
'event_hash': 'b' * 64, 'state': 'appended'}
|
|
self.gates = {'appends': 38024, 'append_highwater': 13, 'bad_appends': 0,
|
|
'rotations': 13, 'rotation_highwater': 13, 'bad_rotations': 0,
|
|
'audits': 0, 'pending': 0, 'reservations': 0, 'queue_leases': 0,
|
|
'blob_leases': 0, 'triggers': 0, 'rules': 0}
|
|
self.counts = {'projection_streams': 34, 'projection_cursors': 34, 'projection_appends': 38024,
|
|
'projection_append_audit': 0, 'projection_rotations': 13, 'projection_jobs': 38024,
|
|
'pipeline_capacity': 1, 'pipeline_leases': 2, 'findings': 1, 'target_scans': 1,
|
|
'target_queue': 1, 'result_reservations': 1, 'result_bundles': 1,
|
|
'pipeline_quarantine': 0, 'docker_content_blobs': 1, 'other_history': 1}
|
|
self.history = {name: {'fixture': PRIVATE, 'state': name} for name in self.counts}
|
|
self.sequence = {'last_value': 38024, 'is_called': True}
|
|
self.online = {'database': 'truf', 'user_name': 'truf', 'data_directory': str(module.DATA / 'postgres-linux'),
|
|
'port': 5432, 'version_num': 160014, 'system_identifier': '2222222222',
|
|
'in_recovery': False, 'superuser': True, 'other_clients': 0,
|
|
'schema_name': 'public', 'search_path': 'public', 'read_only': 'off', 'public_create': False,
|
|
'fsync': 'on', 'full_page_writes': 'on'}
|
|
|
|
@property
|
|
def found(self):
|
|
return next(row for row in self.rows if row['stream_name'] == 'found_secrets')
|
|
|
|
@property
|
|
def updates(self):
|
|
return [(sql, params) for sql, params in self.statements if sql.startswith('UPDATE')]
|
|
|
|
def transaction(self):
|
|
db = self
|
|
|
|
class Transaction:
|
|
def __enter__(self):
|
|
self.before = copy.deepcopy((db.rows, db.history))
|
|
db.info.transaction_status = 2
|
|
db.trace.append('begin')
|
|
|
|
def __exit__(self, kind, value, tb):
|
|
db.info.transaction_status = 0
|
|
if kind is not None or db.fail_commit == 'before':
|
|
db.rows, db.history = self.before
|
|
db.trace.append('rollback')
|
|
if kind is None:
|
|
raise RuntimeError(PRIVATE)
|
|
else:
|
|
db.trace.append('commit')
|
|
if db.fail_commit == 'after':
|
|
raise RuntimeError(PRIVATE)
|
|
|
|
return Transaction()
|
|
|
|
def cursor(self, name):
|
|
db = self
|
|
|
|
class Cursor:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def execute(self, sql):
|
|
db.statements.append((' '.join(sql.split()), ()))
|
|
self.table = re.search(r'FROM public\."([a-z_]+)" AS t', sql)[1]
|
|
if not sql.endswith(' ORDER BY digest') or 'COLLATE "C" AS digest' not in sql:
|
|
raise AssertionError('unbounded or unordered fixture query')
|
|
|
|
def __iter__(self):
|
|
table = self.table
|
|
if table in ('projection_streams', 'projection_cursors'):
|
|
key = 'stream' if table == 'projection_streams' else 'cursor'
|
|
hashes = sorted(sha(encoded(row[key])) for row in db.rows if row['stream_name'] != 'found_secrets')
|
|
else:
|
|
hashes = itertools.repeat(sha(encoded(db.history[table])), db.counts[table])
|
|
return ({'digest': value} for value in hashes)
|
|
|
|
return Cursor()
|
|
|
|
def execute(self, sql, params=()):
|
|
sql = ' '.join(sql.split())
|
|
self.statements.append((sql, params))
|
|
if sql.startswith('SET ') or sql == 'SELECT pg_catalog.pg_stat_clear_snapshot()':
|
|
return Result()
|
|
if 'AS other_clients' in sql:
|
|
self.online_calls += 1
|
|
row = dict(self.online)
|
|
if self.online_calls == self.other_client_call:
|
|
row['other_clients'] = 1
|
|
return Result([row])
|
|
if 'pg_try_advisory_xact_lock' in sql:
|
|
return Result([{'locked': self.advisory_available}])
|
|
if 'c.relkind AS kind' in sql:
|
|
return Result([{'name': name, 'kind': 'r'} for name in sorted(self.counts)])
|
|
if sql.startswith('LOCK TABLE '):
|
|
if not self.tables_available:
|
|
raise RuntimeError(PRIVATE)
|
|
if not sql.endswith(' IN EXCLUSIVE MODE NOWAIT'):
|
|
raise AssertionError('missing writer exclusion')
|
|
return Result()
|
|
if 'AS bad_appends' in sql:
|
|
return Result([self.gates])
|
|
if 'pg_catalog.row_to_json(s) AS stream' in sql:
|
|
return Result(self.rows)
|
|
if "c.relkind = 'S'" in sql:
|
|
return Result([{'name': 'fixture_sequence'}])
|
|
if sql.startswith('SELECT last_value, is_called'):
|
|
return Result([self.sequence])
|
|
if 'FROM public.projection_appends WHERE id = %s' in sql:
|
|
return Result([self.last] if params[0] == self.last['id'] else [])
|
|
if sql.startswith('UPDATE public.projection_'):
|
|
key = 'stream' if 'projection_streams AS s' in sql else 'cursor'
|
|
self.trace.append('update-' + key)
|
|
if self.fail_update == key:
|
|
raise RuntimeError(PRIVATE)
|
|
if json.loads(params[1]) != self.found[key]:
|
|
return Result(rowcount=0)
|
|
if self.rowcount != 1:
|
|
return Result(rowcount=self.rowcount)
|
|
if key == 'stream':
|
|
self.found[key].update(current_generation=14, updated_at=params[0])
|
|
else:
|
|
self.found[key].update(generation=14, committed_offset=0, last_append_id=None, updated_at=params[0])
|
|
if self.corrupt_history:
|
|
self.history['findings']['state'] = 'unintended mutation'
|
|
if self.after_update is not None:
|
|
self.after_update()
|
|
return Result(rowcount=1)
|
|
raise AssertionError('unexpected fixture SQL')
|
|
|
|
|
|
class ProjectionRecoveryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory(prefix='truf-projection-recovery-test-')
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name)
|
|
self.m = load('projection_recovery_under_test', APP / 'container_projection_recovery.py')
|
|
self.importer = load('container_import_fixture', APP / 'container_import.py')
|
|
self.m.DATA, self.m.RUN = self.root / 'data', self.root / 'run'
|
|
self.trace = []
|
|
self.importer.os = SimpleNamespace(**vars(os))
|
|
for key in ('O_NOFOLLOW', 'O_DIRECTORY', 'O_NONBLOCK'):
|
|
setattr(self.importer.os, key, getattr(os, key, 0))
|
|
if os.name == 'nt':
|
|
fingerprint = self.importer._fingerprint
|
|
self.importer._fingerprint = lambda info: fingerprint(info)[:-1] + (0,)
|
|
self.importer._fsync_dir = lambda path: self.trace.append('directory-fsync')
|
|
self.m.os = SimpleNamespace(**vars(os))
|
|
self.m.os.fsync = lambda fd: self.trace.append('journal-fsync')
|
|
self.importer.subprocess = SimpleNamespace(Popen=mock.Mock(side_effect=AssertionError('real processes forbidden')))
|
|
for path in (self.m.DATA, self.m.DATA / 'config', self.m.DATA / 'runtime-linux/results',
|
|
self.m.DATA / 'runtime-linux/postgres', self.m.DATA / 'runtime-linux/logs',
|
|
self.m.RUN / 'control', self.m.RUN / 'authority'):
|
|
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self.denied = set()
|
|
|
|
def private(path, *, directory=False):
|
|
path = Path(path)
|
|
if not path.is_relative_to(self.root) or path in self.denied:
|
|
raise RuntimeError(PRIVATE)
|
|
if path.is_symlink() or not (path.is_dir() if directory else path.is_file()):
|
|
raise RuntimeError(PRIVATE)
|
|
return path
|
|
|
|
self.runtime = SimpleNamespace(DATA=self.m.DATA, RUN=self.m.RUN,
|
|
INITIALIZED=self.m.DATA / 'initialized.json',
|
|
INITIALIZE_LOCK=self.m.DATA / 'initialize.lock',
|
|
private_path=private, require_container=mock.Mock(), _shutdown_requested=False)
|
|
outer = self
|
|
|
|
class FileLock:
|
|
fail_projector = False
|
|
|
|
def __init__(self, path):
|
|
self.path = os.path.normcase(str(path))
|
|
self.acquired = False
|
|
self.releases = 0
|
|
|
|
def __enter__(self):
|
|
if self.fail_projector:
|
|
raise RuntimeError(PRIVATE)
|
|
Path(self.path).touch(exist_ok=True)
|
|
self.acquired = True
|
|
outer.trace.append('projector-lock')
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.acquired = False
|
|
self.releases += 1
|
|
outer.trace.append('projector-unlock')
|
|
|
|
class Authority:
|
|
def __init__(self):
|
|
self.acquired, self.releases = True, 0
|
|
self.data_directory = str(outer.m.DATA / 'postgres-linux')
|
|
self.endpoint_identity = encoded({'database': 'truf', 'host': '127.0.0.1', 'port': 5432,
|
|
'schema': 'public'}).decode('ascii').strip()
|
|
self.path = str(outer.m.DATA / 'runtime-linux/postgres' /
|
|
('.cluster-authority-' + sha(self.data_directory.encode('utf-8')) + '.lock'))
|
|
self.endpoint_path = str(outer.m.RUN / 'authority' /
|
|
('endpoint-' + sha(self.endpoint_identity.encode('ascii')) + '.lock'))
|
|
|
|
self.security = SimpleNamespace(PrivateFileLock=FileLock, ClusterAuthorityLock=Authority)
|
|
self.initialize_lock = FileLock(self.runtime.INITIALIZE_LOCK)
|
|
Path(self.initialize_lock.path).touch()
|
|
self.initialize_lock.acquired = True
|
|
self.authority_lock = Authority()
|
|
modules = mock.patch.dict(sys.modules, {'container_import': self.importer, 'runtime_security': self.security})
|
|
modules.start()
|
|
self.addCleanup(modules.stop)
|
|
self.db = FakeConnection(self.m, self.trace)
|
|
self.identity_path = self.m.DATA / 'runtime-linux/postgres/cluster_identity.json'
|
|
self.identity = {'pg_major': 16, 'system_identifier': '2222222222', 'data_directory': str(self.m.DATA / 'postgres-linux'),
|
|
'database': 'truf', 'user': 'truf', 'port': 5432}
|
|
self.identity_path.write_bytes(encoded(self.identity))
|
|
manifest = {'format': 'truf-windows-snapshot-v1',
|
|
'source': {'root': r'D:\truf', 'postgres_data_dir': r'S:\postgres-data',
|
|
'supervisor_stopped': True, 'postgres_stopped': True},
|
|
'database': {'version_num': 160014, 'system_identifier': '1111111111',
|
|
'database_name': 'source_fixture', 'user_name': 'source_fixture', 'port': 15432,
|
|
'data_directory': r'S:\postgres-data', 'bytes': 6, 'sha256': '1' * 64,
|
|
'table_counts': self.db.counts,
|
|
'sequence_states': {'public': {'fixture_sequence': self.db.sequence}}, 'sequence_count': 1},
|
|
'archive': {'bytes': 10240, 'sha256': '2' * 64},
|
|
'files': [{'path': path, 'size': 0, 'sha256': sha(b'')} for path in sorted(self.importer.REQUIRED_FILES)]}
|
|
self.manifest_path = self.m.DATA / 'config/windows-import-manifest.json'
|
|
raw = encoded(manifest)
|
|
self.manifest_path.write_bytes(raw)
|
|
# Test-only in-memory fixture pin; the production module keeps its reviewed pin.
|
|
self.m.APPROVED_MANIFEST_SHA256 = sha(raw)
|
|
self.journal = self.m.DATA / 'config' / self.m.JOURNAL_NAME
|
|
|
|
def call(self, **overrides):
|
|
args = {'system_identifier': '2222222222', 'manifest_sha256': self.m.APPROVED_MANIFEST_SHA256,
|
|
'initialize_lock': self.initialize_lock, 'authority_lock': self.authority_lock}
|
|
args.update(overrides)
|
|
return self.m.recover_found_secrets_projection(self.runtime, self.db, **args)
|
|
|
|
def assert_held(self):
|
|
self.assertTrue(self.initialize_lock.acquired)
|
|
self.assertTrue(self.authority_lock.acquired)
|
|
self.assertEqual((self.initialize_lock.releases, self.authority_lock.releases), (0, 0))
|
|
self.assertFalse(self.db.closed)
|
|
self.importer.subprocess.Popen.assert_not_called()
|
|
|
|
def prepare_without_commit(self):
|
|
self.db.fail_update = 'cursor'
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.db.fail_update = None
|
|
self.db.statements.clear()
|
|
self.assertTrue(self.journal.is_file())
|
|
|
|
def test_success_changes_only_two_metadata_rows_and_journals_before_commit(self):
|
|
before_rows, before_history = copy.deepcopy(self.db.rows), copy.deepcopy(self.db.history)
|
|
last = copy.deepcopy(self.db.last)
|
|
result = self.call()
|
|
self.assertEqual(result['status'], 'committed')
|
|
self.assertEqual(result['journal_sha256'], sha(self.journal.read_bytes()))
|
|
saved = json.loads(self.journal.read_bytes())
|
|
record = saved['record']
|
|
self.assertEqual(saved['sha256'], sha(self.m._encoded(record)))
|
|
self.assertEqual(record['state'], 'PREPARED')
|
|
self.assertEqual(record['binding']['system_identifier'], '2222222222')
|
|
self.assertEqual(record['binding']['manifest_sha256'], self.m.APPROVED_MANIFEST_SHA256)
|
|
self.assertIn('No rotation or rename occurred', record['loss'])
|
|
self.assertEqual(record['before'], {key: before_rows[1][key] for key in ('stream', 'cursor')})
|
|
self.assertEqual(record['after'], {key: self.db.found[key] for key in ('stream', 'cursor')})
|
|
self.assertEqual(self.db.rows[:1] + self.db.rows[2:], before_rows[:1] + before_rows[2:])
|
|
self.assertEqual(self.db.history, before_history)
|
|
self.assertEqual(self.db.last, last)
|
|
for key in ('last_job_id', 'last_event_id', 'last_event_hash'):
|
|
self.assertEqual(self.db.found['cursor'][key], before_rows[1]['cursor'][key])
|
|
self.assertEqual(len(self.db.updates), 2)
|
|
for setting in ("SET LOCAL statement_timeout = '10800s'", "SET LOCAL temp_file_limit = '4GB'",
|
|
"SET LOCAL work_mem = '128MB'", 'SET LOCAL max_parallel_workers_per_gather = 0'):
|
|
self.assertIn((setting, ()), self.db.statements)
|
|
self.assertTrue(all('pg_catalog.to_jsonb' in sql for sql, _ in self.db.updates))
|
|
self.assertFalse(any(sql.startswith(('INSERT', 'DELETE', 'ALTER', 'CREATE', 'DROP', 'TRUNCATE'))
|
|
for sql, _ in self.db.statements))
|
|
self.assertLess(self.trace.index('directory-fsync'), self.trace.index('update-stream'))
|
|
self.assertLess(self.trace.index('journal-fsync'), self.trace.index('update-stream'))
|
|
self.assertLess(self.trace.index('update-cursor'), self.trace.index('commit'))
|
|
self.assertFalse(self.runtime.INITIALIZED.exists())
|
|
self.assertFalse((self.m.DATA / 'runtime-linux/results/found_secrets.jsonl').exists())
|
|
self.assertNotIn(PRIVATE, self.journal.read_text(encoding='ascii'))
|
|
self.assert_held()
|
|
|
|
def test_exact_after_retry_has_no_sql_rewrites_or_journal_replacement(self):
|
|
first = self.call()
|
|
before, metadata = self.journal.read_bytes(), self.journal.stat()
|
|
rows, history = copy.deepcopy(self.db.rows), copy.deepcopy(self.db.history)
|
|
self.db.statements.clear()
|
|
second = self.call()
|
|
self.assertEqual(second['status'], 'already-committed')
|
|
self.assertEqual(first['journal_sha256'], second['journal_sha256'])
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertEqual((self.db.rows, self.db.history), (rows, history))
|
|
self.assertEqual(self.journal.read_bytes(), before)
|
|
self.assertEqual(self.journal.stat().st_mtime_ns, metadata.st_mtime_ns)
|
|
self.assert_held()
|
|
|
|
def test_failure_after_journal_rolls_back_then_reuses_exact_intent(self):
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.prepare_without_commit()
|
|
self.assertEqual(self.db.rows, before)
|
|
raw = self.journal.read_bytes()
|
|
self.assertEqual(self.call()['status'], 'committed')
|
|
self.assertEqual(self.journal.read_bytes(), raw)
|
|
self.assert_held()
|
|
|
|
def test_commit_failures_are_reconciled_from_journal_without_guessing(self):
|
|
for when in ('before', 'after'):
|
|
with self.subTest(commit_failure=when):
|
|
self.setUp_commit_case(when)
|
|
|
|
def setUp_commit_case(self, when):
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.db.fail_commit = when
|
|
with self.assertRaises(self.m.ProjectionRecoveryError) as caught:
|
|
self.call()
|
|
self.assertNotIn(PRIVATE, str(caught.exception))
|
|
self.assertTrue(self.journal.exists())
|
|
self.assert_held()
|
|
self.db.fail_commit = None
|
|
self.db.statements.clear()
|
|
result = self.call()
|
|
self.assertEqual(result['status'], 'committed' if when == 'before' else 'already-committed')
|
|
self.assertEqual(len(self.db.updates), 2 if when == 'before' else 0)
|
|
# Reset only this synthetic fixture for the second independent crash window.
|
|
self.db.rows = before
|
|
self.db.rows[1]['stream']['current_generation'] = 13
|
|
self.db.rows[1]['cursor'].update(generation=13, committed_offset=97783145, last_append_id=38024)
|
|
self.journal.unlink()
|
|
|
|
def test_partial_journal_is_never_repaired_overwritten_or_committed(self):
|
|
def partial(runtime, path, payload):
|
|
with path.open('xb') as handle:
|
|
handle.write(payload[:50])
|
|
raise OSError(PRIVATE)
|
|
|
|
with mock.patch.object(self.importer, '_write', side_effect=partial), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
raw = self.journal.read_bytes()
|
|
self.assertEqual(self.db.updates, [])
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.journal.read_bytes(), raw)
|
|
self.assertEqual(self.db.found['cursor']['committed_offset'], 97783145)
|
|
|
|
def test_complete_but_unsynced_journal_requires_confirmed_fsync_on_retry(self):
|
|
with mock.patch.object(self.importer, '_fsync_dir', side_effect=OSError(PRIVATE)), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertTrue(self.journal.exists())
|
|
with mock.patch.object(self.m.os, 'fsync', side_effect=OSError(PRIVATE)), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertEqual(self.call()['status'], 'committed')
|
|
|
|
def test_malformed_or_mismatched_journals_are_refused(self):
|
|
self.prepare_without_commit()
|
|
original = self.journal.read_bytes()
|
|
for case in ('truncated', 'whitespace', 'duplicate', 'checksum', 'binding', 'proof', 'after', 'before', 'state'):
|
|
value = json.loads(original)
|
|
if case == 'truncated':
|
|
raw = original[:60]
|
|
elif case == 'whitespace':
|
|
raw = original + b' '
|
|
elif case == 'duplicate':
|
|
raw = b'{"record":{},' + original[1:]
|
|
else:
|
|
record = value['record']
|
|
if case == 'binding':
|
|
record['binding']['system_identifier'] = '3333333333'
|
|
elif case == 'proof':
|
|
record['preserved']['tables']['findings']['sha256'] = '0' * 64
|
|
elif case == 'after':
|
|
record['after']['cursor']['committed_offset'] = 1
|
|
elif case == 'before':
|
|
record['before']['cursor']['last_job_id'] += 1
|
|
elif case == 'state':
|
|
record['state'] = 'COMMITTED'
|
|
value['sha256'] = '0' * 64 if case == 'checksum' else sha(encoded(record))
|
|
raw = encoded(value)
|
|
self.journal.write_bytes(raw)
|
|
self.db.statements.clear()
|
|
with self.subTest(case=case), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertEqual(self.journal.read_bytes(), raw)
|
|
|
|
def test_future_metadata_or_output_never_gets_rewound(self):
|
|
self.call()
|
|
stable = copy.deepcopy(self.db.rows)
|
|
for change in ({'committed_offset': 1}, {'last_event_id': 'later'}, {'generation': 15}):
|
|
self.db.rows = copy.deepcopy(stable)
|
|
self.db.found['cursor'].update(change)
|
|
if 'generation' in change:
|
|
self.db.found['stream']['current_generation'] = 15
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.db.statements.clear()
|
|
with self.subTest(change=change), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.assertEqual(self.db.updates, [])
|
|
self.db.rows = stable
|
|
path = self.m.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
|
path.write_bytes(b'future fixture output')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(path.read_bytes(), b'future fixture output')
|
|
|
|
def test_after_state_without_journal_is_refused(self):
|
|
self.call()
|
|
self.journal.unlink()
|
|
self.db.statements.clear()
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertFalse(self.journal.exists())
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_all_history_and_highwater_gates_refuse_drift(self):
|
|
original = dict(self.db.gates)
|
|
for key in original:
|
|
self.db.gates = dict(original)
|
|
self.db.gates[key] += 1
|
|
with self.subTest(gate=key), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertFalse(self.journal.exists())
|
|
|
|
def test_last_cursor_reference_must_match_retained_append(self):
|
|
original = dict(self.db.last)
|
|
for key, value in (('id', 1), ('job_id', 1), ('event_id', 'wrong'), ('event_hash', 'c' * 64),
|
|
('generation', 12), ('byte_offset', 0), ('byte_length', 1), ('state', 'prepared')):
|
|
self.db.last = dict(original)
|
|
self.db.last[key] = value
|
|
with self.subTest(field=key), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assertFalse(self.journal.exists())
|
|
|
|
def test_coverage_and_reviewed_metadata_are_strict(self):
|
|
original = copy.deepcopy(self.db.rows)
|
|
for case in ('duplicate', 'missing', 'orphan', 'mismatch', 'path', 'generation', 'offset', 'null reference', 'negative', 'bool'):
|
|
self.db.rows = copy.deepcopy(original)
|
|
if case == 'duplicate':
|
|
self.db.rows.append(copy.deepcopy(self.db.rows[-1]))
|
|
elif case == 'missing':
|
|
self.db.rows[-1]['cursor_stream_name'] = None
|
|
self.db.rows[-1]['cursor'] = None
|
|
elif case == 'orphan':
|
|
self.db.rows[-1]['stream_name'] = None
|
|
self.db.rows[-1]['stream'] = None
|
|
elif case == 'mismatch':
|
|
self.db.rows[-1]['cursor_stream_name'] = 'other'
|
|
elif case == 'path':
|
|
self.db.found['stream']['base_relative_path'] = '../found_secrets.jsonl'
|
|
elif case == 'generation':
|
|
self.db.found['stream']['current_generation'] = 14
|
|
self.db.found['cursor']['generation'] = 14
|
|
elif case == 'offset':
|
|
self.db.found['cursor']['committed_offset'] -= 1
|
|
elif case == 'null reference':
|
|
self.db.found['cursor']['last_append_id'] = None
|
|
else:
|
|
self.db.found['cursor']['committed_offset'] = -1 if case == 'negative' else True
|
|
with self.subTest(case=case), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_family_absence_includes_generations_manifest_ledger_and_links(self):
|
|
results = self.m.DATA / 'runtime-linux/results'
|
|
for name in ('found_secrets', 'found_secrets.jsonl', 'found_secrets.g000013.jsonl',
|
|
'found_secrets.manifest.json', 'found_secrets.publication-ledger.sqlite3',
|
|
'Found_Secrets.publication-ledger.sqlite3-wal'):
|
|
path = results / name
|
|
path.write_bytes(b'retained fixture must not be deleted')
|
|
with self.subTest(name=name), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(path.read_bytes(), b'retained fixture must not be deleted')
|
|
path.unlink()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_private_paths_and_journal_links_are_rejected(self):
|
|
self.denied.add(self.m.DATA / 'config')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.denied.clear()
|
|
self.prepare_without_commit()
|
|
real_lstat = Path.lstat
|
|
|
|
def linked(path, *args, **kwargs):
|
|
value = real_lstat(path, *args, **kwargs)
|
|
if path == self.journal:
|
|
return SimpleNamespace(st_mode=stat.S_IFLNK, st_file_attributes=0)
|
|
return value
|
|
|
|
with mock.patch.object(Path, 'lstat', linked), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
os.link(self.journal, self.root / 'journal-hardlink')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.journal.stat().st_nlink, 2)
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_initialized_or_supervisor_markers_block_every_call(self):
|
|
for path in (self.runtime.INITIALIZED, self.m.RUN / 'control/supervisor.instance.json',
|
|
self.m.DATA / 'runtime-linux/logs/supervisor.pid'):
|
|
path.write_bytes(b'fixture marker')
|
|
with self.subTest(path=path.name), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(path.read_bytes(), b'fixture marker')
|
|
path.unlink()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_bindings_and_connection_mode_cannot_be_relaxed(self):
|
|
for args in ({'manifest_sha256': '0' * 64}, {'system_identifier': '1111111111'},
|
|
{'system_identifier': 'not-an-identity'}):
|
|
with self.subTest(args=args), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call(**args)
|
|
for attribute, bad in (('autocommit', False), ('closed', True), ('broken', True)):
|
|
old = getattr(self.db, attribute)
|
|
setattr(self.db, attribute, bad)
|
|
with self.subTest(attribute=attribute), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
setattr(self.db, attribute, old)
|
|
self.db.info.transaction_status = 2
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_bound_file_and_manifest_tampering_are_rejected(self):
|
|
self.manifest_path.write_bytes(self.manifest_path.read_bytes() + b' ')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_outer_lock_objects_are_required_and_never_released(self):
|
|
self.initialize_lock.acquired = False
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.initialize_lock.acquired = True
|
|
self.authority_lock.data_directory = 'not-the-bound-target'
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assert_held()
|
|
|
|
def test_live_clients_foreign_identity_schema_and_recovery_are_refused(self):
|
|
baseline = dict(self.db.online)
|
|
for key, value in (('other_clients', 1), ('system_identifier', '3333333333'), ('data_directory', '/foreign'),
|
|
('database', 'foreign'), ('port', 15432), ('version_num', 170001),
|
|
('in_recovery', True), ('superuser', False), ('search_path', 'public,other'),
|
|
('schema_name', 'other'), ('read_only', 'on'), ('public_create', True),
|
|
('fsync', 'off'), ('full_page_writes', 'off')):
|
|
self.db.online = dict(baseline)
|
|
self.db.online[key] = value
|
|
with self.subTest(field=key), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_advisory_file_and_table_writer_conflicts_fail_closed(self):
|
|
self.db.advisory_available = False
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.db.advisory_available, self.db.tables_available = True, False
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.db.tables_available = True
|
|
self.security.PrivateFileLock.fail_projector = True
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
self.assert_held()
|
|
|
|
def test_new_client_at_precommit_rolls_back_with_prepared_journal(self):
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.db.other_client_call = 3
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.assertTrue(self.journal.exists())
|
|
self.assertIn('rollback', self.trace)
|
|
self.assert_held()
|
|
|
|
def test_cancel_or_new_output_at_precommit_rolls_back_without_deleting_files(self):
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.runtime._shutdown_requested = True
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertFalse(self.journal.exists())
|
|
self.runtime._shutdown_requested = False
|
|
self.db.after_update = lambda: setattr(self.runtime, '_shutdown_requested', True)
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.runtime._shutdown_requested = False
|
|
path = self.m.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
|
self.db.after_update = lambda: path.write_bytes(b'external fixture output')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.assertEqual(path.read_bytes(), b'external fixture output')
|
|
self.assert_held()
|
|
|
|
def test_unknown_tables_counts_and_late_catalog_changes_are_refused(self):
|
|
self.db.counts['unexpected_table'] = 0
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
del self.db.counts['unexpected_table']
|
|
self.db.counts['findings'] += 1
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.db.counts['findings'] -= 1
|
|
self.assertFalse(self.journal.exists())
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.db.after_update = lambda: self.db.counts.update(unexpected_table=0)
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.assertTrue(self.journal.exists())
|
|
|
|
def test_bound_identity_and_source_identity_must_remain_independent(self):
|
|
original = self.identity_path.read_bytes()
|
|
for field, value in (('system_identifier', '3333333333'), ('data_directory', '/foreign'),
|
|
('database', 'foreign'), ('user', 'foreign'), ('port', 15432), ('pg_major', 17)):
|
|
invalid = dict(self.identity)
|
|
invalid[field] = value
|
|
self.identity_path.write_bytes(encoded(invalid))
|
|
with self.subTest(field=field), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.identity_path.write_bytes(encoded(dict(self.identity, system_identifier='1111111111')))
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call(system_identifier='1111111111')
|
|
self.identity_path.write_bytes(original)
|
|
before = copy.deepcopy(self.db.rows)
|
|
self.db.after_update = lambda: self.identity_path.write_bytes(original + b' ')
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before)
|
|
self.assert_held()
|
|
|
|
def test_wrong_affected_row_count_and_history_side_effects_roll_back(self):
|
|
before_rows, before_history = copy.deepcopy(self.db.rows), copy.deepcopy(self.db.history)
|
|
for count in (0, 2):
|
|
self.db.rowcount = count
|
|
with self.subTest(rowcount=count), self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before_rows)
|
|
self.db.rowcount, self.db.corrupt_history = 1, True
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.rows, before_rows)
|
|
self.assertEqual(self.db.history, before_history)
|
|
self.assert_held()
|
|
|
|
def test_retry_refuses_protected_history_or_sequence_drift(self):
|
|
self.prepare_without_commit()
|
|
self.db.history['pipeline_leases']['state'] = 'changed by another operation'
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.db.history['pipeline_leases']['state'] = 'pipeline_leases'
|
|
self.db.sequence['last_value'] += 1
|
|
with self.assertRaises(self.m.ProjectionRecoveryError):
|
|
self.call()
|
|
self.assertEqual(self.db.updates, [])
|
|
|
|
def test_raw_exception_payloads_are_not_diagnostics(self):
|
|
self.db.fail_update = 'cursor'
|
|
try:
|
|
self.call()
|
|
except self.m.ProjectionRecoveryError as exc:
|
|
diagnostic = ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
|
self.assertNotIn(PRIVATE, diagnostic)
|
|
self.assertIn('retain caller maintenance authority', str(exc))
|
|
else:
|
|
self.fail('fixture error was not refused')
|
|
self.assert_held()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|