Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,668 @@
|
||||
"""Real PG16 regression; run only in the trusted, offline disposable test image.
|
||||
|
||||
Main provisions a fresh truf-projection-loss-test-<32 hex> volume separately.
|
||||
Run python -u -I -S -B /opt/truf/tests/container_projection_recovery_e2e.py
|
||||
--budget-seconds 300. The budget is cooperative: it NEVER kills PostgreSQL or
|
||||
releases uncertain lifecycle authority. A FAILED_HOLD may outlive that budget.
|
||||
|
||||
The actual schema/migration helpers, queries, transactions and locks are used.
|
||||
Only the recovery module's manifest pin is substituted in memory for this fresh
|
||||
fixture. Fault adapters raise only AFTER real SQL execution or real commit.
|
||||
No accepted journal is deleted, no old output is rebuilt, and no application
|
||||
initialized marker, supervisor, scanner, ingester or provider is started.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import runpy
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
OUTPUT = sys.stdout
|
||||
STOPPED = False
|
||||
IMAGE_PATH = Path('/opt/truf/tests/container_projection_recovery_e2e.py')
|
||||
STAMP = '2026-09-16T00:00:00+00:00'
|
||||
APPENDS = 38024
|
||||
OLD_OFFSET = 97783145
|
||||
|
||||
|
||||
def require(value, check):
|
||||
if not value:
|
||||
raise AssertionError(check)
|
||||
|
||||
|
||||
def emit(**values):
|
||||
try:
|
||||
print(json.dumps(values, sort_keys=True), file=OUTPUT, flush=True)
|
||||
except BaseException:
|
||||
pass
|
||||
|
||||
|
||||
def safe_error(error):
|
||||
try:
|
||||
line, state, current = 0, None, error
|
||||
for _ in range(8):
|
||||
tb = current.__traceback__
|
||||
while tb is not None:
|
||||
if tb.tb_frame.f_code.co_filename == str(IMAGE_PATH):
|
||||
line = tb.tb_lineno
|
||||
tb = tb.tb_next
|
||||
candidate = getattr(current, 'sqlstate', None)
|
||||
if state is None and isinstance(candidate, str) and re.fullmatch(r'[A-Z0-9]{5}', candidate):
|
||||
state = candidate
|
||||
current = current.__cause__ or current.__context__
|
||||
if current is None:
|
||||
break
|
||||
name = type(error).__name__
|
||||
if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,63}', name or ''):
|
||||
name = 'other'
|
||||
return {'error_type': next((i for i, kind in enumerate(
|
||||
(TimeoutError, OSError, ValueError, RuntimeError, KeyboardInterrupt, AssertionError,
|
||||
TypeError, KeyError, AttributeError, ImportError, LookupError), 1)
|
||||
if isinstance(error, kind)), 0), 'line': line, 'sqlstate': state, 'error_class': name}
|
||||
except BaseException:
|
||||
return {'error_type': 0, 'line': 0, 'sqlstate': None, 'error_class': 'other'}
|
||||
|
||||
|
||||
def hold_pause():
|
||||
try:
|
||||
time.sleep(2)
|
||||
except BaseException:
|
||||
pass
|
||||
|
||||
|
||||
def stop_confirmed(backend, pg):
|
||||
"""Retain this exact backend and the caller's two locks until positive stop."""
|
||||
global STOPPED
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
attempts += 1
|
||||
result = backend.stop()
|
||||
require(result.completed is True and result.stopped is True, 'stop_result')
|
||||
require(backend.probe().kind == pg.ProbeKind.STOPPED, 'stopped_probe')
|
||||
backend.close()
|
||||
STOPPED = True
|
||||
emit(stage='stop', stopped=True, attempts=attempts)
|
||||
return
|
||||
except BaseException as error:
|
||||
emit(stage='stop', failed_hold=True, attempts=attempts, **safe_error(error))
|
||||
hold_pause()
|
||||
|
||||
|
||||
def initialize_empty(runtime, config_path):
|
||||
"""The real lifecycle child owns initialization compensation; never kill it."""
|
||||
try:
|
||||
child = subprocess.Popen(runtime._bootstrap_command(
|
||||
'postgres-runtime', 'initialize-empty', '--config', str(config_path)),
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
except BaseException as error:
|
||||
emit(stage='initialize_empty', failed_hold=True, **safe_error(error))
|
||||
return None
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
code = child.wait(timeout=10)
|
||||
emit(stage='initialize_empty', completed=code == 0, exit_code=code)
|
||||
return code
|
||||
except BaseException as error:
|
||||
attempts += 1
|
||||
emit(stage='initialize_empty', failed_hold=True, attempts=attempts, **safe_error(error))
|
||||
hold_pause()
|
||||
|
||||
|
||||
def checkpoint(runtime, deadline):
|
||||
require(runtime._shutdown_requested is False, 'shutdown_requested')
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError('driver_budget')
|
||||
|
||||
|
||||
def identifier(name):
|
||||
require(isinstance(name, str) and re.fullmatch(r'[a-z_][a-z0-9_]*', name), 'fixture_identifier')
|
||||
return '"' + name + '"'
|
||||
|
||||
|
||||
def digest(payload):
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def metadata(connection):
|
||||
row = connection.execute("""SELECT pg_catalog.row_to_json(s) AS stream,
|
||||
pg_catalog.row_to_json(c) AS cursor FROM public.projection_streams s
|
||||
JOIN public.projection_cursors c USING (stream_name)
|
||||
WHERE s.stream_name = 'found_secrets'""").fetchone()
|
||||
parsed = {}
|
||||
for key in ('stream', 'cursor'):
|
||||
value = row[key]
|
||||
if isinstance(value, str):
|
||||
value = json.loads(value)
|
||||
require(isinstance(value, dict), 'metadata_object')
|
||||
parsed[key] = value
|
||||
return parsed
|
||||
|
||||
|
||||
def proof(connection, check):
|
||||
"""Independent, bounded-fixture whole-row digests; never return raw rows."""
|
||||
tables = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'r' ORDER BY c.relname""").fetchall()
|
||||
result = {'tables': {}, 'sequences': {'public': {}}}
|
||||
for table in tables:
|
||||
check()
|
||||
name = table['name']
|
||||
where = " WHERE t.stream_name <> 'found_secrets'" if name in ('projection_streams', 'projection_cursors') else ''
|
||||
row = connection.execute("""SELECT count(*) AS rows, pg_catalog.encode(pg_catalog.sha256(
|
||||
pg_catalog.convert_to(COALESCE(string_agg(h, '' ORDER BY h), ''), 'UTF8')), 'hex') AS sha256
|
||||
FROM (SELECT pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(
|
||||
pg_catalog.row_to_json(t)::text, 'UTF8')), 'hex') COLLATE "C" AS h FROM public."""
|
||||
+ identifier(name) + ' AS t' + where + ') AS hashes').fetchone()
|
||||
require(0 <= row['rows'] <= 100000, 'bounded_fixture')
|
||||
result['tables'][name] = row
|
||||
sequences = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'S' ORDER BY c.relname""").fetchall()
|
||||
for row in sequences:
|
||||
result['sequences']['public'][row['name']] = connection.execute(
|
||||
'SELECT last_value, is_called FROM public.' + identifier(row['name'])).fetchone()
|
||||
return result
|
||||
|
||||
|
||||
def journal_snapshot(runtime, recovery):
|
||||
from container_import import _input, _json
|
||||
|
||||
path = runtime.DATA / 'config' / recovery.JOURNAL_NAME
|
||||
with _input(path, runtime) as (handle, before):
|
||||
require(0 < before[4] <= recovery.MAX_JOURNAL, 'journal_bound')
|
||||
raw = handle.read(recovery.MAX_JOURNAL + 1)
|
||||
value = _json(raw)
|
||||
require(raw == recovery._encoded(value) and value['record']['state'] == 'PREPARED'
|
||||
and value['sha256'] == digest(recovery._encoded(value['record'])), 'durable_journal')
|
||||
return raw, before, value
|
||||
|
||||
|
||||
class ObservedConnection:
|
||||
"""Delegate everything to psycopg; inject only after a real operation."""
|
||||
def __init__(self, connection, runtime, recovery, check, *, after_update=False, lost_ack=False):
|
||||
self.connection, self.runtime, self.recovery, self.check = connection, runtime, recovery, check
|
||||
self.after_update, self.lost_ack = after_update, lost_ack
|
||||
self.updates, self.commits, self.injected = 0, 0, False
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.connection, name)
|
||||
|
||||
def execute(self, query, *args, **kwargs):
|
||||
result = self.connection.execute(query, *args, **kwargs)
|
||||
sql = ' '.join(query.split()).upper() if isinstance(query, str) else ''
|
||||
if sql.startswith('UPDATE '):
|
||||
self.updates += 1
|
||||
if self.after_update and sql.startswith('UPDATE PUBLIC.PROJECTION_STREAMS '):
|
||||
require(result.rowcount == 1, 'real_first_update')
|
||||
journal_snapshot(self.runtime, self.recovery)
|
||||
self.after_update, self.injected = False, True
|
||||
raise OSError('synthetic_transport_after_real_update')
|
||||
self.check()
|
||||
return result
|
||||
|
||||
@contextlib.contextmanager
|
||||
def transaction(self, *args, **kwargs):
|
||||
with self.connection.transaction(*args, **kwargs) as transaction:
|
||||
yield transaction
|
||||
self.commits += 1
|
||||
if self.lost_ack:
|
||||
self.lost_ack, self.injected = False, True
|
||||
raise OSError('synthetic_ack_loss_after_real_commit')
|
||||
|
||||
|
||||
def seed_history(connection, runtime, importer):
|
||||
"""Seed real production tables; historical byte proofs need no old files."""
|
||||
with connection.transaction():
|
||||
connection.execute("""INSERT INTO public.target_scans(
|
||||
id, scan_event_id, scan_event_hash, source, target, normalized_target, scan_type,
|
||||
status, ended_at, findings_count, raw_result_storage, created_at)
|
||||
SELECT n, lpad(to_hex(n), 32, '0'), encode(sha256(convert_to('event-' || n, 'UTF8')), 'hex'),
|
||||
'fixture', 'fixture:' || n, 'fixture:' || n, 'fixture', 'found', %s, 1, 'normalized_v2', %s
|
||||
FROM generate_series(1, 38024) AS g(n)""", (STAMP, STAMP))
|
||||
connection.execute("""INSERT INTO public.findings(
|
||||
id, target_scan_id, source, target, detector_name, detector_type, verified,
|
||||
raw_secret, redacted_secret, secret_hash, finding_uid, created_at)
|
||||
SELECT id, id, 'fixture', target, 'Fixture', 'fixture', 0,
|
||||
'synthetic-only-' || id, 'fixture', encode(sha256(convert_to('synthetic-only-' || id, 'UTF8')), 'hex'),
|
||||
encode(sha256(convert_to('finding-' || id, 'UTF8')), 'hex'), %s FROM public.target_scans""", (STAMP,))
|
||||
connection.execute("""INSERT INTO public.scan_result_compat(
|
||||
target_scan_id, schema_version, metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at)
|
||||
SELECT id, 2, '{}', encode(sha256(convert_to('{}', 'UTF8')), 'hex'), 2, 'exact', %s
|
||||
FROM public.target_scans""", (STAMP,))
|
||||
connection.execute("""INSERT INTO public.finding_compat_payloads(
|
||||
finding_id, raw_value, extension_json, payload_sha256, payload_bytes, payload_omitted, created_at)
|
||||
SELECT id, raw_secret, '{}', encode(sha256(convert_to(raw_secret, 'UTF8')), 'hex'),
|
||||
octet_length(raw_secret), 0, %s FROM public.findings""", (STAMP,))
|
||||
connection.execute("""INSERT INTO public.projection_jobs(
|
||||
id, job_kind, event_id, event_hash, target_scan_id, status, required_stream_mask,
|
||||
capacity_items, capacity_bytes, capacity_released, attempts, created_at, updated_at, completed_at)
|
||||
SELECT id, 'scan_event', scan_event_id, scan_event_hash, id, 'completed', 2,
|
||||
1, 65536, 1, 1, %s, %s, %s FROM public.target_scans""", (STAMP, STAMP, STAMP))
|
||||
connection.execute("""WITH positions AS (
|
||||
SELECT id, event_id, event_hash, (id - 1) / 2716 AS generation,
|
||||
CASE WHEN id > 35308 THEN 97783145::bigint ELSE 134217728::bigint END AS bytes,
|
||||
(id - 1) %% 2716 AS ordinal FROM public.projection_jobs)
|
||||
INSERT INTO public.projection_appends(id, job_id, stream_name, event_id, event_hash,
|
||||
generation, byte_offset, byte_length, payload_sha256, record_count, state, prepared_at, appended_at)
|
||||
SELECT id, id, 'found_secrets', event_id, event_hash, generation,
|
||||
bytes * ordinal / 2716, bytes * (ordinal + 1) / 2716 - bytes * ordinal / 2716,
|
||||
encode(sha256(convert_to('historical-output-' || id, 'UTF8')), 'hex'),
|
||||
1, 'appended', %s, %s FROM positions""", (STAMP, STAMP))
|
||||
connection.execute("""INSERT INTO public.projection_rotations(
|
||||
id, stream_name, from_generation, to_generation, source_bytes, segment_relative_path, state, created_at, completed_at)
|
||||
SELECT n + 1, 'found_secrets', n, n + 1, 134217728,
|
||||
'found_secrets.g' || lpad(n::text, 6, '0') || '.jsonl', 'completed', %s, %s
|
||||
FROM generate_series(0, 12) AS g(n)""", (STAMP, STAMP))
|
||||
for i in range(18):
|
||||
provider = f'p{i:02d}'
|
||||
for suffix, filename in (('results', 'Results.jsonl'), ('status', 'Checked.txt')):
|
||||
if suffix == 'status' and i >= 13:
|
||||
continue
|
||||
name = f'keycheck:{provider}:{suffix}'
|
||||
connection.execute("""INSERT INTO public.projection_streams(
|
||||
stream_name, base_relative_path, current_generation, rotation_bytes, max_generations, created_at, updated_at)
|
||||
VALUES (%s, %s, 0, 33554432, 16, %s, %s)""", (name, f'{provider}/{provider}{filename}', STAMP, STAMP))
|
||||
connection.execute("""INSERT INTO public.projection_cursors(stream_name, generation, committed_offset, updated_at)
|
||||
VALUES (%s, 0, %s, %s)""", (name, 1000 + i if suffix == 'status' else 0, STAMP))
|
||||
connection.execute("UPDATE public.projection_streams SET current_generation = 13 WHERE stream_name = 'found_secrets'")
|
||||
connection.execute("""UPDATE public.projection_cursors SET generation = 13, committed_offset = 97783145,
|
||||
last_append_id = 38024, last_job_id = 38024,
|
||||
last_event_id = (SELECT event_id FROM public.projection_jobs WHERE id = 38024),
|
||||
last_event_hash = (SELECT event_hash FROM public.projection_jobs WHERE id = 38024)
|
||||
WHERE stream_name = 'found_secrets'""")
|
||||
for worker in ('result_ingester', 'jsonl_projector'):
|
||||
connection.execute("""INSERT INTO public.pipeline_leases(worker_name, generation, lease_token,
|
||||
supervisor_instance_id, owner_pid, owner_creation_time, owner_executable, state,
|
||||
acquired_at, heartbeat_at, lease_expires_at, updated_at)
|
||||
VALUES (%s, 7, 'synthetic-expired', 'synthetic-expired', 999999, 'synthetic',
|
||||
'/synthetic/expired', 'ready', %s, %s, %s, %s)""", (worker, STAMP, STAMP, STAMP, STAMP))
|
||||
for table in ('target_scans', 'findings', 'projection_jobs', 'projection_appends', 'projection_rotations'):
|
||||
connection.execute('SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence(%s, %s), '
|
||||
+ '(SELECT max(id) FROM public.' + identifier(table) + '), true)', ('public.' + table, 'id'))
|
||||
status_files = []
|
||||
for i in range(13):
|
||||
provider = f'p{i:02d}'
|
||||
directory = runtime.DATA / 'runtime-linux/keychecks' / provider
|
||||
directory.mkdir(mode=0o700)
|
||||
path = directory / f'{provider}Checked.txt'
|
||||
importer._write(runtime, path, b'synthetic status snapshot\n')
|
||||
status_files.append(path)
|
||||
return status_files
|
||||
|
||||
|
||||
def fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, system_identifier):
|
||||
# Required source labels are inert format metadata, never opened as paths.
|
||||
paths = list(status_files)
|
||||
for name in sorted(importer.REQUIRED_FILES):
|
||||
path = runtime.DATA / name
|
||||
if not os.path.lexists(path):
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
importer._write(runtime, path, b'')
|
||||
paths.append(path)
|
||||
files = []
|
||||
for path in paths:
|
||||
with importer._input(path, runtime) as (handle, before):
|
||||
raw = handle.read(4096)
|
||||
require(len(raw) == before[4], 'status_fixture_bound')
|
||||
files.append({'path': path.relative_to(runtime.DATA).as_posix(), 'size': len(raw), 'sha256': digest(raw)})
|
||||
# Historical bytes are intentionally not materialized; this is not a restore test.
|
||||
files.append({'path': 'runtime-linux/results/found_secrets.jsonl', 'size': OLD_OFFSET,
|
||||
'sha256': digest(b'intentionally-unavailable-synthetic-output')})
|
||||
value = {'format': 'truf-windows-snapshot-v1',
|
||||
'source': {'root': r'D:\truf', 'postgres_data_dir': r'S:\postgres-data',
|
||||
'supervisor_stopped': True, 'postgres_stopped': True},
|
||||
'database': {'version_num': connection.execute("SELECT current_setting('server_version_num')::int AS version").fetchone()['version'],
|
||||
'system_identifier': '1' if system_identifier != '1' else '2',
|
||||
'database_name': 'synthetic_source', 'user_name': 'synthetic_source', 'port': 15432,
|
||||
'data_directory': r'S:\postgres-data', 'bytes': 6, 'sha256': digest(b'PGDMPx'),
|
||||
'table_counts': {name: item['rows'] + int(name in ('projection_streams', 'projection_cursors'))
|
||||
for name, item in baseline['tables'].items()},
|
||||
'sequence_states': baseline['sequences'], 'sequence_count': len(baseline['sequences']['public'])},
|
||||
'archive': {'bytes': sum(item['size'] for item in files) + 10240,
|
||||
'sha256': digest(b'synthetic archive identity')}, 'files': files}
|
||||
raw = importer._encoded(value)
|
||||
pin = digest(raw)
|
||||
importer._manifest(raw, pin)
|
||||
importer._write(runtime, runtime.DATA / 'config/windows-import-manifest.json', raw)
|
||||
recovery.APPROVED_MANIFEST_SHA256 = pin
|
||||
return pin
|
||||
|
||||
|
||||
def exercise_projector(connection, db, runtime, config, check):
|
||||
from jsonl_projector import JsonlProjector
|
||||
from migrate_runtime_safety import initialize_projection_cursors_from_existing_files, require_legacy_cutover_clear
|
||||
|
||||
# This is a real offline cutover after recovery, not a mocked readiness gate.
|
||||
db.require_runtime_safety_schema()
|
||||
cursors = initialize_projection_cursors_from_existing_files(db, config)
|
||||
legacy = require_legacy_cutover_clear(db, config)
|
||||
migrations = db.conn.execute('SELECT version, code_sha256 FROM runtime_schema_migrations ORDER BY version').fetchall()
|
||||
db.conn.commit()
|
||||
db.record_final_cutover({'legacy': legacy, 'projection_cursors': cursors,
|
||||
'schema_migrations': [dict(row) for row in migrations]})
|
||||
db.require_final_cutover()
|
||||
check()
|
||||
event_id, event_hash, finding_uid = 'f' * 32, digest(b'future-event'), digest(b'future-finding')
|
||||
with connection.transaction():
|
||||
scan_id = connection.execute("""INSERT INTO public.target_scans(
|
||||
scan_event_id, scan_event_hash, target, normalized_target, scan_type, status, ended_at,
|
||||
findings_count, raw_result_storage, created_at)
|
||||
VALUES (%s, %s, 'fixture:future', 'fixture:future', 'fixture', 'found', %s, 1, 'normalized_v2', %s)
|
||||
RETURNING id""", (event_id, event_hash, STAMP, STAMP)).fetchone()['id']
|
||||
connection.execute("""INSERT INTO public.scan_result_compat(target_scan_id, schema_version,
|
||||
metadata_json, metadata_sha256, metadata_bytes, reconstruction_status, created_at)
|
||||
VALUES (%s, 2, '{}', %s, 2, 'exact', %s)""", (scan_id, digest(b'{}'), STAMP))
|
||||
finding_id = connection.execute("""INSERT INTO public.findings(target_scan_id, detector_name,
|
||||
detector_type, verified, raw_secret, redacted_secret, finding_uid, created_at)
|
||||
VALUES (%s, 'Fixture', 'fixture', 0, 'synthetic-future', 'fixture', %s, %s) RETURNING id""",
|
||||
(scan_id, finding_uid, STAMP)).fetchone()['id']
|
||||
connection.execute("""INSERT INTO public.finding_compat_payloads(finding_id, raw_value,
|
||||
extension_json, payload_sha256, payload_bytes, payload_omitted, created_at)
|
||||
VALUES (%s, 'synthetic-future', '{}', %s, 16, 0, %s)""", (finding_id, digest(b'synthetic-future'), STAMP))
|
||||
job_id = connection.execute("""INSERT INTO public.projection_jobs(job_kind, event_id, event_hash,
|
||||
target_scan_id, status, required_stream_mask, capacity_items, capacity_bytes, created_at, updated_at)
|
||||
VALUES ('scan_event', %s, %s, %s, 'pending', 2, 1, 65536, %s, %s) RETURNING id""",
|
||||
(event_id, event_hash, scan_id, STAMP, STAMP)).fetchone()['id']
|
||||
connection.execute("""UPDATE public.pipeline_capacity SET projection_items = projection_items + 1,
|
||||
projection_bytes = projection_bytes + 65536 WHERE id = 1""")
|
||||
worker = JsonlProjector(db, str(runtime.DATA / 'runtime-linux/results'), 'projection-loss-e2e',
|
||||
keycheck_dir=str(runtime.DATA / 'runtime-linux/keychecks'))
|
||||
try:
|
||||
worker.start()
|
||||
require(worker.process_one() is True, 'projector_processed')
|
||||
require(worker.process_one() is False, 'projector_idle')
|
||||
row = connection.execute("""SELECT a.generation, a.byte_offset, a.byte_length,
|
||||
a.payload_sha256, a.state, j.status, j.capacity_released
|
||||
FROM public.projection_appends a JOIN public.projection_jobs j ON j.id = a.job_id
|
||||
WHERE a.job_id = %s AND a.stream_name = 'found_secrets'""", (job_id,)).fetchone()
|
||||
path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
||||
runtime.private_path(path)
|
||||
require(path.stat().st_size < 65536, 'bounded_new_output')
|
||||
raw = path.read_bytes()
|
||||
require(row and row['generation'] == 14 and row['byte_offset'] == 0
|
||||
and row['byte_length'] == len(raw) and row['payload_sha256'] == digest(raw)
|
||||
and row['state'] == 'appended' and row['status'] == 'completed' and row['capacity_released'] == 1,
|
||||
'projector_fenced_append')
|
||||
require(json.loads(raw)['finding_uid'] == finding_uid, 'projector_real_payload')
|
||||
cursor = metadata(connection)['cursor']
|
||||
require(cursor['generation'] == 14 and cursor['committed_offset'] == len(raw)
|
||||
and cursor['last_job_id'] == job_id, 'projector_cursor')
|
||||
capacity = connection.execute('SELECT projection_items, projection_bytes FROM public.pipeline_capacity WHERE id = 1').fetchone()
|
||||
require(capacity == {'projection_items': 0, 'projection_bytes': 0}, 'projector_capacity')
|
||||
emit(stage='projector', passed=True, generation=14, records=1, bytes=len(raw))
|
||||
finally:
|
||||
worker.stop()
|
||||
|
||||
|
||||
def run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check):
|
||||
import container_import as importer
|
||||
import container_projection_recovery as recovery
|
||||
from runtime_security import PrivateFileLock
|
||||
from scanner_db import ScannerDB, migrate_runtime_safety_schema
|
||||
|
||||
schema_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False)
|
||||
resources.append(schema_db)
|
||||
require(schema_db.enabled and schema_db.conn.is_postgres, 'real_schema_connection')
|
||||
try:
|
||||
migrate_runtime_safety_schema(schema_db, initialize_base=True)
|
||||
schema_db.require_runtime_safety_schema()
|
||||
finally:
|
||||
schema_db.close()
|
||||
emit(stage='schema', passed=True)
|
||||
check()
|
||||
status_files = seed_history(connection, runtime, importer)
|
||||
emit(stage='seed', passed=True)
|
||||
baseline = proof(connection, check)
|
||||
emit(stage='proof', passed=True, tables=len(baseline['tables']), sequences=len(baseline['sequences']['public']))
|
||||
before = metadata(connection)
|
||||
require(before['stream']['current_generation'] == before['cursor']['generation'] == 13
|
||||
and before['cursor']['committed_offset'] == OLD_OFFSET, 'reviewed_before')
|
||||
require(baseline['tables']['projection_appends']['rows'] == APPENDS
|
||||
and baseline['tables']['projection_rotations']['rows'] == 13
|
||||
and baseline['tables']['projection_append_audit']['rows'] == 0, 'reviewed_history')
|
||||
original_pin = recovery.APPROVED_MANIFEST_SHA256
|
||||
pin = fixture_manifest(connection, runtime, importer, recovery, baseline, status_files, identity['system_identifier'])
|
||||
journal_path = runtime.DATA / 'config' / recovery.JOURNAL_NAME
|
||||
status_before = {path: (path.stat().st_ino, path.read_bytes()) for path in status_files}
|
||||
emit(stage='fixture', passed=True, streams=34, appends=APPENDS, rotations=13,
|
||||
tables=len(baseline['tables']), sequences=len(baseline['sequences']['public']))
|
||||
|
||||
def recover(observed):
|
||||
return recovery.recover_found_secrets_projection(runtime, observed,
|
||||
system_identifier=identity['system_identifier'], manifest_sha256=pin,
|
||||
initialize_lock=initialize_lock, authority_lock=authority_lock)
|
||||
|
||||
def refused(observed):
|
||||
try:
|
||||
recover(observed)
|
||||
except recovery.ProjectionRecoveryError as error:
|
||||
if not observed.injected and (observed.after_update or observed.lost_ack):
|
||||
emit(stage='fault_not_reached', passed=False, **safe_error(error))
|
||||
return
|
||||
raise AssertionError('expected_recovery_refusal')
|
||||
|
||||
def alone():
|
||||
while True:
|
||||
check()
|
||||
connection.execute('SELECT pg_catalog.pg_stat_clear_snapshot()')
|
||||
count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_stat_activity
|
||||
WHERE backend_type = 'client backend' AND pid <> pg_backend_pid()""").fetchone()['count']
|
||||
if count == 0:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
|
||||
try:
|
||||
alone()
|
||||
contender = connect()
|
||||
resources.append(contender)
|
||||
try:
|
||||
observed = ObservedConnection(connection, runtime, recovery, check)
|
||||
refused(observed)
|
||||
require(observed.updates == 0, 'other_client_no_updates')
|
||||
with contender.transaction():
|
||||
contender.execute('LOCK TABLE public.projection_streams IN ROW EXCLUSIVE MODE')
|
||||
observed = ObservedConnection(connection, runtime, recovery, check)
|
||||
refused(observed)
|
||||
require(observed.updates == 0, 'writer_contention_no_updates')
|
||||
finally:
|
||||
contender.close()
|
||||
alone()
|
||||
with PrivateFileLock(str(runtime.DATA / 'runtime-linux/results/.jsonl-projector.lock')):
|
||||
observed = ObservedConnection(connection, runtime, recovery, check)
|
||||
refused(observed)
|
||||
require(observed.updates == 0, 'projector_file_lock_no_updates')
|
||||
require(metadata(connection) == before and proof(connection, check) == baseline
|
||||
and not os.path.lexists(journal_path), 'contention_unchanged')
|
||||
emit(stage='contention', passed=True, cases=3, updates=0)
|
||||
|
||||
observed = ObservedConnection(connection, runtime, recovery, check, after_update=True)
|
||||
refused(observed)
|
||||
require(observed.injected and observed.updates == 1 and observed.commits == 0, 'rollback_fault_window')
|
||||
require(int(connection.info.transaction_status) == 0 and metadata(connection) == before
|
||||
and proof(connection, check) == baseline, 'both_rows_rolled_back')
|
||||
journal = journal_snapshot(runtime, recovery)
|
||||
require(journal[2]['record']['before'] == before, 'journal_before')
|
||||
emit(stage='rollback', passed=True, updates=1, commits=0, journal_retained=True)
|
||||
|
||||
observed = ObservedConnection(connection, runtime, recovery, check, lost_ack=True)
|
||||
refused(observed)
|
||||
require(observed.injected and observed.updates == 2 and observed.commits == 1, 'real_commit_before_ack_loss')
|
||||
after = metadata(connection)
|
||||
require(after == journal[2]['record']['after'] and proof(connection, check) == baseline
|
||||
and journal_snapshot(runtime, recovery) == journal, 'committed_despite_ack_loss')
|
||||
emit(stage='lost_ack', passed=True, updates=2, commits=1, history_unchanged=True)
|
||||
|
||||
observed = ObservedConnection(connection, runtime, recovery, check)
|
||||
result = recover(observed)
|
||||
require(result['status'] == 'already-committed' and observed.updates == 0
|
||||
and result['journal_sha256'] == digest(journal[0]) and metadata(connection) == after
|
||||
and proof(connection, check) == baseline and journal_snapshot(runtime, recovery) == journal,
|
||||
'idempotent_readonly_retry')
|
||||
require({path: (path.stat().st_ino, path.read_bytes()) for path in status_files} == status_before,
|
||||
'other_output_unchanged')
|
||||
emit(stage='retry', passed=True, updates=0, journal_unchanged=True, history_unchanged=True, sequences_unchanged=True)
|
||||
|
||||
writer_db = ScannerDB(db_url=os.environ['SCANNER_DB_URL'], initialize=False)
|
||||
resources.append(writer_db)
|
||||
require(writer_db.enabled and writer_db.conn.is_postgres, 'real_writer_connection')
|
||||
try:
|
||||
exercise_projector(connection, writer_db, runtime, config, check)
|
||||
finally:
|
||||
writer_db.close()
|
||||
alone()
|
||||
advanced, advanced_proof = metadata(connection), proof(connection, check)
|
||||
future_path = runtime.DATA / 'runtime-linux/results/found_secrets.jsonl'
|
||||
future_bytes, future_inode = future_path.read_bytes(), future_path.stat().st_ino
|
||||
observed = ObservedConnection(connection, runtime, recovery, check)
|
||||
refused(observed)
|
||||
require(observed.updates == 0 and advanced['cursor']['committed_offset'] > 0
|
||||
and metadata(connection) == advanced and proof(connection, check) == advanced_proof
|
||||
and (future_path.read_bytes(), future_path.stat().st_ino) == (future_bytes, future_inode)
|
||||
and journal_snapshot(runtime, recovery) == journal, 'future_output_not_rewound')
|
||||
emit(stage='future_output', passed=True, updates=0, advanced_cursor_retained=True, journal_unchanged=True)
|
||||
finally:
|
||||
recovery.APPROVED_MANIFEST_SHA256 = original_pin
|
||||
|
||||
|
||||
def main():
|
||||
global OUTPUT
|
||||
# Native pg_ctl also inherits fd 1/2: Python stream redirection alone is insufficient.
|
||||
OUTPUT = os.fdopen(os.dup(1), 'w', encoding='utf-8', buffering=1)
|
||||
sink = os.open(os.devnull, os.O_WRONLY)
|
||||
try:
|
||||
os.dup2(sink, 1)
|
||||
os.dup2(sink, 2)
|
||||
finally:
|
||||
os.close(sink)
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False)
|
||||
parser.add_argument('--budget-seconds', type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
require(30 <= args.budget_seconds <= 3600, 'budget')
|
||||
started = time.monotonic()
|
||||
deadline = started + args.budget_seconds
|
||||
require(sys.platform == 'linux' and os.geteuid() == os.getuid() == 10001
|
||||
and os.getegid() == os.getgid() == 10001, 'test_identity')
|
||||
require(Path(__file__) == IMAGE_PATH and sys.flags.isolated and sys.flags.no_site
|
||||
and sys.flags.dont_write_bytecode, 'test_image')
|
||||
require({name for _, name in socket.if_nameindex()} == {'lo'}, 'offline_test')
|
||||
raw_mounts = Path('/proc/self/mountinfo').read_bytes()
|
||||
require(len(raw_mounts) <= 1024 * 1024, 'mount_bound')
|
||||
mounts = [line.split() for line in raw_mounts.decode('utf-8', errors='strict').splitlines()]
|
||||
data = [row for row in mounts if len(row) > 6 and (row[4] == '/data' or row[4].startswith('/data/'))]
|
||||
require(len(data) == 1 and data[0][4] == '/data' and '-' in data[0]
|
||||
and re.fullmatch(r'/var/lib/docker/volumes/truf-projection-loss-test-[a-f0-9]{32}/_data', data[0][3])
|
||||
and data[0][data[0].index('-') + 1] == 'ext4', 'fresh_test_volume')
|
||||
runtime = SimpleNamespace(**runpy.run_path('/opt/truf/app/container_runtime.py'))
|
||||
runtime.require_container()
|
||||
runtime.private_path(IMAGE_PATH.parent, directory=True)
|
||||
runtime.private_path(IMAGE_PATH)
|
||||
runtime._shutdown_requested = False
|
||||
for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
|
||||
signal.signal(sig, lambda *_: setattr(runtime, '_shutdown_requested', True))
|
||||
|
||||
def fresh():
|
||||
runtime.private_path(runtime.DATA / 'postgres-linux', directory=True)
|
||||
require(not any((runtime.DATA / 'postgres-linux').iterdir()), 'empty_pgdata')
|
||||
require(not any((runtime.DATA / 'runtime-linux/results').iterdir()), 'empty_results')
|
||||
for name in ('runtime-linux/postgres/cluster_identity.json', 'initialized.json',
|
||||
'config/windows-import-manifest.json', 'config/found-secrets-loss-g13-g14.prepared.json'):
|
||||
require(not os.path.lexists(runtime.DATA / name), 'fresh_fixture')
|
||||
|
||||
fresh()
|
||||
config_path = runtime.DEFAULT_CONFIG
|
||||
config = runtime.prepare_environment(config_path)
|
||||
os.environ.update(TRUF_DB_STATEMENT_TIMEOUT_MS='120000', TRUF_DB_LOCK_TIMEOUT_MS='5000',
|
||||
TRUF_DB_IDLE_TRANSACTION_TIMEOUT_MS='300000')
|
||||
import postgres_runtime as pg
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from runtime_security import ClusterAuthorityLock, PrivateFileLock
|
||||
|
||||
def connect():
|
||||
return psycopg.connect(os.environ['SCANNER_DB_URL'], autocommit=True, row_factory=dict_row,
|
||||
connect_timeout=5, application_name='truf-projection-loss-e2e', tcp_user_timeout=30000,
|
||||
options='-c search_path=public -c statement_timeout=120000 -c lock_timeout=5000 '
|
||||
'-c idle_in_transaction_session_timeout=300000 -c row_security=off '
|
||||
'-c log_min_error_statement=panic -c log_min_messages=panic '
|
||||
'-c log_statement=none -c log_min_duration_statement=-1')
|
||||
|
||||
resources = []
|
||||
check = lambda: checkpoint(runtime, deadline)
|
||||
with PrivateFileLock(str(runtime.INITIALIZE_LOCK)) as initialize_lock:
|
||||
fresh()
|
||||
authority_lock = ClusterAuthorityLock(config, endpoint_dsn=os.environ['SCANNER_DB_URL'])
|
||||
code = initialize_empty(runtime, config_path)
|
||||
while True:
|
||||
try:
|
||||
authority_lock.acquire()
|
||||
break
|
||||
except BaseException as error:
|
||||
emit(stage='authority', failed_hold=True, **safe_error(error))
|
||||
hold_pause()
|
||||
try:
|
||||
backend = None
|
||||
while backend is None:
|
||||
try:
|
||||
backend = pg.PostgresBackend(config, stop_timeout_sec=60)
|
||||
except BaseException as error:
|
||||
emit(stage='backend', failed_hold=True, **safe_error(error))
|
||||
hold_pause()
|
||||
try:
|
||||
require(code == 0, 'initialize_empty_exit')
|
||||
check()
|
||||
require(backend.probe().kind == pg.ProbeKind.STOPPED, 'initial_stopped_probe')
|
||||
identity = pg.verify_cluster_identity(config)
|
||||
require(identity['pg_major'] == 16 and identity['data_directory'] == '/data/postgres-linux', 'bound_fixture')
|
||||
require(backend.start().accepted is True, 'direct_backend_start')
|
||||
while True:
|
||||
check()
|
||||
probe = backend.probe()
|
||||
if probe.kind == pg.ProbeKind.READY:
|
||||
break
|
||||
require(probe.kind == pg.ProbeKind.RECOVERING, 'authenticated_start')
|
||||
time.sleep(0.1)
|
||||
emit(stage='start', ready=True)
|
||||
connection = connect()
|
||||
resources.append(connection)
|
||||
count = connection.execute("""SELECT count(*) AS count FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind IN ('r','p','f')""").fetchone()['count']
|
||||
require(count == 0, 'virgin_schema')
|
||||
run_cases(connection, connect, resources, runtime, config, identity, initialize_lock, authority_lock, check)
|
||||
require(not os.path.lexists(runtime.INITIALIZED), 'no_application_marker')
|
||||
finally:
|
||||
try:
|
||||
for resource in reversed(resources):
|
||||
try:
|
||||
resource.close()
|
||||
except BaseException as error:
|
||||
emit(stage='client_close', failed_hold=True, **safe_error(error))
|
||||
finally:
|
||||
stop_confirmed(backend, pg)
|
||||
finally:
|
||||
authority_lock.release()
|
||||
emit(stage='finished', passed=True, stopped=STOPPED, application_initialized=False,
|
||||
elapsed_ms=round((time.monotonic() - started) * 1000))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
result = main()
|
||||
except BaseException as error:
|
||||
emit(stage='finished', passed=False, stopped=STOPPED, **safe_error(error))
|
||||
result = 1
|
||||
raise SystemExit(result)
|
||||
Reference in New Issue
Block a user