sashatrask/truf-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Initial server source import

Browse Source
This commit is contained in:
sashatrask
2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/container_projection_recovery.py
+416
View File
@@ -0,0 +1,416 @@
"""Explicit maintenance-only loss acknowledgement for the reviewed copied output.
No CLI, startup hook, connection creation, PostgreSQL lifecycle, or output rebuild.
The caller must retain initialize.lock and ClusterAuthorityLock through this call
AND subsequent positively verified maintenance stop, including every exception or
uncertain commit. It must keep the target isolated with no workers/network clients.
Use an idle, writable, autocommit=True psycopg connection with dict_row and quiet server log
settings. The supplied runtime is the prepared container_runtime module, not its
main()/initialize() entrypoint. Never call this during normal initialized startup.
The immutable PREPARED journal describes intent, not a fabricated completed rename.
An exact before state can be applied; an exact after state is a read-only retry.
Partial journals and later output require review, never automatic cleanup/rewind.
"""
import hashlib
import json
import os
from pathlib import Path
import re
import time
from datetime import datetime, timezone
DATA = Path('/data')
RUN = Path('/run/truf')
APPROVED_MANIFEST_SHA256 = '08344147133c37d4b6f404cf4fac3e59d58f94917f1fa58a77cbb68c36db7e8a'
JOURNAL_NAME = 'found-secrets-loss-g13-g14.prepared.json'
FORMAT = 'truf-found-secrets-loss-g13-g14-v1'
LOSS = 'Previously published copied output intentionally lost; all PostgreSQL history retained. No rotation or rename occurred.'
OLD_GENERATION, NEW_GENERATION, OLD_OFFSET = 13, 14, 97783145
APPEND_COUNT, ROTATION_COUNT = 38024, 13
MAX_JOURNAL = 1024 * 1024
STREAM_COLUMNS = {'stream_name', 'base_relative_path', 'current_generation', 'rotation_bytes',
'max_generations', 'created_at', 'updated_at'}
CURSOR_COLUMNS = {'stream_name', 'generation', 'committed_offset', 'last_append_id', 'last_job_id',
'last_event_id', 'last_event_hash', 'updated_at'}
class ProjectionRecoveryError(RuntimeError):
"""Safe diagnostic only; caller still owns maintenance/stop authority."""
def _encoded(value):
return (json.dumps(value, ensure_ascii=True, sort_keys=True,
separators=(',', ':'), allow_nan=False) + '\n').encode('ascii')
def recover_found_secrets_projection(runtime, connection, *, system_identifier, manifest_sha256,
initialize_lock, authority_lock):
"""Apply only the pinned g13 loss transition, or recognize its exact retry.
Caller-owned locks must be acquired runtime_security lock objects for the fixed
target. This function never closes the connection or releases those locks.
Its own file lock and transaction-scoped advisory/table locks exclude writers.
Returned 'committed'/'already-committed' is not target readiness or stop proof.
journal_sha256 hashes the complete immutable file, not just its inner record.
"""
stage = 'preflight'
deadline = time.monotonic() + 10800
try:
from container_import import _fsync_dir, _identifier, _input, _integer, _json, _manifest, _regular, _write, MAX_MANIFEST
from runtime_security import ClusterAuthorityLock, PrivateFileLock
runtime.require_container()
if (runtime.DATA != DATA or runtime.RUN != RUN
or runtime.INITIALIZED != DATA / 'initialized.json'
or runtime.INITIALIZE_LOCK != DATA / 'initialize.lock'
or manifest_sha256 != APPROVED_MANIFEST_SHA256
or not isinstance(system_identifier, str)
or re.fullmatch(r'[1-9][0-9]{0,19}', system_identifier) is None
or connection.closed or connection.broken or connection.autocommit is not True
or int(connection.info.transaction_status) != 0):
raise ValueError()
config_dir, results = DATA / 'config', DATA / 'runtime-linux/results'
identity_path = DATA / 'runtime-linux/postgres/cluster_identity.json'
manifest_path = config_dir / 'windows-import-manifest.json'
journal_path = config_dir / JOURNAL_NAME
projector_lock = results / '.jsonl-projector.lock'
endpoint = _encoded({'database': 'truf', 'host': '127.0.0.1', 'port': 5432,
'schema': 'public'}).decode('ascii').strip()
data_hash = hashlib.sha256(str(DATA / 'postgres-linux').encode('utf-8')).hexdigest()
endpoint_hash = hashlib.sha256(endpoint.encode('ascii')).hexdigest()
def files_stopped():
if (time.monotonic() >= deadline or getattr(runtime, '_shutdown_requested', True) is not False
or not isinstance(initialize_lock, PrivateFileLock) or initialize_lock.acquired is not True
or initialize_lock.path != os.path.normcase(str(runtime.INITIALIZE_LOCK))
or not isinstance(authority_lock, ClusterAuthorityLock) or authority_lock.acquired is not True
or authority_lock.data_directory != str(DATA / 'postgres-linux')
or authority_lock.endpoint_identity != endpoint
or authority_lock.path != str(DATA / 'runtime-linux/postgres' / f'.cluster-authority-{data_hash}.lock')
or authority_lock.endpoint_path != str(RUN / 'authority' / f'endpoint-{endpoint_hash}.lock')):
raise ValueError()
for directory in (DATA, config_dir, results, identity_path.parent,
DATA / 'runtime-linux/logs', RUN, RUN / 'control'):
runtime.private_path(directory, directory=True)
_regular(runtime.INITIALIZE_LOCK, runtime)
for path in (runtime.INITIALIZED, RUN / 'control/supervisor.instance.json',
RUN / 'control/supervisor.pid', DATA / 'runtime-linux/logs/supervisor.instance.json',
DATA / 'runtime-linux/logs/supervisor.pid'):
try:
path.lstat()
except FileNotFoundError:
continue
raise ValueError()
with os.scandir(results) as entries:
for index, entry in enumerate(entries):
if index >= 100000 or entry.name.casefold() == 'found_secrets' or entry.name.casefold().startswith('found_secrets.'):
raise ValueError()
def read_private(path, limit):
with _input(path, runtime) as (handle, before):
if not 0 < before[4] <= limit:
raise ValueError()
raw = handle.read(limit + 1)
if len(raw) != before[4]:
raise ValueError()
return raw, _json(raw)
files_stopped()
identity_raw, identity = read_private(identity_path, MAX_JOURNAL)
manifest_raw, _ = read_private(manifest_path, MAX_MANIFEST)
manifest, _ = _manifest(manifest_raw, manifest_sha256)
expected_identity = {'pg_major': 16, 'system_identifier': system_identifier,
'data_directory': str(DATA / 'postgres-linux'), 'database': 'truf',
'user': 'truf', 'port': 5432}
if (not isinstance(identity, dict) or any(identity.get(k) != v for k, v in expected_identity.items())
or manifest['database']['system_identifier'] == system_identifier
or 'sequence_states' not in manifest['database']):
raise ValueError()
binding = {'system_identifier': system_identifier, 'manifest_sha256': manifest_sha256,
'identity_sha256': hashlib.sha256(identity_raw).hexdigest()}
def online():
if time.monotonic() >= deadline or runtime._shutdown_requested:
raise ValueError()
connection.execute('SELECT pg_catalog.pg_stat_clear_snapshot()')
row = connection.execute("""SELECT pg_catalog.current_database() AS database,
current_user AS user_name, pg_catalog.current_setting('data_directory') AS data_directory,
pg_catalog.current_setting('port')::int AS port,
pg_catalog.current_setting('server_version_num')::int AS version_num,
pg_catalog.pg_is_in_recovery() AS in_recovery,
(SELECT system_identifier::text FROM pg_catalog.pg_control_system()) AS system_identifier,
(SELECT rolsuper FROM pg_catalog.pg_roles WHERE rolname = current_user) AS superuser,
(SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE backend_type = 'client backend'
AND pid <> pg_catalog.pg_backend_pid()) AS other_clients,
pg_catalog.current_schema() AS schema_name,
pg_catalog.current_setting('search_path') AS search_path,
pg_catalog.current_setting('transaction_read_only') AS read_only,
pg_catalog.current_setting('fsync') AS fsync,
pg_catalog.current_setting('full_page_writes') AS full_page_writes,
EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n CROSS JOIN LATERAL pg_catalog.aclexplode(
COALESCE(n.nspacl, pg_catalog.acldefault('n', n.nspowner))) acl
WHERE n.nspname = 'public' AND acl.grantee = 0 AND acl.privilege_type = 'CREATE') AS public_create""").fetchone()
expected = {'database': 'truf', 'user_name': 'truf', 'data_directory': str(DATA / 'postgres-linux'),
'port': 5432, 'system_identifier': system_identifier, 'in_recovery': False,
'superuser': True, 'other_clients': 0, 'schema_name': 'public',
'search_path': 'public', 'read_only': 'off', 'public_create': False,
'fsync': 'on', 'full_page_writes': 'on'}
if (not isinstance(row, dict) or any(row.get(k) != v for k, v in expected.items())
or _integer(row.get('version_num')) // 10000 != 16):
raise ValueError()
def metadata():
rows = connection.execute("""SELECT s.stream_name, c.stream_name AS cursor_stream_name,
pg_catalog.row_to_json(s) AS stream, pg_catalog.row_to_json(c) AS cursor
FROM public.projection_streams s FULL JOIN public.projection_cursors c
ON c.stream_name = s.stream_name""").fetchall()
seen, found = set(), None
scans = {'scan_results': 'scan_results.jsonl', 'found_secrets': 'found_secrets.jsonl',
'scan_errors': 'scan_errors.log'}
for row in rows:
name, stream, cursor = row['stream_name'], row['stream'], row['cursor']
if (not isinstance(name, str) or name in seen or row['cursor_stream_name'] != name
or not isinstance(stream, dict) or set(stream) != STREAM_COLUMNS
or not isinstance(cursor, dict) or set(cursor) != CURSOR_COLUMNS
or stream['stream_name'] != name or cursor['stream_name'] != name
or _integer(stream['current_generation']) != _integer(cursor['generation'])):
raise ValueError()
seen.add(name)
_integer(cursor['committed_offset'])
_integer(stream['rotation_bytes'], 1)
_integer(stream['max_generations'])
expected_path = scans.get(name)
if expected_path is None:
match = re.fullmatch(r'keycheck:([a-z0-9][a-z0-9_.-]{0,63}):(results|status)', name)
if not match:
raise ValueError()
suffix = 'Results.jsonl' if match[2] == 'results' else 'Checked.txt'
expected_path = f'{match[1]}/{match[1]}{suffix}'
if stream['base_relative_path'] != expected_path:
raise ValueError()
if name == 'found_secrets':
found = {'stream': stream, 'cursor': cursor}
if len(seen) != 34 or not scans.keys() <= seen:
raise ValueError()
return found
tables = sorted(manifest['database']['table_counts'])
sequences = manifest['database']['sequence_states']
if set(sequences) != {'public'}:
raise ValueError()
def preserved():
proof = {}
for table in tables:
if time.monotonic() >= deadline or runtime._shutdown_requested:
raise ValueError()
where = " WHERE t.stream_name <> 'found_secrets'" if table in ('projection_streams', 'projection_cursors') else ''
digest, count = hashlib.sha256(), 0
with connection.cursor(name='found_loss_digest') as cursor:
cursor.itersize = 1000
cursor.execute("SELECT pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to("
"pg_catalog.row_to_json(t)::text, 'UTF8')), 'hex') COLLATE \"C\" AS digest FROM public."
+ _identifier(table) + ' AS t' + where + ' ORDER BY digest')
for row in cursor:
value = row['digest']
if not isinstance(value, str) or re.fullmatch(r'[0-9a-f]{64}', value) is None:
raise ValueError()
digest.update(value.encode('ascii'))
count += 1
if count % 1000 == 0 and (time.monotonic() >= deadline or runtime._shutdown_requested):
raise ValueError()
if count != manifest['database']['table_counts'][table] - int(bool(where)):
raise ValueError()
proof[table] = {'rows': count, 'sha256': digest.hexdigest()}
rows = connection.execute("""SELECT c.relname AS name FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relkind = 'S' ORDER BY c.relname""").fetchall()
if {row['name'] for row in rows} != set(sequences['public']):
raise ValueError()
for row in rows:
state = connection.execute('SELECT last_value, is_called FROM public.' + _identifier(row['name'])).fetchone()
if _encoded(state) != _encoded(sequences['public'][row['name']]):
raise ValueError()
return {'tables': proof, 'sequences': {'count': len(rows),
'sha256': hashlib.sha256(_encoded(sequences)).hexdigest()}}
with PrivateFileLock(str(projector_lock)):
_regular(projector_lock, runtime)
with connection.transaction():
stage = 'database-fences'
connection.execute('SET TRANSACTION ISOLATION LEVEL READ COMMITTED')
online()
connection.execute("SET LOCAL lock_timeout = '5s'")
connection.execute("SET LOCAL statement_timeout = '10800s'")
connection.execute("SET LOCAL temp_file_limit = '4GB'")
connection.execute("SET LOCAL work_mem = '128MB'")
connection.execute('SET LOCAL max_parallel_workers_per_gather = 0')
connection.execute("SET LOCAL row_security = off")
connection.execute('SET LOCAL synchronous_commit = on')
for setting, value in (('log_min_error_statement', 'panic'), ('log_min_messages', 'panic'),
('log_statement', 'none'), ('log_min_duration_statement', '-1')):
connection.execute('SET LOCAL ' + setting + " = '" + value + "'")
for arguments in ((1414681926, 1785753445), (1414681926, 1768842867), (781273968142991337,)):
lock = connection.execute('SELECT pg_catalog.pg_try_advisory_xact_lock('
+ ','.join(['%s'] * len(arguments)) + ') AS locked', arguments).fetchone()
if not lock or lock['locked'] is not True:
raise ValueError()
catalog_sql = """SELECT c.relname AS name, c.relkind AS kind FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relkind IN ('r','p','f') ORDER BY c.relname"""
catalog = connection.execute(catalog_sql).fetchall()
if len(catalog) != len(tables) or {row['name'] for row in catalog} != set(tables) or any(row['kind'] != 'r' for row in catalog):
raise ValueError()
connection.execute('LOCK TABLE ' + ','.join('public.' + _identifier(table) for table in tables)
+ ' IN EXCLUSIVE MODE NOWAIT')
online()
stage = 'history-gates'
gates = connection.execute("""WITH a AS (
SELECT count(*) AS appends, max(a.generation) AS append_highwater,
count(*) FILTER (WHERE a.state IS DISTINCT FROM 'appended' OR j.id IS NULL
OR j.status IS DISTINCT FROM 'completed' OR j.capacity_released IS DISTINCT FROM 1
OR j.job_kind IS DISTINCT FROM 'scan_event' OR (j.required_stream_mask & 2) IS DISTINCT FROM 2
OR a.event_id IS DISTINCT FROM j.event_id OR a.event_hash IS DISTINCT FROM j.event_hash
OR a.generation IS NULL OR a.generation NOT BETWEEN 0 AND 13
OR a.byte_offset IS NULL OR a.byte_offset < 0 OR a.byte_length IS NULL OR a.byte_length < 0
OR a.record_count IS NULL OR a.record_count < 0
OR (a.generation = 13 AND a.byte_offset::numeric + a.byte_length::numeric > 97783145)) AS bad_appends
FROM public.projection_appends a LEFT JOIN public.projection_jobs j ON j.id = a.job_id
WHERE a.stream_name = 'found_secrets'), r AS (
SELECT count(*) AS rotations, max(to_generation) AS rotation_highwater,
count(*) FILTER (WHERE state IS DISTINCT FROM 'completed' OR from_generation IS NULL OR from_generation < 0
OR to_generation IS DISTINCT FROM from_generation + 1 OR to_generation > 13
OR source_bytes IS NULL OR source_bytes < 0
OR segment_relative_path IS DISTINCT FROM
'found_secrets.g' || lpad(from_generation::text, 6, '0') || '.jsonl') AS bad_rotations
FROM public.projection_rotations WHERE stream_name = 'found_secrets')
SELECT a.*, r.*, (SELECT count(*) FROM public.projection_append_audit) AS audits,
(SELECT count(*) FROM public.projection_jobs WHERE (required_stream_mask & 2) <> 0
AND status <> 'completed') AS pending,
(SELECT count(*) FROM public.result_reservations
WHERE state IN ('scanning','ready','ingesting','db_committed')) AS reservations,
(SELECT count(*) FROM public.target_queue q LEFT JOIN public.result_reservations r
ON r.id = q.current_result_reservation_id WHERE q.status = 'in_progress'
OR r.state IN ('scanning','ready','ingesting','db_committed')) AS queue_leases,
(SELECT count(*) FROM public.docker_content_blobs
WHERE state IN ('leased','submitted') OR lease_reservation_id IS NOT NULL) AS blob_leases,
(SELECT count(*) FROM pg_catalog.pg_trigger WHERE NOT tgisinternal
AND tgrelid IN ('public.projection_streams'::regclass,'public.projection_cursors'::regclass)) AS triggers,
(SELECT count(*) FROM pg_catalog.pg_rewrite
WHERE ev_class IN ('public.projection_streams'::regclass,'public.projection_cursors'::regclass)) AS rules
FROM a CROSS JOIN r""").fetchone()
expected_gates = {'appends': APPEND_COUNT, 'append_highwater': 13, 'bad_appends': 0,
'rotations': ROTATION_COUNT, 'rotation_highwater': 13, 'bad_rotations': 0,
'audits': 0, 'pending': 0, 'reservations': 0, 'queue_leases': 0,
'blob_leases': 0, 'triggers': 0, 'rules': 0}
if _encoded(gates) != _encoded(expected_gates):
raise ValueError()
current = metadata()
proof = preserved()
stage = 'journal'
try:
journal_path.lstat()
except FileNotFoundError:
journal = None
else:
raw, journal = read_private(journal_path, MAX_JOURNAL)
if (not isinstance(journal, dict) or set(journal) != {'record', 'sha256'}
or raw != _encoded(journal)
or journal['sha256'] != hashlib.sha256(_encoded(journal['record'])).hexdigest()):
raise ValueError()
if journal is None:
stamp = datetime.now(timezone.utc).isoformat(timespec='seconds')
before = current
record = {'format': FORMAT, 'state': 'PREPARED', 'loss': LOSS, 'binding': binding,
'prepared_at': stamp, 'before': before, 'preserved': proof,
'digest_algorithm': 'sha256-concatenated-sorted-pg-row-sha256-hex-v1'}
else:
record = journal['record']
if (not isinstance(record, dict) or set(record) != {'format', 'state', 'loss', 'binding',
'prepared_at', 'before', 'after', 'preserved', 'digest_algorithm'}
or record['format'] != FORMAT or record['state'] != 'PREPARED' or record['loss'] != LOSS
or _encoded(record['binding']) != _encoded(binding)
or _encoded(record['preserved']) != _encoded(proof)
or record['digest_algorithm'] != 'sha256-concatenated-sorted-pg-row-sha256-hex-v1'):
raise ValueError()
stamp, before = record['prepared_at'], record['before']
if (not isinstance(stamp, str) or datetime.fromisoformat(stamp).isoformat(timespec='seconds') != stamp
or not stamp.endswith('+00:00') or set(before) != {'stream', 'cursor'}
or set(before['stream']) != STREAM_COLUMNS or set(before['cursor']) != CURSOR_COLUMNS
or before['stream']['stream_name'] != 'found_secrets'
or before['stream']['base_relative_path'] != 'found_secrets.jsonl'
or before['cursor']['stream_name'] != 'found_secrets'
or _integer(before['stream']['current_generation']) != OLD_GENERATION
or _integer(before['cursor']['generation']) != OLD_GENERATION
or _integer(before['cursor']['committed_offset']) != OLD_OFFSET):
raise ValueError()
_integer(before['cursor']['last_append_id'], 1)
_integer(before['cursor']['last_job_id'], 1)
last = connection.execute("""SELECT id, job_id, stream_name, generation, byte_offset,
byte_length, event_id, event_hash, state FROM public.projection_appends WHERE id = %s""",
(before['cursor']['last_append_id'],)).fetchone()
if (not last or last['id'] != before['cursor']['last_append_id']
or last['job_id'] != before['cursor']['last_job_id'] or last['stream_name'] != 'found_secrets'
or last['generation'] != OLD_GENERATION or last['state'] != 'appended'
or _integer(last['byte_offset']) + _integer(last['byte_length']) != OLD_OFFSET
or last['event_id'] != before['cursor']['last_event_id']
or last['event_hash'] != before['cursor']['last_event_hash']
or not isinstance(last['event_id'], str) or not last['event_id']
or re.fullmatch(r'[0-9a-f]{64}', last['event_hash'] or '') is None):
raise ValueError()
after = {key: dict(value) for key, value in before.items()}
after['stream'].update(current_generation=NEW_GENERATION, updated_at=stamp)
after['cursor'].update(generation=NEW_GENERATION, committed_offset=0, last_append_id=None, updated_at=stamp)
if journal is not None and _encoded(record['after']) != _encoded(after):
raise ValueError()
already = _encoded(current) == _encoded(after)
if (not already and _encoded(current) != _encoded(before)) or (already and journal is None):
raise ValueError()
if journal is None:
record['after'] = after
journal = {'record': record, 'sha256': hashlib.sha256(_encoded(record)).hexdigest()}
encoded = _encoded(journal)
if len(encoded) > MAX_JOURNAL:
raise ValueError()
_write(runtime, journal_path, encoded)
if read_private(journal_path, MAX_JOURNAL)[0] != encoded:
raise ValueError()
# A prior failure may have left complete bytes without a confirmed fsync.
with _input(journal_path, runtime) as (handle, _):
if handle.read(MAX_JOURNAL + 1) != _encoded(journal):
raise ValueError()
os.fsync(handle.fileno())
_fsync_dir(config_dir)
if not already:
stage = 'compare-and-swap'
result = connection.execute("""UPDATE public.projection_streams AS s
SET current_generation = 14, updated_at = %s
WHERE s.stream_name = 'found_secrets' AND pg_catalog.to_jsonb(s) = %s::jsonb""",
(stamp, _encoded(before['stream']).decode('ascii')))
if result.rowcount != 1:
raise ValueError()
result = connection.execute("""UPDATE public.projection_cursors AS c
SET generation = 14, committed_offset = 0, last_append_id = NULL, updated_at = %s
WHERE c.stream_name = 'found_secrets' AND pg_catalog.to_jsonb(c) = %s::jsonb""",
(stamp, _encoded(before['cursor']).decode('ascii')))
if result.rowcount != 1:
raise ValueError()
if _encoded(metadata()) != _encoded(after) or _encoded(preserved()) != _encoded(proof):
raise ValueError()
stage = 'precommit'
files_stopped()
if (read_private(identity_path, MAX_JOURNAL)[0] != identity_raw
or read_private(manifest_path, MAX_MANIFEST)[0] != manifest_raw
or read_private(journal_path, MAX_JOURNAL)[0] != _encoded(journal)
or _encoded(connection.execute(catalog_sql).fetchall()) != _encoded(catalog)):
raise ValueError()
online()
stage = 'commit'
return {'status': 'already-committed' if already else 'committed', 'journal_path': str(journal_path),
'journal_sha256': hashlib.sha256(_encoded(journal)).hexdigest(), **binding, 'generation': NEW_GENERATION}
except BaseException:
raise ProjectionRecoveryError('Projection recovery refused at ' + stage
+ '; retain caller maintenance authority and verify stop.') from None