sashatrask
1083 lines
52 KiB
Python
1083 lines
52 KiB
Python
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
APP_DIR = ROOT / 'app'
|
|
sys.path.insert(0, str(APP_DIR))
|
|
|
|
import migrate_runtime_safety
|
|
import migrate_observability_db
|
|
import scanner
|
|
from paths import apply_path_config
|
|
from query_policy import QueryPolicyError, validate_rejected_query_policy
|
|
from runtime_security import ensure_private_directory, preflight_lifecycle_paths, private_directory_ready, private_file_ready
|
|
|
|
|
|
class RowCursor:
|
|
def __init__(self, row=None, rows=None):
|
|
self.row = row
|
|
self.rows = list(rows or [])
|
|
|
|
def fetchone(self):
|
|
return self.row
|
|
|
|
def fetchall(self):
|
|
return self.rows
|
|
|
|
|
|
class MigrationAuthorityTests(unittest.TestCase):
|
|
@staticmethod
|
|
def rejected_query_entry(**changes):
|
|
entry = {
|
|
'source': 'dockerhub',
|
|
'query': 'retired',
|
|
'status': 'rejected_zero_alive',
|
|
'evidence_cutoff': '2026-09-07T00:00:00+00:00',
|
|
'successful_scans': 1000,
|
|
'findings': 10,
|
|
'unique_credentials': 2,
|
|
'pending_candidates': 0,
|
|
'ever_alive_credentials': 0,
|
|
'reviewed_queue_rows': 3,
|
|
}
|
|
entry.update(changes)
|
|
return entry
|
|
|
|
def test_hardening_marker_does_not_treat_reused_pid_as_live_owner(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
candidate_dir = os.path.join(work_dir, 'trufflehog-run-fixture')
|
|
ensure_private_directory(work_dir, reject_reparse=True)
|
|
ensure_private_directory(candidate_dir, reject_reparse=True)
|
|
marker = os.path.join(candidate_dir, '.scanner-owner.json')
|
|
Path(marker).write_text(json.dumps({
|
|
'owner_pid': 42,
|
|
'owner_creation_time': 'fixture-old-process',
|
|
'owner_executable': os.path.join(temp_dir, 'trufflehog.exe'),
|
|
}), encoding='utf-8')
|
|
scanner.harden_private_file(marker)
|
|
config = {'global': {'work_dir': work_dir}}
|
|
|
|
with mock.patch(
|
|
'process_identity.exact_process_identity_state', return_value='reused',
|
|
), mock.patch.object(migrate_runtime_safety, '_pid_running', return_value=True):
|
|
self.assertEqual(
|
|
migrate_runtime_safety.known_application_process_markers(config),
|
|
[],
|
|
)
|
|
|
|
with mock.patch(
|
|
'process_identity.exact_process_identity_state', return_value='unknown',
|
|
), mock.patch.object(migrate_runtime_safety, '_pid_running', return_value=True):
|
|
self.assertEqual(
|
|
migrate_runtime_safety.known_application_process_markers(config),
|
|
[(42, marker)],
|
|
)
|
|
|
|
def test_production_legacy_spool_placeholder_resolves_to_reviewed_path(self):
|
|
config_path = APP_DIR / 'config.yaml'
|
|
config = migrate_runtime_safety.load_config(str(config_path))
|
|
expected = os.path.normcase(os.path.abspath(r'D:\truf\runtime\result_spool'))
|
|
self.assertEqual(
|
|
os.path.normcase(config['global']['legacy_result_spool_dir']), expected,
|
|
)
|
|
self.assertEqual(
|
|
os.path.normcase(migrate_runtime_safety.reviewed_legacy_result_spool(config)),
|
|
expected,
|
|
)
|
|
|
|
def test_oversized_legacy_backfill_registers_review_without_fetching_payload(self):
|
|
class Connection:
|
|
is_postgres = True
|
|
|
|
def __init__(self):
|
|
self.candidate_calls = 0
|
|
self.sql = []
|
|
|
|
def execute(self, sql, params=None):
|
|
self.sql.append(sql)
|
|
if 'SELECT s.id, octet_length' in sql:
|
|
self.candidate_calls += 1
|
|
return RowCursor(
|
|
{'id': 77, 'payload_bytes': 300 * 1024 * 1024}
|
|
if self.candidate_calls == 1 else None
|
|
)
|
|
if 'SELECT COUNT(*) AS count FROM target_scans' in sql:
|
|
return RowCursor({'count': 1})
|
|
return RowCursor()
|
|
|
|
@staticmethod
|
|
def commit():
|
|
return None
|
|
|
|
@staticmethod
|
|
def rollback():
|
|
return None
|
|
|
|
connection = Connection()
|
|
report = migrate_runtime_safety.backfill_normalized_raw_results(
|
|
SimpleNamespace(conn=connection), max_rows=10,
|
|
max_bytes=512 * 1024 * 1024, max_object_bytes=192 * 1024 * 1024,
|
|
)
|
|
self.assertEqual(report['reviewed_oversized'], 1)
|
|
self.assertFalse(any('SELECT id, raw_result_json' in sql for sql in connection.sql))
|
|
self.assertTrue(any("'legacy_payload_oversized'" in sql for sql in connection.sql))
|
|
|
|
def test_reconstruction_sql_never_selects_unbounded_target_scan_blob(self):
|
|
source = (APP_DIR / 'scanner_db.py').read_text(encoding='utf-8')
|
|
start = source.index(' def projection_scan_header(')
|
|
end = source.index('\n def iter_projection_findings(', start)
|
|
method = source[start:end]
|
|
self.assertNotIn('SELECT ts.*', method)
|
|
self.assertIn('raw_result_bytes', method)
|
|
self.assertIn("'OCTET_LENGTH(raw_result_json)'", method)
|
|
self.assertIn('AND {payload_size_sql} = ?', method)
|
|
|
|
def setUp(self):
|
|
self.cluster_lock = mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'ClusterAuthorityLock',
|
|
return_value=contextlib.nullcontext(),
|
|
)
|
|
self.preflight = mock.patch.object(migrate_runtime_safety, 'preflight_lifecycle_paths')
|
|
self.cluster_lock.start()
|
|
self.preflight.start()
|
|
|
|
def tearDown(self):
|
|
self.preflight.stop()
|
|
self.cluster_lock.stop()
|
|
|
|
def args(self, **changes):
|
|
values = {
|
|
'config': 'config.yaml', 'database_url': None, 'sqlite': None,
|
|
'initialize_base': False, 'todo': None, 'source': None, 'platform': None,
|
|
'query': 'offline', 'max_rows': 10, 'max_bytes': 1024, 'max_seconds': 1,
|
|
'until_complete': False, 'max_batches': 10, 'resolve_issue': [],
|
|
'harden_runtime': False, 'reconcile_jsonl_projections': False,
|
|
'resolve_jsonl_issue': [],
|
|
'resolve_jsonl_issue_manifest': None,
|
|
'resolve_jsonl_conflict_manifest': None,
|
|
'recover_dead_scan_slots': False,
|
|
'repair_docker_timeout_attempts': False,
|
|
'cold_stale_backlog': False,
|
|
'reactivate_cold_backlog': False,
|
|
'policy_manifest': None,
|
|
'policy_source': None,
|
|
'policy_platform': None,
|
|
'policy_query': None,
|
|
'dry_run': False,
|
|
'apply': True, 'sources_stopped': True,
|
|
}
|
|
values.update(changes)
|
|
return SimpleNamespace(**values)
|
|
|
|
def test_target_queue_query_policy_preserves_exact_case(self):
|
|
report = migrate_runtime_safety.configured_target_queue_query_policy({
|
|
'sources': {
|
|
'dockerhub': {'queries': ['openai', 'OPENAI_API_KEY']},
|
|
'github_actions': {'enabled': False, 'queries': ['logs']},
|
|
},
|
|
})
|
|
self.assertEqual(
|
|
report['queries'][('dockerhub', 'docker')],
|
|
('openai', 'OPENAI_API_KEY'),
|
|
)
|
|
self.assertEqual(report['queries'][('github_actions', 'github_actions')], ('logs',))
|
|
self.assertRegex(report['policy_sha256'], r'^[a-f0-9]{64}$')
|
|
|
|
def test_rejected_query_policy_is_exact_and_source_specific(self):
|
|
config = {
|
|
'sources': {
|
|
'dockerhub': {'queries': ['kept']},
|
|
'npm': {'queries': ['retired']},
|
|
},
|
|
'query_policy': {
|
|
'rejected': [self.rejected_query_entry()],
|
|
},
|
|
}
|
|
entries = validate_rejected_query_policy(config)
|
|
self.assertEqual(
|
|
[(entry['source'], entry['query']) for entry in entries],
|
|
[('dockerhub', 'retired')],
|
|
)
|
|
apply_path_config(config)
|
|
report = migrate_runtime_safety.configured_target_queue_query_policy(config)
|
|
self.assertEqual(report['rejected'][('dockerhub', 'docker')], ('retired',))
|
|
self.assertEqual(report['queries'][('npm', 'npm')], ('retired',))
|
|
self.assertNotIn(('npm', 'npm'), report['rejected'])
|
|
self.assertTrue(report['rejected_registry_present'])
|
|
|
|
def test_rejected_query_policy_fails_closed_on_invalid_evidence(self):
|
|
base = {
|
|
'sources': {'dockerhub': {'queries': ['kept']}},
|
|
'query_policy': {'rejected': [self.rejected_query_entry()]},
|
|
}
|
|
invalid_entries = (
|
|
self.rejected_query_entry(successful_scans=999),
|
|
self.rejected_query_entry(pending_candidates=1),
|
|
self.rejected_query_entry(ever_alive_credentials=1),
|
|
self.rejected_query_entry(evidence_cutoff='2026-09-07T00:00:00'),
|
|
{**self.rejected_query_entry(), 'unexpected': 1},
|
|
)
|
|
for entry in invalid_entries:
|
|
with self.subTest(entry=entry):
|
|
config = {**base, 'query_policy': {'rejected': [entry]}}
|
|
with self.assertRaises(QueryPolicyError):
|
|
validate_rejected_query_policy(config)
|
|
|
|
def test_rejected_query_policy_rejects_overlap_duplicate_and_sentinel(self):
|
|
overlap = {
|
|
'sources': {'dockerhub': {'queries': ['retired']}},
|
|
'query_policy': {'rejected': [self.rejected_query_entry()]},
|
|
}
|
|
with self.assertRaisesRegex(QueryPolicyError, 'overlap'):
|
|
validate_rejected_query_policy(overlap)
|
|
|
|
duplicate = {
|
|
'sources': {'dockerhub': {'queries': ['kept']}},
|
|
'query_policy': {
|
|
'rejected': [self.rejected_query_entry(), self.rejected_query_entry()],
|
|
},
|
|
}
|
|
with self.assertRaisesRegex(QueryPolicyError, 'duplicate'):
|
|
validate_rejected_query_policy(duplicate)
|
|
|
|
sentinel = {
|
|
'sources': {'huggingface': {'queries': ['kept']}},
|
|
'query_policy': {'rejected': [self.rejected_query_entry(
|
|
source='huggingface', query='spaces',
|
|
)]},
|
|
}
|
|
with self.assertRaisesRegex(QueryPolicyError, 'sentinel'):
|
|
validate_rejected_query_policy(sentinel)
|
|
|
|
def test_rejected_registry_without_scope_selects_nothing(self):
|
|
db = mock.Mock()
|
|
policy = {
|
|
'queries': {('npm', 'npm'): ('kept',)},
|
|
'rejected': {('dockerhub', 'docker'): ('retired',)},
|
|
'rejected_registry_present': True,
|
|
}
|
|
self.assertEqual(
|
|
migrate_runtime_safety._stale_target_queue_rows(
|
|
db, policy, 'npm', 'npm', 10,
|
|
),
|
|
([], []),
|
|
)
|
|
db.conn.execute.assert_not_called()
|
|
|
|
def test_main_requires_postgres_dsn_unless_sqlite_is_explicit(self):
|
|
config = {'global': {'database_path': r'D:\scanner_active.db'}, 'supervisor': {}}
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=self.args()), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment', return_value=None), \
|
|
mock.patch.dict(os.environ, {'SCANNER_DB_URL': '', 'DATABASE_URL': ''}):
|
|
with self.assertRaisesRegex(SystemExit, 'PostgreSQL DSN is required'):
|
|
migrate_runtime_safety.main()
|
|
|
|
def test_migration_preflight_uses_server_authority_without_trufflehog(self):
|
|
config = {'global': {}, 'supervisor': {}}
|
|
args = self.args()
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=args), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment'), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety, 'database_url_from_env',
|
|
return_value='postgresql://truf:fixture@127.0.0.1:5432/truf',
|
|
), mock.patch.object(
|
|
migrate_runtime_safety, 'preflight_lifecycle_paths',
|
|
side_effect=OSError('fixture stop'),
|
|
) as preflight:
|
|
with self.assertRaisesRegex(OSError, 'fixture stop'):
|
|
migrate_runtime_safety.main()
|
|
preflight.assert_called_once_with(
|
|
os.path.abspath('config.yaml'), config, authority_profile='server',
|
|
)
|
|
|
|
def test_dead_scan_slot_recovery_deletes_only_definitively_dead_identities(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
state_dir = os.path.join(temp_dir, 'state')
|
|
ensure_private_directory(state_dir, reject_reparse=True)
|
|
path = os.path.join(state_dir, 'scan_limiter.db')
|
|
connection = sqlite3.connect(path)
|
|
connection.executescript(scanner.SCAN_SLOT_SCHEMA)
|
|
rows = (
|
|
('dead', 101, 201),
|
|
('reused', 102, 202),
|
|
('live-owner', 103, 203),
|
|
('live-child', 104, 204),
|
|
('uncertain', 105, 205),
|
|
('failure', 106, 206),
|
|
)
|
|
for index, (slot_id, owner_pid, child_pid) in enumerate(rows):
|
|
connection.execute(
|
|
'''INSERT INTO scan_slots (
|
|
slot_id, owner_pid, owner_thread, owner_source,
|
|
owner_creation_time, owner_executable, child_pid,
|
|
child_creation_time, child_executable, command,
|
|
acquired_at, updated_at
|
|
) VALUES (?, ?, ?, 'fixture', ?, 'owner.exe', ?, ?, 'child.exe', '', ?, ?)''',
|
|
(
|
|
slot_id, owner_pid, index + 1, f'owner-{slot_id}',
|
|
child_pid, f'child-{slot_id}', 100.0 + index, 200.0 + index,
|
|
),
|
|
)
|
|
connection.commit()
|
|
connection.close()
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
|
|
def identity_live(pid, creation_time, executable):
|
|
if pid in (101, 201, 102, 202):
|
|
return False
|
|
if pid == 103:
|
|
return True
|
|
if pid == 204:
|
|
return True
|
|
if pid == 105:
|
|
return None
|
|
if pid == 106:
|
|
raise OSError('identity lookup denied')
|
|
return False
|
|
|
|
report = migrate_runtime_safety.recover_dead_scan_slots(
|
|
{'global': {'state_dir': state_dir, 'scan_limiter_db': path}},
|
|
identity_live=identity_live,
|
|
now=1000.0,
|
|
)
|
|
self.assertEqual(report['deleted'], 2)
|
|
self.assertEqual(report['remaining'], 4)
|
|
deleted = {row['slot_id'] for row in report['rows'] if row['deleted']}
|
|
self.assertEqual(deleted, {'dead', 'reused'})
|
|
retained = {row['slot_id'] for row in report['rows'] if not row['deleted']}
|
|
self.assertEqual(retained, {'live-owner', 'live-child', 'uncertain', 'failure'})
|
|
with contextlib.closing(sqlite3.connect(path)) as verify:
|
|
remaining = {row[0] for row in verify.execute('SELECT slot_id FROM scan_slots')}
|
|
self.assertEqual(remaining, retained)
|
|
|
|
def test_reused_pid_with_different_creation_identity_is_dead(self):
|
|
identity = SimpleNamespace(creation_time='new-creation', executable=r'C:\fixture\owner.exe')
|
|
process = mock.Mock(identity=identity)
|
|
process.is_running.return_value = True
|
|
with mock.patch.object(scanner, 'open_process', return_value=process):
|
|
self.assertFalse(scanner.exact_process_identity_live(
|
|
1234, 'old-creation', r'C:\fixture\owner.exe',
|
|
))
|
|
process.close.assert_called_once_with()
|
|
|
|
def test_recover_dead_scan_slots_cli_uses_shared_authority_without_postgres_stop(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
state_dir = os.path.join(temp_dir, 'state')
|
|
ensure_private_directory(state_dir, reject_reparse=True)
|
|
path = os.path.join(state_dir, 'scan_limiter.db')
|
|
connection = sqlite3.connect(path)
|
|
connection.executescript(scanner.SCAN_SLOT_SCHEMA)
|
|
connection.execute(
|
|
'''INSERT INTO scan_slots (
|
|
slot_id, owner_pid, owner_thread, owner_source,
|
|
owner_creation_time, owner_executable, child_pid,
|
|
child_creation_time, child_executable, command,
|
|
acquired_at, updated_at
|
|
) VALUES ('dead', 99999991, 1, 'fixture', 'old-owner', 'owner.exe',
|
|
99999992, 'old-child', 'child.exe', '', 1, 2)'''
|
|
)
|
|
connection.commit()
|
|
connection.close()
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
config = {
|
|
'global': {
|
|
'root_dir': temp_dir,
|
|
'state_dir': state_dir,
|
|
'scan_limiter_db': path,
|
|
},
|
|
'supervisor': {},
|
|
}
|
|
args = self.args(recover_dead_scan_slots=True)
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=args), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment'), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'database_url_from_env',
|
|
return_value='postgresql://truf:fixture@127.0.0.1:5432/truf',
|
|
), \
|
|
mock.patch.object(migrate_runtime_safety, 'require_local_sources_stopped') as stopped:
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
stopped.assert_called_once_with(config, inspect_scan_slots=False)
|
|
with contextlib.closing(sqlite3.connect(path)) as verify:
|
|
self.assertEqual(verify.execute('SELECT COUNT(*) FROM scan_slots').fetchone()[0], 0)
|
|
|
|
def test_sqlite_one_batch_is_nonzero_until_bounded_reconciliation_completes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
runtime = os.path.join(temp_dir, 'runtime')
|
|
config = {
|
|
'global': {
|
|
'root_dir': temp_dir,
|
|
'runtime_dir': runtime,
|
|
'state_dir': os.path.join(runtime, 'state'),
|
|
'log_dir': os.path.join(runtime, 'logs'),
|
|
'control_dir': os.path.join(runtime, 'control'),
|
|
},
|
|
'supervisor': {
|
|
'instance_file': os.path.join(runtime, 'control', 'supervisor.instance.json'),
|
|
'log_dir': os.path.join(runtime, 'logs'),
|
|
},
|
|
}
|
|
todo = os.path.join(temp_dir, 'todo.txt')
|
|
Path(todo).write_text(
|
|
'https://github.com/example/one\nhttps://github.com/example/two\n',
|
|
encoding='ascii',
|
|
)
|
|
db_path = os.path.join(temp_dir, 'migration.sqlite')
|
|
one_batch = self.args(
|
|
sqlite=db_path, initialize_base=True, todo=todo,
|
|
source='source', platform='github', max_rows=1,
|
|
)
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=one_batch), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config):
|
|
self.assertEqual(migrate_runtime_safety.main(), 2)
|
|
until_complete = self.args(
|
|
sqlite=db_path, todo=todo, source='source', platform='github',
|
|
max_rows=1, until_complete=True, max_batches=10,
|
|
)
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=until_complete), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config):
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
|
|
def test_online_identity_checks_every_bound_cluster_field(self):
|
|
identity = {
|
|
'database': 'truf', 'user': 'truf', 'port': 5432,
|
|
'data_directory': r'D:\bound-data', 'system_identifier': '12345',
|
|
}
|
|
row = {
|
|
'database': 'truf', 'user_name': 'truf', 'port': 5432,
|
|
'data_directory': r'D:\bound-data', 'system_identifier': '12345',
|
|
}
|
|
|
|
class Connection:
|
|
def execute(self, sql, params=None):
|
|
return RowCursor(row=row)
|
|
|
|
def commit(self):
|
|
pass
|
|
|
|
db = SimpleNamespace(conn=Connection(), url=None)
|
|
dsn = 'postgresql://truf:secret@127.0.0.1:5432/truf'
|
|
with mock.patch.object(migrate_runtime_safety, 'verify_cluster_identity', return_value=identity):
|
|
found, canonical = migrate_runtime_safety._online_postgres_identity(db, dsn, {})
|
|
self.assertEqual(found, identity)
|
|
self.assertEqual(canonical, dsn)
|
|
|
|
bad = dict(row, system_identifier='99999')
|
|
db.conn.execute = lambda sql, params=None: RowCursor(row=bad)
|
|
with mock.patch.object(migrate_runtime_safety, 'verify_cluster_identity', return_value=identity):
|
|
with self.assertRaisesRegex(RuntimeError, 'system_identifier'):
|
|
migrate_runtime_safety._online_postgres_identity(db, dsn, {})
|
|
|
|
def test_postgres_guard_sets_migration_timeouts_and_holds_advisory_lock(self):
|
|
statements = []
|
|
|
|
class Connection:
|
|
def execute(self, sql, params=None):
|
|
statements.append(sql)
|
|
if 'current_schema()' in sql:
|
|
return RowCursor(row={
|
|
'schema_name': 'public',
|
|
'search_path': 'public',
|
|
'public_create': False,
|
|
})
|
|
if 'pg_stat_activity' in sql:
|
|
return RowCursor(rows=[])
|
|
if 'pg_try_advisory_lock' in sql:
|
|
return RowCursor(row={'locked': True})
|
|
return RowCursor(row={'pg_advisory_unlock': True})
|
|
|
|
def commit(self):
|
|
pass
|
|
|
|
def rollback(self):
|
|
pass
|
|
|
|
with migrate_runtime_safety.postgres_migration_guard(SimpleNamespace(conn=Connection())):
|
|
statements.append('MIGRATION BODY')
|
|
self.assertLess(statements.index('SET statement_timeout = 0'), statements.index('MIGRATION BODY'))
|
|
self.assertTrue(any('lock_timeout' in sql for sql in statements))
|
|
self.assertTrue(any('pg_try_advisory_lock' in sql for sql in statements))
|
|
self.assertTrue(any('pg_advisory_unlock' in sql for sql in statements))
|
|
|
|
def test_redirecting_migration_dsn_is_rejected_before_scanner_db_construction(self):
|
|
config = {'global': {'root_dir': r'D:\bound'}, 'supervisor': {}}
|
|
identity = {'database': 'truf', 'user': 'truf', 'port': 5432}
|
|
attacks = (
|
|
'postgresql://truf:secret@127.0.0.1:5432/truf?host=attacker',
|
|
'postgresql://truf:secret@127.0.0.1:5432,attacker:5432/truf',
|
|
'postgresql://other:secret@127.0.0.1:5432/truf',
|
|
)
|
|
for attack in attacks:
|
|
with self.subTest(attack=attack), \
|
|
mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=self.args(database_url=attack)), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment'), \
|
|
mock.patch.object(migrate_runtime_safety, 'require_local_sources_stopped'), \
|
|
mock.patch.object(migrate_runtime_safety, 'verify_cluster_identity', return_value=identity), \
|
|
mock.patch.object(migrate_runtime_safety, 'ScannerDB') as scanner_db:
|
|
with self.assertRaisesRegex(RuntimeError, 'authority validation'):
|
|
migrate_runtime_safety.main()
|
|
scanner_db.assert_not_called()
|
|
|
|
def test_hardening_action_never_constructs_a_database(self):
|
|
config = {'global': {'root_dir': r'D:\bound'}, 'supervisor': {}}
|
|
with mock.patch.object(
|
|
migrate_runtime_safety, 'parse_args', return_value=self.args(harden_runtime=True)
|
|
), mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'database_url_from_env', return_value='postgresql://truf:fixture@127.0.0.1:5432/truf'), \
|
|
mock.patch.object(migrate_runtime_safety, 'require_runtime_hardening_stopped'), \
|
|
mock.patch.object(migrate_runtime_safety, 'find_postgres_environment_path', return_value=None), \
|
|
mock.patch.object(migrate_runtime_safety, 'harden_runtime_paths', return_value=3), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment') as load_environment, \
|
|
mock.patch.object(migrate_runtime_safety.sqlite3, 'connect') as sqlite_connect, \
|
|
mock.patch.object(migrate_runtime_safety, 'ScannerDB') as scanner_db:
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
load_environment.assert_called_once_with(os.path.abspath('config.yaml'), config)
|
|
sqlite_connect.assert_not_called()
|
|
scanner_db.assert_not_called()
|
|
|
|
def test_hardening_cli_loads_private_configured_postgres_environment_without_exposing_dsn(self):
|
|
secret = 'fixture-secret-not-for-output'
|
|
dsn = f'postgresql://truf:{secret}@127.0.0.1:5432/truf'
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
project_dir = os.path.join(temp_dir, 'app')
|
|
ensure_private_directory(project_dir, reject_reparse=True)
|
|
config_path = os.path.join(project_dir, 'config.yaml')
|
|
env_path = os.path.join(temp_dir, '.env.postgres')
|
|
Path(config_path).write_text(
|
|
f'global:\n root_dir: "{Path(temp_dir).as_posix()}"\n project_dir: "."\n',
|
|
encoding='ascii',
|
|
)
|
|
Path(env_path).write_text(f'DATABASE_URL={dsn}\n', encoding='ascii')
|
|
migrate_runtime_safety.harden_private_file(env_path)
|
|
self.assertTrue(private_file_ready(env_path))
|
|
argv = [
|
|
'migrate_runtime_safety.py', '--config', config_path,
|
|
'--harden-runtime', '--apply', '--sources-stopped',
|
|
]
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
cleared_environment = {
|
|
key: '' for key in (
|
|
'TRUF_MANAGED_POSTGRES_DSN', 'SCANNER_DB_URL', 'DATABASE_URL',
|
|
'TRUF_POSTGRES_PASSWORD', 'TRUF_POSTGRES_DB',
|
|
'TRUF_POSTGRES_USER', 'TRUF_POSTGRES_PORT',
|
|
)
|
|
}
|
|
with mock.patch.object(sys, 'argv', argv), \
|
|
mock.patch.dict(os.environ, cleared_environment), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'ClusterAuthorityLock',
|
|
return_value=contextlib.nullcontext(),
|
|
) as authority_lock, \
|
|
mock.patch.object(migrate_runtime_safety, 'require_runtime_hardening_stopped'), \
|
|
mock.patch.object(migrate_runtime_safety, 'harden_runtime_paths', return_value=3) as harden_paths, \
|
|
contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
|
|
authority_lock.assert_called_once()
|
|
self.assertEqual(authority_lock.call_args.kwargs['endpoint_dsn'], dsn)
|
|
self.assertEqual(harden_paths.call_args.args[1], os.path.abspath(env_path))
|
|
self.assertNotIn(secret, ' '.join(argv))
|
|
self.assertNotIn(secret, stdout.getvalue())
|
|
self.assertNotIn(secret, stderr.getvalue())
|
|
|
|
def test_jsonl_projection_cli_builds_ledgers_without_rewriting_history(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
results_dir = os.path.join(temp_dir, 'results')
|
|
ensure_private_directory(results_dir, reject_reparse=True)
|
|
scan_first = b'{"scan_event_id":"event-1"}\n'
|
|
scan_corrupt = b'1, "legacy":"reviewed"}\n'
|
|
scan_last = b'{"scan_event_id":"event-2"}\n'
|
|
payloads = {
|
|
'scan_results.jsonl': (scan_first + scan_corrupt + scan_last, 'scan_event_id'),
|
|
'found_secrets.jsonl': (b'{"finding_uid":"finding-1"}\n', 'finding_uid'),
|
|
'scan_errors.log': (b'event-1:1\twhen\tgithub\ttarget\terror\n', 'error_row_id'),
|
|
}
|
|
for name, (payload, _) in payloads.items():
|
|
path = os.path.join(results_dir, name)
|
|
Path(path).write_bytes(payload)
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
manifest_path = os.path.join(temp_dir, 'review-manifest.json')
|
|
Path(manifest_path).write_text(json.dumps({
|
|
'schema': 1,
|
|
'type': 'truf-jsonl-projection-review',
|
|
'audit_sha256': hashlib.sha256(b'fixture-audit').hexdigest(),
|
|
'issues': [{
|
|
'ledger_kind': 'scan_results',
|
|
'file': 'scan_results.jsonl',
|
|
'offset': len(scan_first),
|
|
'length': len(scan_corrupt),
|
|
'sha256': hashlib.sha256(scan_corrupt).hexdigest(),
|
|
'classification': 'legacy_numeric_prefix_corrupt_json',
|
|
'safe_identity': False,
|
|
}],
|
|
}), encoding='ascii')
|
|
migrate_runtime_safety.harden_private_file(manifest_path)
|
|
config = {
|
|
'global': {
|
|
'root_dir': temp_dir,
|
|
'runtime_dir': os.path.join(temp_dir, 'runtime'),
|
|
'results_dir': results_dir,
|
|
},
|
|
'supervisor': {},
|
|
}
|
|
args = self.args(
|
|
reconcile_jsonl_projections=True,
|
|
until_complete=True,
|
|
max_rows=1,
|
|
max_bytes=1024,
|
|
max_seconds=1,
|
|
max_batches=10,
|
|
resolve_jsonl_issue_manifest=manifest_path,
|
|
)
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=args), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment'), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'database_url_from_env',
|
|
return_value='postgresql://truf:fixture@127.0.0.1:5432/truf',
|
|
), \
|
|
mock.patch.object(migrate_runtime_safety, 'require_runtime_hardening_stopped'):
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
for name, (payload, identity_key) in payloads.items():
|
|
path = os.path.join(results_dir, name)
|
|
self.assertEqual(Path(path).read_bytes(), payload)
|
|
ledger_path = os.path.splitext(path)[0] + '.publication-ledger.sqlite3'
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
marker = ledger.execute(
|
|
'SELECT value FROM publication_meta WHERE key = ?',
|
|
(f'bootstrapped:{identity_key}',),
|
|
).fetchone()
|
|
self.assertEqual(marker, ('1',))
|
|
self.assertTrue(private_file_ready(ledger_path))
|
|
|
|
def test_v2_projection_cursors_start_at_existing_active_file_sizes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
results_dir = os.path.join(temp_dir, 'results')
|
|
ensure_private_directory(results_dir, reject_reparse=True)
|
|
for name, payload in (
|
|
('scan_results.jsonl', b'{"legacy":1}\n'),
|
|
('found_secrets.jsonl', b'{"legacy":2}\n'),
|
|
('scan_errors.log', b'legacy-error\n'),
|
|
):
|
|
path = os.path.join(results_dir, name)
|
|
Path(path).write_bytes(payload)
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
db = migrate_runtime_safety.ScannerDB(
|
|
db_path=os.path.join(temp_dir, 'scanner.db'), db_url='', initialize=True,
|
|
)
|
|
try:
|
|
report = migrate_runtime_safety.initialize_projection_cursors_from_existing_files(
|
|
db, {'global': {'results_dir': results_dir}},
|
|
)
|
|
self.assertEqual(report, {
|
|
'scan_results': len(b'{"legacy":1}\n'),
|
|
'found_secrets': len(b'{"legacy":2}\n'),
|
|
'scan_errors': len(b'legacy-error\n'),
|
|
})
|
|
repeated = migrate_runtime_safety.initialize_projection_cursors_from_existing_files(
|
|
db, {'global': {'results_dir': results_dir}},
|
|
)
|
|
self.assertEqual(repeated, report)
|
|
finally:
|
|
db.close()
|
|
|
|
def test_docker_timeout_attempt_repair_is_fenced_and_preserves_results(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
db = migrate_runtime_safety.ScannerDB(
|
|
db_path=os.path.join(temp_dir, 'scanner.db'), db_url='', initialize=True,
|
|
)
|
|
now = '2026-09-01T00:00:00+00:00'
|
|
|
|
def add_queue(target, lease=False):
|
|
cursor = db.conn.execute(
|
|
'''INSERT INTO target_queue(
|
|
source, platform, target, normalized_target, status, attempts,
|
|
lease_owner, lease_token, available_after, created_at, updated_at
|
|
) VALUES ('dockerhub', 'docker', ?, ?, 'deferred', 0, ?, ?, ?, ?, ?)''',
|
|
(
|
|
target, target.lower(), 'worker' if lease else None,
|
|
'active-token' if lease else None, now, now, now,
|
|
),
|
|
)
|
|
return cursor.lastrowid
|
|
|
|
def add_scan(queue_id, target, metadata):
|
|
cursor = db.conn.execute(
|
|
'''INSERT INTO target_scans(
|
|
queue_id, source, target, normalized_target, scan_type, status,
|
|
error_count, created_at
|
|
) VALUES (?, 'dockerhub', ?, ?, 'docker', 'error', 1, ?)''',
|
|
(queue_id, target, target.lower(), now),
|
|
)
|
|
scan_id = cursor.lastrowid
|
|
payload = json.dumps(metadata, sort_keys=True, separators=(',', ':'))
|
|
encoded = payload.encode('ascii')
|
|
db.conn.execute(
|
|
'''INSERT INTO scan_result_compat(
|
|
target_scan_id, schema_version, metadata_json, metadata_sha256,
|
|
metadata_bytes, reconstruction_status, created_at
|
|
) VALUES (?, 2, ?, ?, ?, 'exact', ?)''',
|
|
(scan_id, payload, hashlib.sha256(encoded).hexdigest(), len(encoded), now),
|
|
)
|
|
return scan_id
|
|
|
|
exhausted_target = 'owner/exhausted@sha256:' + ('a' * 64)
|
|
ordinary_target = 'owner/ordinary@sha256:' + ('b' * 64)
|
|
active_target = 'owner/active@sha256:' + ('c' * 64)
|
|
exhausted_id = add_queue(exhausted_target)
|
|
ordinary_id = add_queue(ordinary_target)
|
|
active_id = add_queue(active_target, lease=True)
|
|
exhausted_scans = [
|
|
add_scan(exhausted_id, exhausted_target, {
|
|
'error_class': 'timeout', 'scan_meta': {'command_timed_out': True},
|
|
})
|
|
for _ in range(3)
|
|
]
|
|
add_scan(ordinary_id, ordinary_target, {'error_class': 'network'})
|
|
add_scan(active_id, active_target, {
|
|
'error_class': 'timeout', 'scan_meta': {'command_timed_out': True},
|
|
})
|
|
db.conn.execute(
|
|
'''INSERT INTO findings(
|
|
target_scan_id, source, target, detector_name, created_at
|
|
) VALUES (?, 'dockerhub', ?, 'FixtureDetector', ?)''',
|
|
(exhausted_scans[0], exhausted_target, now),
|
|
)
|
|
db.conn.commit()
|
|
try:
|
|
report = migrate_runtime_safety.repair_docker_timeout_attempts(
|
|
db, max_attempts=3, max_rows=100,
|
|
)
|
|
self.assertEqual(report, {
|
|
'examined': 2, 'repaired': 1, 'terminal': 1, 'unchanged': 1,
|
|
})
|
|
exhausted = db.conn.execute(
|
|
'SELECT status, attempts, available_after, last_error FROM target_queue WHERE id = ?',
|
|
(exhausted_id,),
|
|
).fetchone()
|
|
self.assertEqual((exhausted['status'], exhausted['attempts']), ('failed', 3))
|
|
self.assertIsNone(exhausted['available_after'])
|
|
self.assertEqual(
|
|
exhausted['last_error'],
|
|
'Docker timeout attempts exhausted by guarded repair',
|
|
)
|
|
ordinary = db.conn.execute(
|
|
'SELECT status, attempts FROM target_queue WHERE id = ?', (ordinary_id,),
|
|
).fetchone()
|
|
active = db.conn.execute(
|
|
'SELECT status, attempts, lease_token FROM target_queue WHERE id = ?',
|
|
(active_id,),
|
|
).fetchone()
|
|
self.assertEqual((ordinary['status'], ordinary['attempts']), ('deferred', 0))
|
|
self.assertEqual(
|
|
(active['status'], active['attempts'], active['lease_token']),
|
|
('deferred', 0, 'active-token'),
|
|
)
|
|
self.assertEqual(
|
|
db.conn.execute('SELECT COUNT(*) FROM target_scans').fetchone()[0], 5,
|
|
)
|
|
self.assertEqual(
|
|
db.conn.execute('SELECT COUNT(*) FROM scan_result_compat').fetchone()[0], 5,
|
|
)
|
|
self.assertEqual(
|
|
db.conn.execute('SELECT COUNT(*) FROM findings').fetchone()[0], 1,
|
|
)
|
|
finally:
|
|
db.close()
|
|
|
|
def test_jsonl_issue_manifest_rejects_raw_payload_fields(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'review.json')
|
|
Path(path).write_text(json.dumps({
|
|
'schema': 1,
|
|
'type': 'truf-jsonl-projection-review',
|
|
'audit_sha256': 'a' * 64,
|
|
'issues': [{
|
|
'ledger_kind': 'scan_results',
|
|
'file': 'scan_results.000001.jsonl',
|
|
'offset': 1,
|
|
'length': 2,
|
|
'sha256': 'b' * 64,
|
|
'classification': 'invalid_json',
|
|
'safe_identity': False,
|
|
'raw_payload': 'forbidden',
|
|
}],
|
|
}), encoding='ascii')
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
with self.assertRaisesRegex(RuntimeError, 'entry is invalid'):
|
|
migrate_runtime_safety.load_jsonl_issue_review_manifest(path)
|
|
|
|
def test_jsonl_conflict_manifest_registers_exact_historical_variants(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
results_dir = os.path.join(temp_dir, 'results')
|
|
ensure_private_directory(results_dir, reject_reparse=True)
|
|
path = os.path.join(results_dir, 'scan_errors.log')
|
|
identity = '2026-05-23T23:20:51.522121'
|
|
first = f'{identity}\twhen\tgithub\t{{"error":"one"}}\n'.encode()
|
|
second = f'{identity}\twhen\tgithub\t{{"error":"two","code":2}}\n'.encode()
|
|
Path(path).write_bytes(first + second)
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
sys.path.insert(0, str(APP_DIR))
|
|
import scanner
|
|
variants = []
|
|
for offset, row, value in (
|
|
(0, first, {'error': 'one'}),
|
|
(len(first), second, {'error': 'two', 'code': 2}),
|
|
):
|
|
variants.append({
|
|
'ledger_kind': 'scan_errors',
|
|
'file': 'scan_errors.log',
|
|
'offset': offset,
|
|
'length': len(row),
|
|
'identity_sha256': hashlib.sha256(identity.encode()).hexdigest(),
|
|
'payload_sha256': hashlib.sha256(row).hexdigest(),
|
|
'field_name_set_sha256': scanner._projection_field_name_set_sha256(value),
|
|
'classification': 'historical_payload_variant',
|
|
'occurrences': 1,
|
|
})
|
|
manifest_path = os.path.join(temp_dir, 'conflict-review.json')
|
|
Path(manifest_path).write_text(json.dumps({
|
|
'schema': 1,
|
|
'type': 'truf-jsonl-projection-conflict-review',
|
|
'audit_sha256': hashlib.sha256(b'conflict-audit').hexdigest(),
|
|
'variants': variants,
|
|
}), encoding='ascii')
|
|
migrate_runtime_safety.harden_private_file(manifest_path)
|
|
config = {
|
|
'global': {
|
|
'root_dir': temp_dir,
|
|
'runtime_dir': os.path.join(temp_dir, 'runtime'),
|
|
'results_dir': results_dir,
|
|
},
|
|
'supervisor': {},
|
|
}
|
|
args = self.args(
|
|
reconcile_jsonl_projections=True,
|
|
resolve_jsonl_conflict_manifest=manifest_path,
|
|
until_complete=True,
|
|
max_rows=10,
|
|
max_bytes=1024 * 1024,
|
|
max_seconds=1,
|
|
max_batches=10,
|
|
)
|
|
with mock.patch.object(migrate_runtime_safety, 'parse_args', return_value=args), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_config', return_value=config), \
|
|
mock.patch.object(migrate_runtime_safety, 'load_postgres_environment'), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'database_url_from_env',
|
|
return_value='postgresql://truf:fixture@127.0.0.1:5432/truf',
|
|
), \
|
|
mock.patch.object(migrate_runtime_safety, 'require_runtime_hardening_stopped'):
|
|
self.assertEqual(migrate_runtime_safety.main(), 0)
|
|
ledger_path = os.path.splitext(path)[0] + '.publication-ledger.sqlite3'
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
count = ledger.execute(
|
|
'''SELECT COUNT(*) FROM publication_identity_variant
|
|
WHERE identity_key = 'error_row_id' AND identity_value = ?''',
|
|
(identity,),
|
|
).fetchone()[0]
|
|
self.assertEqual(count, 2)
|
|
|
|
def test_jsonl_conflict_manifest_rejects_raw_payload_fields(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'conflicts.json')
|
|
Path(path).write_text(json.dumps({
|
|
'schema': 1,
|
|
'type': 'truf-jsonl-projection-conflict-review',
|
|
'audit_sha256': 'a' * 64,
|
|
'variants': [{
|
|
'ledger_kind': 'scan_errors',
|
|
'file': 'scan_errors.000001.log',
|
|
'offset': 1,
|
|
'length': 2,
|
|
'identity_sha256': 'b' * 64,
|
|
'payload_sha256': 'c' * 64,
|
|
'field_name_set_sha256': 'd' * 64,
|
|
'classification': 'historical_payload_variant',
|
|
'occurrences': 1,
|
|
'raw_payload': 'forbidden',
|
|
}],
|
|
}), encoding='ascii')
|
|
migrate_runtime_safety.harden_private_file(path)
|
|
with self.assertRaisesRegex(RuntimeError, 'entry is invalid'):
|
|
migrate_runtime_safety.load_jsonl_conflict_review_manifest(path)
|
|
|
|
@unittest.skipUnless(os.name == 'nt', 'legacy Windows private-native hardening policy')
|
|
def test_offline_hardening_creates_full_layout_and_hardens_required_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
ensure_private_directory(temp_dir, reject_reparse=True)
|
|
project = os.path.join(temp_dir, 'app')
|
|
os.makedirs(project)
|
|
config_path = os.path.join(project, 'config.yaml')
|
|
secrets_path = os.path.join(project, 'secrets.yaml')
|
|
env_path = os.path.join(temp_dir, '.env.postgres')
|
|
provider_dir = os.path.join(project, 'keycheckers', 'fixture')
|
|
os.makedirs(provider_dir)
|
|
provider_path = os.path.join(provider_dir, 'fixtureKeycheck.py')
|
|
runtime = os.path.join(temp_dir, 'runtime')
|
|
os.makedirs(runtime)
|
|
git_dir = os.path.join(runtime, 'git', 'cmd')
|
|
os.makedirs(git_dir)
|
|
git_path = os.path.join(git_dir, 'git.exe')
|
|
external_postgres_data = os.path.join(temp_dir, 'external-postgres-data')
|
|
openrouter_path = os.path.join(runtime, 'check-openrouter-keys.ps1')
|
|
start_launcher = os.path.join(temp_dir, 'start_runtime.ps1')
|
|
stop_launcher = os.path.join(temp_dir, 'stop_runtime.ps1')
|
|
trufflehog_path = os.path.join(temp_dir, 'trufflehog.exe')
|
|
for path, content in (
|
|
(config_path, 'global: {}\n'),
|
|
(secrets_path, 'auth_pools: {}\n'),
|
|
(env_path, 'TRUF_POSTGRES_PASSWORD=fixture\n'),
|
|
(provider_path, 'pass\n'),
|
|
(openrouter_path, '# fixture\n'),
|
|
(start_launcher, '# start fixture\n'),
|
|
(stop_launcher, '# stop fixture\n'),
|
|
(trufflehog_path, 'fixture executable\n'),
|
|
(git_path, 'fixture Git executable\n'),
|
|
):
|
|
Path(path).write_text(content, encoding='ascii')
|
|
global_config = {
|
|
'root_dir': temp_dir,
|
|
'project_dir': project,
|
|
'runtime_dir': runtime,
|
|
'results_dir': os.path.join(runtime, 'results'),
|
|
'result_spool_dir': os.path.join(runtime, 'result_spool'),
|
|
'control_dir': os.path.join(runtime, 'control'),
|
|
'queue_dir': os.path.join(runtime, 'queues'),
|
|
'state_dir': os.path.join(runtime, 'state'),
|
|
'keycheck_dir': os.path.join(runtime, 'keychecks'),
|
|
'postman_cache_dir': os.path.join(runtime, 'postman_cache'),
|
|
'gharchive_cache_dir': os.path.join(runtime, 'state', 'gharchive_cache'),
|
|
'log_dir': os.path.join(runtime, 'logs'),
|
|
'work_dir': os.path.join(temp_dir, 'work'),
|
|
'postgres_data_dir': external_postgres_data,
|
|
'secrets_file': secrets_path,
|
|
'trufflehog_path': trufflehog_path,
|
|
}
|
|
config = {
|
|
'global': global_config,
|
|
'supervisor': {
|
|
'control_dir': global_config['control_dir'],
|
|
'log_dir': global_config['log_dir'],
|
|
'state_dir': global_config['state_dir'],
|
|
},
|
|
}
|
|
with mock.patch.object(migrate_runtime_safety, 'harden_private_tree', return_value=0), \
|
|
mock.patch.object(
|
|
migrate_runtime_safety,
|
|
'harden_private_file',
|
|
wraps=migrate_runtime_safety.harden_private_file,
|
|
) as harden_file:
|
|
migrate_runtime_safety.harden_runtime_paths(
|
|
config,
|
|
env_path=env_path,
|
|
config_path=config_path,
|
|
)
|
|
hardened_files = {
|
|
os.path.normcase(os.path.realpath(os.path.abspath(call.args[0])))
|
|
for call in harden_file.call_args_list
|
|
}
|
|
self.assertTrue({
|
|
os.path.normcase(os.path.realpath(start_launcher)),
|
|
os.path.normcase(os.path.realpath(stop_launcher)),
|
|
os.path.normcase(os.path.realpath(git_path)),
|
|
}.issubset(hardened_files))
|
|
required_dirs = [
|
|
global_config[key]
|
|
for key in (
|
|
'root_dir', 'project_dir', 'runtime_dir', 'results_dir',
|
|
'result_spool_dir', 'control_dir', 'queue_dir', 'state_dir',
|
|
'keycheck_dir', 'postman_cache_dir', 'gharchive_cache_dir',
|
|
'log_dir', 'work_dir',
|
|
)
|
|
]
|
|
required_dirs.append(os.path.join(runtime, 'postgres'))
|
|
required_dirs.append(external_postgres_data)
|
|
self.assertTrue(all(private_directory_ready(path) for path in required_dirs))
|
|
self.assertTrue(all(private_file_ready(path) for path in (
|
|
config_path, secrets_path, env_path, provider_path, openrouter_path,
|
|
start_launcher, stop_launcher, trufflehog_path, git_path,
|
|
)))
|
|
self.assertTrue(preflight_lifecycle_paths(config_path, config))
|
|
|
|
def test_legacy_postgres_import_tool_is_retired_before_opening_any_database(self):
|
|
with mock.patch.object(migrate_observability_db, 'sqlite_connect_ro') as sqlite_connect, \
|
|
mock.patch.object(migrate_observability_db, 'ScannerDB') as scanner_db:
|
|
with self.assertRaisesRegex(SystemExit, 'retired'):
|
|
migrate_observability_db.main()
|
|
sqlite_connect.assert_not_called()
|
|
scanner_db.assert_not_called()
|
|
|
|
def test_active_database_session_or_supervisor_metadata_refuses_migration(self):
|
|
class Connection:
|
|
def execute(self, sql, params=None):
|
|
return RowCursor(rows=[{'pid': 9, 'application_name': 'scanner', 'state': 'idle'}])
|
|
|
|
def commit(self):
|
|
pass
|
|
|
|
with self.assertRaisesRegex(RuntimeError, 'application session'):
|
|
migrate_runtime_safety._require_no_postgres_application_sessions(SimpleNamespace(conn=Connection()))
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
instance = os.path.join(temp_dir, 'supervisor.instance.json')
|
|
Path(instance).write_text('{}', encoding='ascii')
|
|
config = {
|
|
'global': {'state_dir': os.path.join(temp_dir, 'state')},
|
|
'supervisor': {'instance_file': instance, 'log_dir': os.path.join(temp_dir, 'logs')},
|
|
}
|
|
with self.assertRaisesRegex(RuntimeError, 'supervisor metadata'):
|
|
migrate_runtime_safety.require_local_sources_stopped(config)
|
|
|
|
def test_scheduled_keycheck_module_contains_no_schema_ddl(self):
|
|
source = (APP_DIR / 'keycheck_runner.py').read_text(encoding='utf-8')
|
|
for token in ('ALTER TABLE', 'CREATE TABLE', 'CREATE INDEX', 'CREATE UNIQUE INDEX'):
|
|
self.assertNotIn(token, source)
|
|
|
|
def test_maintenance_uses_only_the_shared_cluster_authority_lock(self):
|
|
source = (APP_DIR / 'migrate_runtime_safety.py').read_text(encoding='utf-8')
|
|
self.assertNotIn('MigrationLocalLock', source)
|
|
self.assertEqual(source.count('with ClusterAuthorityLock('), 12)
|
|
|
|
def test_docker_compose_postgres_is_fail_closed_and_noncanonical(self):
|
|
compose = (ROOT / 'docker-compose.postgres.yml').read_text(encoding='utf-8')
|
|
self.assertIn('noncanonical-manual-recovery', compose)
|
|
self.assertIn('restart: "no"', compose)
|
|
self.assertIn('TRUF_DOCKER_POSTGRES_PORT:?', compose)
|
|
self.assertIn('D:/truf/runtime/postgres-docker-noncanonical', compose)
|
|
self.assertNotIn('restart: unless-stopped', compose)
|
|
self.assertNotIn('truf_postgres_data:', compose)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|