Initial server source import

2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
@@ -0,0 +1,316 @@
+import contextlib
+import io
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+
+ROOT = Path(__file__).resolve().parents[1]
+APP_DIR = ROOT / 'app'
+sys.path.insert(0, str(APP_DIR))
+
+import docker_depth_experiment as depth
+import docker_depth_operator as operator
+from runtime_security import PrivateFileError, ensure_private_directory
+
+
+POLICY_SHA256 = 'b' * 64
+SECRET_SENTINEL = 'operator-secret-sentinel'
+
+
+def experiment_config(*, enabled=False, secret_query=False):
+    queries = [f'operator-query-{index:02d}' for index in range(61)]
+    if secret_query:
+        queries[17] = SECRET_SENTINEL
+    return {
+        'global': {
+            'database_url': (
+                f'postgresql://scanner:{SECRET_SENTINEL}@127.0.0.1:5432/truf'
+            ),
+            'sync_file_queues': False,
+        },
+        'sources': {'dockerhub': {
+            'mode': 'search',
+            'require_digest': True,
+            'queries': queries,
+            'pages': 30,
+            'per_page': 100,
+            'docker_platform_candidate_tags': 20,
+            'docker_images_per_repository': 3,
+            'docker_depth_experiment': {
+                'experiment_key': 'operator-test-v1',
+                'enabled': enabled,
+                'queries': queries,
+                'repositories_per_query': 10,
+                'shallow_images_per_repository': 1,
+                'deep_repositories_per_query': 1,
+                'deep_images_per_repository': 10,
+                'target_limit': 1200,
+                'selector_version': depth.DOCKER_DEPTH_SELECTOR_VERSION,
+            },
+        }},
+    }
+
+
+def validated_experiment(*, enabled=False, secret_query=False):
+    return depth.validate_docker_depth_config(
+        experiment_config(enabled=enabled, secret_query=secret_query),
+        managed_postgres=True,
+        final_cutover=True,
+    )
+
+
+def cohort_manifest(experiment):
+    candidates = {}
+    for query_ordinal, query in enumerate(experiment.queries):
+        candidates[query] = [{
+            'repository_queue_id': query_ordinal * 100 + rank,
+            'eligibility_page_id': 10000 + query_ordinal * 100 + rank,
+            'best_search_rank': rank,
+            'valid_distinct_graph_count': rank,
+        } for rank in range(1, 11)]
+    return depth.build_docker_depth_cohort_manifest(
+        experiment, POLICY_SHA256, candidates,
+    )
+
+
+class _Result:
+    def __init__(self, row=None):
+        self.row = row
+
+    def fetchone(self):
+        return self.row
+
+
+class _StatusConnection:
+    is_postgres = True
+
+    def __init__(self):
+        self.statements = []
+        self.commits = 0
+
+    def execute(self, sql, params=None):
+        self.statements.append(str(sql).strip())
+        if 'FROM docker_depth_experiments' in str(sql):
+            return _Result(None)
+        raise AssertionError(f'unexpected status SQL: {sql}')
+
+    def commit(self):
+        self.commits += 1
+
+
+class _StatusDB:
+    def __init__(self):
+        self.conn = _StatusConnection()
+
+
+class DockerDepthOperatorTests(unittest.TestCase):
+    def test_status_is_read_only_and_does_not_emit_queries_or_secrets(self):
+        validated = validated_experiment(secret_query=True)
+        db = _StatusDB()
+
+        @contextlib.contextmanager
+        def opened(*_args, **_kwargs):
+            yield db
+
+        coverage = {
+            'query_count': 61,
+            'complete_query_count': 61,
+            'incomplete_query_count': 0,
+            'required_repository_count': 610,
+            'eligible_repository_slots': 610,
+            'coverage_sha256': 'c' * 64,
+        }
+        stdout = io.StringIO()
+        with mock.patch.object(operator, 'load_config', return_value=validated.config), \
+                mock.patch.object(operator, 'operator_database', side_effect=opened), \
+                mock.patch.object(
+                    operator, 'summarize_docker_depth_fresh_coverage',
+                    return_value=coverage,
+                ), contextlib.redirect_stdout(stdout):
+            self.assertEqual(operator.main(['--config', 'private.yaml', '--status']), 0)
+
+        report = json.loads(stdout.getvalue())
+        self.assertEqual(report['action'], 'status')
+        self.assertEqual(report['fresh_coverage'], coverage)
+        self.assertNotIn(SECRET_SENTINEL, stdout.getvalue())
+        self.assertTrue(db.conn.statements)
+        self.assertTrue(all(statement.upper().startswith('SELECT')
+                            for statement in db.conn.statements))
+
+    def test_failure_output_redacts_raw_exception_text(self):
+        validated = validated_experiment(secret_query=True)
+        stdout = io.StringIO()
+        with mock.patch.object(operator, 'load_config', return_value=validated.config), \
+                mock.patch.object(
+                    operator, 'operator_database',
+                    side_effect=RuntimeError(SECRET_SENTINEL),
+                ), contextlib.redirect_stdout(stdout), self.assertRaises(SystemExit) as raised:
+            operator.main(['--status'])
+        self.assertEqual(stdout.getvalue(), '')
+        self.assertNotIn(SECRET_SENTINEL, str(raised.exception))
+
+    def test_private_manifest_publish_is_exclusive_and_exact_replay_is_safe(self):
+        manifest = cohort_manifest(validated_experiment().experiment)
+        with tempfile.TemporaryDirectory() as directory:
+            ensure_private_directory(directory, reject_reparse=True)
+            path = os.path.join(directory, 'cohort.json')
+            published, created = operator._publish_manifest(path, manifest)
+            replayed, replay_created = operator._publish_manifest(path, manifest)
+            self.assertEqual((published, created), (path, True))
+            self.assertEqual((replayed, replay_created), (path, False))
+
+            changed = dict(manifest)
+            changed['target_limit'] = 1199
+            with self.assertRaisesRegex(RuntimeError, 'different reviewed manifest'):
+                operator._publish_manifest(path, changed)
+
+    def test_private_manifest_paths_reject_untrusted_parent_and_reparse(self):
+        with mock.patch.object(
+            operator, 'require_private_directory',
+            side_effect=PrivateFileError('wrong owner'),
+        ):
+            with self.assertRaises(PrivateFileError):
+                operator._manifest_path('untrusted/cohort.json', existing=False)
+        with mock.patch.object(
+            operator, 'reject_reparse_components',
+            side_effect=PrivateFileError('reparse'),
+        ):
+            with self.assertRaises(PrivateFileError):
+                operator._manifest_path('linked/cohort.json', existing=False)
+
+    def test_approval_mismatch_fails_before_database_access(self):
+        experiment = validated_experiment().experiment
+        manifest = cohort_manifest(experiment)
+        with tempfile.TemporaryDirectory() as directory:
+            ensure_private_directory(directory, reject_reparse=True)
+            path = os.path.join(directory, 'cohort.json')
+            operator._publish_manifest(path, manifest)
+            with self.assertRaisesRegex(ValueError, 'approval hash conflicts'):
+                operator._read_approved_manifest(
+                    path, depth.validate_docker_depth_cohort_manifest,
+                    experiment, POLICY_SHA256, 'f' * 64,
+                )
+
+    def test_phase_config_gates_and_semantic_hash_are_stable(self):
+        disabled = validated_experiment(enabled=False).experiment
+        enabled = validated_experiment(enabled=True).experiment
+        self.assertEqual(disabled.config_hash, enabled.config_hash)
+        operator._require_action_config_state(disabled, 'apply-hold')
+        operator._require_action_config_state(enabled, 'apply-reactivation')
+        operator._require_action_config_state(enabled, 'apply-resolver-disposition')
+        operator._require_action_config_state(enabled, 'apply-resolver-refund')
+        with self.assertRaisesRegex(RuntimeError, 'requires disabled config'):
+            operator._require_action_config_state(enabled, 'generate-cohort')
+        with self.assertRaisesRegex(RuntimeError, 'requires enabled config'):
+            operator._require_action_config_state(disabled, 'generate-reactivation')
+
+    def test_sources_running_refusal_precedes_identity_and_database(self):
+        dsn = 'postgresql://scanner@127.0.0.1:5432/truf'
+        with mock.patch.object(operator, 'preflight_lifecycle_paths'), \
+                mock.patch.object(operator, 'load_postgres_environment'), \
+                mock.patch.object(operator, 'canonical_database_url', return_value=dsn), \
+                mock.patch.object(operator, 'is_postgres_url', return_value=True), \
+                mock.patch.object(
+                    operator, 'ClusterAuthorityLock',
+                    return_value=contextlib.nullcontext(),
+                ), mock.patch.object(
+                    operator, 'require_local_sources_stopped',
+                    side_effect=RuntimeError('sources active'),
+                ), mock.patch.object(operator, 'verify_cluster_identity') as identity, \
+                mock.patch.object(operator, 'ScannerDB') as scanner_db:
+            with self.assertRaisesRegex(RuntimeError, 'sources active'):
+                with operator.operator_database('config.yaml', {}, read_only=True):
+                    pass
+        identity.assert_not_called()
+        scanner_db.assert_not_called()
+
+    def test_database_preflight_enforces_identity_guard_schema_and_cutover(self):
+        dsn = 'postgresql://scanner@127.0.0.1:5432/truf'
+        identity = {
+            'database': 'truf', 'user': 'scanner', 'port': 5432,
+            'data_directory': r'D:\private\postgres', 'system_identifier': '123',
+        }
+        connection = mock.Mock()
+        database = mock.Mock(enabled=True, conn=connection)
+        events = []
+
+        def mark(name):
+            return lambda *_args, **_kwargs: events.append(name)
+
+        with mock.patch.object(
+                    operator, 'preflight_lifecycle_paths', side_effect=mark('preflight'),
+                ), mock.patch.object(
+                    operator, 'load_postgres_environment', side_effect=mark('environment'),
+                ), mock.patch.object(
+                    operator, 'canonical_database_url', return_value=dsn,
+                ), mock.patch.object(
+                    operator, 'is_postgres_url', return_value=True,
+                ), mock.patch.object(
+                    operator, 'ClusterAuthorityLock',
+                    return_value=contextlib.nullcontext(),
+                ), mock.patch.object(
+                    operator, 'require_local_sources_stopped', side_effect=mark('sources'),
+                ), mock.patch.object(
+                    operator, 'verify_cluster_identity',
+                    side_effect=lambda *_args, **_kwargs: events.append('identity') or identity,
+                ), mock.patch.object(
+                    operator, 'canonical_postgres_url', return_value=dsn,
+                ), mock.patch.object(
+                    operator, 'ScannerDB', return_value=database,
+                ), mock.patch.object(
+                    operator, '_verify_online_cluster_identity', side_effect=mark('online'),
+                ), mock.patch.object(
+                    operator, 'postgres_migration_guard',
+                    return_value=contextlib.nullcontext(),
+                ) as guard:
+            with operator.operator_database('config.yaml', {}, read_only=True) as opened:
+                self.assertIs(opened, database)
+
+        self.assertEqual(
+            events, ['preflight', 'environment', 'sources', 'identity', 'online'],
+        )
+        guard.assert_called_once_with(database)
+        database.require_runtime_safety_schema.assert_called_once_with()
+        database.require_final_cutover.assert_called_once_with()
+        connection.execute.assert_called_once_with(
+            'SET default_transaction_read_only = on'
+        )
+        database.close.assert_called_once_with()
+
+    def test_apply_cli_requires_all_explicit_approval_switches(self):
+        for argv in (
+            ['--apply-cohort-manifest', 'cohort.json'],
+            ['--apply-cohort-manifest', 'cohort.json', '--apply'],
+            [
+                '--apply-cohort-manifest', 'cohort.json', '--apply',
+                '--sources-stopped', '--approve-sha256', 'not-a-hash',
+            ],
+        ):
+            with self.subTest(argv=argv), contextlib.redirect_stderr(io.StringIO()), \
+                    self.assertRaises(SystemExit):
+                operator.parse_args(argv)
+
+        args, action = operator.parse_args([
+            '--apply-resolver-refund-manifest', 'refund.json', '--apply',
+            '--sources-stopped', '--approve-sha256', 'a' * 64,
+        ])
+        self.assertEqual(action, 'apply-resolver-refund')
+        self.assertEqual(args.apply_resolver_refund_manifest, 'refund.json')
+
+        args, action = operator.parse_args([
+            '--apply-resolver-disposition-manifest', 'disposition.json', '--apply',
+            '--sources-stopped', '--approve-sha256', 'b' * 64,
+        ])
+        self.assertEqual(action, 'apply-resolver-disposition')
+        self.assertEqual(
+            args.apply_resolver_disposition_manifest, 'disposition.json',
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()