Initial server source import

2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
@@ -0,0 +1,126 @@
+from datetime import datetime
+import re
+
+
+REJECTED_QUERY_STATUS = 'rejected_zero_alive'
+REJECTED_QUERY_KEYS = {
+    'source', 'query', 'status', 'evidence_cutoff', 'successful_scans',
+    'findings', 'unique_credentials', 'pending_candidates',
+    'ever_alive_credentials', 'reviewed_queue_rows',
+}
+REJECTED_QUERY_COUNT_KEYS = {
+    'successful_scans', 'findings', 'unique_credentials',
+    'pending_candidates', 'ever_alive_credentials', 'reviewed_queue_rows',
+}
+OPERATIONAL_QUERY_SENTINELS = {
+    ('github_archive', 'gharchive'),
+    ('github_archive_files', 'gharchive-files'),
+    ('github_gists', 'gists'),
+    ('github_actions', 'logs'),
+    ('gitlab_ci', 'logs'),
+    ('huggingface', 'spaces'),
+}
+SOURCE_RE = re.compile(r'[a-z][a-z0-9_]{0,63}')
+MAX_QUERY_LENGTH = 512
+MAX_EVIDENCE_COUNT = (1 << 63) - 1
+
+
+class QueryPolicyError(ValueError):
+    pass
+
+
+def _active_queries(source, source_config):
+    raw_queries = source_config.get('queries', [])
+    if isinstance(raw_queries, str):
+        raw_queries = raw_queries.split(',')
+    if not isinstance(raw_queries, (list, tuple)):
+        raise QueryPolicyError(f'active query policy is invalid for source {source}')
+    queries = set()
+    for raw_query in raw_queries:
+        query = str(raw_query or '').strip()
+        if not query:
+            raise QueryPolicyError(f'active query policy contains an empty query for source {source}')
+        queries.add(query)
+    return queries
+
+
+def validate_rejected_query_policy(config):
+    config = config or {}
+    raw_policy = config.get('query_policy')
+    if raw_policy is None:
+        return ()
+    if not isinstance(raw_policy, dict) or set(raw_policy) != {'rejected'}:
+        raise QueryPolicyError('query_policy must contain only the rejected registry')
+    raw_entries = raw_policy.get('rejected')
+    if not isinstance(raw_entries, list):
+        raise QueryPolicyError('query_policy.rejected must be a list')
+
+    sources = config.get('sources') or {}
+    if not isinstance(sources, dict):
+        raise QueryPolicyError('configured sources must be a mapping')
+
+    normalized = []
+    seen = set()
+    for raw_entry in raw_entries:
+        if not isinstance(raw_entry, dict) or set(raw_entry) != REJECTED_QUERY_KEYS:
+            raise QueryPolicyError('rejected query evidence shape is invalid')
+        source = raw_entry.get('source')
+        query = raw_entry.get('query')
+        if not isinstance(source, str) or not SOURCE_RE.fullmatch(source):
+            raise QueryPolicyError('rejected query source is invalid')
+        if source not in sources or not isinstance(sources[source], dict):
+            raise QueryPolicyError(f'rejected query source is not configured: {source}')
+        if (
+            not isinstance(query, str)
+            or query != query.strip()
+            or not query
+            or len(query) > MAX_QUERY_LENGTH
+        ):
+            raise QueryPolicyError(f'rejected query text is invalid for source {source}')
+        pair = (source, query)
+        if pair in seen:
+            raise QueryPolicyError('rejected query registry contains a duplicate pair')
+        if pair in OPERATIONAL_QUERY_SENTINELS:
+            raise QueryPolicyError('operational query sentinel cannot be rejected')
+        if query in _active_queries(source, sources[source]):
+            raise QueryPolicyError('active and rejected query policy overlap')
+        if raw_entry.get('status') != REJECTED_QUERY_STATUS:
+            raise QueryPolicyError('rejected query status is invalid')
+
+        cutoff = raw_entry.get('evidence_cutoff')
+        if not isinstance(cutoff, str) or not cutoff or cutoff != cutoff.strip():
+            raise QueryPolicyError('rejected query evidence cutoff is invalid')
+        try:
+            parsed_cutoff = datetime.fromisoformat(cutoff.replace('Z', '+00:00'))
+        except ValueError as exc:
+            raise QueryPolicyError('rejected query evidence cutoff is invalid') from exc
+        if parsed_cutoff.tzinfo is None or parsed_cutoff.utcoffset() is None:
+            raise QueryPolicyError('rejected query evidence cutoff must include a timezone')
+
+        counts = {}
+        for name in REJECTED_QUERY_COUNT_KEYS:
+            value = raw_entry.get(name)
+            if (
+                isinstance(value, bool)
+                or not isinstance(value, int)
+                or value < 0
+                or value > MAX_EVIDENCE_COUNT
+            ):
+                raise QueryPolicyError(f'rejected query {name} is invalid')
+            counts[name] = value
+        if counts['successful_scans'] < 1000:
+            raise QueryPolicyError('rejected query has fewer than 1000 successful scans')
+        if counts['pending_candidates'] != 0:
+            raise QueryPolicyError('rejected query still has pending candidates')
+        if counts['ever_alive_credentials'] != 0:
+            raise QueryPolicyError('rejected query has historical alive credentials')
+
+        seen.add(pair)
+        normalized.append({
+            'source': source,
+            'query': query,
+            'status': REJECTED_QUERY_STATUS,
+            'evidence_cutoff': cutoff,
+            **{name: counts[name] for name in sorted(REJECTED_QUERY_COUNT_KEYS)},
+        })
+    return tuple(sorted(normalized, key=lambda entry: (entry['source'], entry['query'])))