sashatrask
1462 lines
58 KiB
Python
1462 lines
58 KiB
Python
from collections.abc import Hashable
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import math
|
|
import posixpath
|
|
import re
|
|
from urllib.parse import urlsplit
|
|
|
|
import yaml
|
|
|
|
from capacity_model import MAX_RESULT_BUNDLE_BYTES, validate_remote_assignment_capacity
|
|
from managed_files import (
|
|
ManagedFileConfigurationError,
|
|
managed_file_root_registry_from_config,
|
|
normalize_managed_file_roots,
|
|
)
|
|
from query_policy import QueryPolicyError, validate_rejected_query_policy
|
|
|
|
|
|
_MESSAGES = {
|
|
'invalid_input': 'YAML document input is invalid',
|
|
'size': 'YAML document exceeds its byte bound',
|
|
'encoding': 'YAML document is not valid UTF-8',
|
|
'duplicate_key': 'YAML document contains a duplicate mapping key',
|
|
'syntax': 'YAML document is invalid',
|
|
'mapping_root': 'Runtime document root must be a mapping',
|
|
'unknown_key': 'Runtime document contains an unsupported field',
|
|
'type': 'Runtime document field has an invalid type',
|
|
'bounds': 'Runtime document field exceeds its bound',
|
|
'schema': 'Runtime document is incomplete',
|
|
'core_profile': 'Runtime core profile is invalid',
|
|
'auth_pool': 'Runtime credential pool is invalid',
|
|
'reference': 'Runtime document reference is invalid',
|
|
'capability': 'Runtime package capability is invalid',
|
|
'deployment_path': 'Runtime deployment path is invalid',
|
|
}
|
|
_MERGE_KEY = object()
|
|
_MISSING = object()
|
|
|
|
MAX_DOCUMENT_DEPTH = 64
|
|
MAX_DOCUMENT_NODES = 100000
|
|
MAX_CONFIG_DOCUMENT_BYTES = 4 * 1024 * 1024
|
|
MAX_SECRETS_DOCUMENT_BYTES = 1024 * 1024
|
|
MAX_MAPPING_ENTRIES = 10000
|
|
MAX_SEQUENCE_ITEMS = 100000
|
|
MAX_MAPPING_KEY_BYTES = 512
|
|
MAX_SCALAR_BYTES = 1024 * 1024
|
|
MAX_PATH_BYTES = 4096
|
|
MAX_NAME_BYTES = 128
|
|
MAX_AUTH_POOLS = 64
|
|
MAX_AUTH_ENTRIES = 256
|
|
MAX_COMPATIBILITY_PROFILES = 16
|
|
MAX_PACKAGE_CAPABILITIES = 16
|
|
_MIN_INTEGER = -(1 << 63)
|
|
_MAX_INTEGER = (1 << 63) - 1
|
|
|
|
_SAFE_DOCUMENTS = frozenset({'config', 'secrets', 'package_capabilities', 'schema'})
|
|
_SAFE_FIELD_PATH = re.compile(r'^[a-z0-9_.\[\]-]{1,512}$')
|
|
_ENV_NAME = re.compile(r'^[A-Z_][A-Z0-9_]{0,127}$')
|
|
_PLACEHOLDER = re.compile(r'{([A-Za-z_][A-Za-z0-9_]*)}')
|
|
_CORE_SOURCES = ('gitlab', 'dockerhub', 'huggingface')
|
|
_KNOWN_CAPABILITIES = frozenset({
|
|
('github', 'github', 'exact_git_v1'),
|
|
('gitlab', 'gitlab', 'exact_git_v1'),
|
|
('dockerhub', 'docker', 'docker_direct_v1'),
|
|
('huggingface', 'huggingface', 'huggingface_space_v1'),
|
|
})
|
|
_SOURCE_CAPABILITIES = {
|
|
'gitlab': ('gitlab', 'gitlab', 'exact_git_v1'),
|
|
'dockerhub': ('dockerhub', 'docker', 'docker_direct_v1'),
|
|
'huggingface': ('huggingface', 'huggingface', 'huggingface_space_v1'),
|
|
}
|
|
_REQUIRED_CONFIG_PATHS = (
|
|
('global',),
|
|
('supervisor',),
|
|
('sources',),
|
|
('global', 'root_dir'),
|
|
('global', 'project_dir'),
|
|
('global', 'runtime_dir'),
|
|
('global', 'postgres_data_dir'),
|
|
('global', 'postgres_bin_dir'),
|
|
('global', 'result_bundle_dir'),
|
|
('global', 'work_dir'),
|
|
('global', 'control_dir'),
|
|
('global', 'secrets_file'),
|
|
('supervisor', 'enabled_sources'),
|
|
('supervisor', 'control_dir'),
|
|
('supervisor', 'worker_api'),
|
|
('supervisor', 'worker_api', 'enabled'),
|
|
('supervisor', 'worker_api', 'sources'),
|
|
('supervisor', 'worker_api', 'auth_entries'),
|
|
('supervisor', 'worker_api', 'compatibility_profiles'),
|
|
('sources', 'gitlab'),
|
|
('sources', 'gitlab', 'enabled'),
|
|
('sources', 'gitlab', 'auth_pool'),
|
|
('sources', 'dockerhub'),
|
|
('sources', 'dockerhub', 'enabled'),
|
|
('sources', 'dockerhub', 'auth_pool'),
|
|
('sources', 'huggingface'),
|
|
('sources', 'huggingface', 'enabled'),
|
|
('sources', 'huggingface', 'auth_pool'),
|
|
)
|
|
_FIXED_PATHS = {
|
|
'root_dir': '/opt/truf',
|
|
'project_dir': '/opt/truf/app',
|
|
'runtime_dir': '/data/runtime-linux',
|
|
'postgres_data_dir': '/data/postgres-linux',
|
|
'postgres_bin_dir': '/usr/lib/postgresql/16/bin',
|
|
'result_bundle_dir': '/data/scanner-result-bundles',
|
|
'work_dir': '/data/scanner-work',
|
|
'control_dir': '/run/truf/control',
|
|
'secrets_file': '/data/config/secrets.yaml',
|
|
}
|
|
_GLOBAL_PATH_FIELDS = frozenset({
|
|
'result_bundle_dir', 'work_dir', 'root_dir', 'project_dir', 'runtime_dir',
|
|
'postgres_data_dir', 'postgres_bin_dir', 'control_dir', 'secrets_file',
|
|
'result_spool_dir', 'legacy_result_spool_dir', 'results_dir', 'queue_dir',
|
|
'state_dir', 'log_dir', 'keycheck_dir', 'postman_cache_dir',
|
|
'gharchive_cache_dir', 'database_path', 'state_file', 'api_proxy_file',
|
|
'download_proxy_file', 'dashboard_db_path', 'scan_limiter_db',
|
|
'dockerhub_tag_cache_path', 'proxy_file', 'trufflehog_path',
|
|
'trufflehog_config',
|
|
})
|
|
_SUPERVISOR_PATH_FIELDS = frozenset({
|
|
'log_dir', 'control_dir', 'instance_file', 'lock_file', 'supervisor_log',
|
|
'status_file', 'dashboard_log', 'state_dir',
|
|
})
|
|
_KEYCHECK_PATH_FIELDS = frozenset({
|
|
'input', 'proxy_file', 'keycheck_dir', 'summary_tsv', 'summary_json',
|
|
'alive_summary_tsv',
|
|
})
|
|
_SOURCE_PATH_FIELDS = frozenset({
|
|
'target_file', 'trufflehog_config', 'postman_cache_dir',
|
|
'gharchive_cache_dir',
|
|
})
|
|
_CONFIG_TEMPLATE_SCHEMA_SHA256 = (
|
|
'45d957d8168c796a9d4081ab0936d83d797c4675c8d477baf63195117363a9d8'
|
|
)
|
|
|
|
|
|
class RuntimeDocumentError(ValueError):
|
|
def __init__(self, category, line=None, column=None, *, document=None, path=None):
|
|
self.category = str(category)
|
|
self.line = int(line) if line is not None else None
|
|
self.column = int(column) if column is not None else None
|
|
self.document = document if document in _SAFE_DOCUMENTS else None
|
|
self.path = (
|
|
path if type(path) is str and _SAFE_FIELD_PATH.fullmatch(path) else None
|
|
)
|
|
message = _MESSAGES[self.category]
|
|
if self.line is not None and self.column is not None:
|
|
message += f' at line {self.line}, column {self.column}'
|
|
if self.document is not None:
|
|
message += f' in {self.document}'
|
|
if self.path is not None:
|
|
message += f' at {self.path}'
|
|
super().__init__(message)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidatedRuntimeDocuments:
|
|
config: dict
|
|
secrets: dict
|
|
|
|
|
|
class _MarkedDocumentError(Exception):
|
|
def __init__(self, mark):
|
|
self.mark = mark
|
|
|
|
|
|
class _DuplicateKeyError(_MarkedDocumentError):
|
|
pass
|
|
|
|
|
|
class _DocumentSyntaxError(_MarkedDocumentError):
|
|
pass
|
|
|
|
|
|
class _StrictSafeLoader(yaml.SafeLoader):
|
|
def _mapping_key(self, key_node, deep):
|
|
key = None
|
|
try:
|
|
if key_node.tag == 'tag:yaml.org,2002:merge':
|
|
return _MERGE_KEY
|
|
key = self.construct_object(key_node, deep=deep)
|
|
if not isinstance(key, Hashable):
|
|
raise _DocumentSyntaxError(key_node.start_mark)
|
|
return key
|
|
finally:
|
|
key_node = deep = key = None
|
|
|
|
def _reject_duplicate_keys(self, pairs, deep):
|
|
seen = key_node = _value_node = key = duplicate = None
|
|
try:
|
|
seen = {}
|
|
for key_node, _value_node in pairs:
|
|
key = self._mapping_key(key_node, deep)
|
|
try:
|
|
duplicate = key in seen
|
|
except Exception:
|
|
raise _DocumentSyntaxError(key_node.start_mark) from None
|
|
if duplicate:
|
|
raise _DuplicateKeyError(key_node.start_mark)
|
|
seen[key] = None
|
|
finally:
|
|
pairs = deep = seen = key_node = _value_node = key = duplicate = None
|
|
|
|
def flatten_mapping(self, node):
|
|
try:
|
|
# SafeLoader recursively calls this method for anonymous merge sources.
|
|
self._reject_duplicate_keys(node.value, False)
|
|
return super().flatten_mapping(node)
|
|
finally:
|
|
node = None
|
|
|
|
def construct_mapping(self, node, deep=False):
|
|
mapping = key_node = value_node = key = None
|
|
try:
|
|
if not isinstance(node, yaml.MappingNode):
|
|
raise _DocumentSyntaxError(node.start_mark)
|
|
|
|
self.flatten_mapping(node)
|
|
self._reject_duplicate_keys(node.value, deep)
|
|
|
|
mapping = {}
|
|
for key_node, value_node in node.value:
|
|
key = self._mapping_key(key_node, deep)
|
|
mapping[key] = self.construct_object(value_node, deep=deep)
|
|
return mapping
|
|
finally:
|
|
node = deep = mapping = key_node = value_node = key = None
|
|
|
|
|
|
def _location(mark):
|
|
if mark is None:
|
|
return None, None
|
|
line = getattr(mark, 'line', None)
|
|
column = getattr(mark, 'column', None)
|
|
if type(line) is not int or type(column) is not int:
|
|
return None, None
|
|
return line + 1, column + 1
|
|
|
|
|
|
def _parse_yaml_document(payload):
|
|
text = None
|
|
try:
|
|
text = payload.decode('utf-8', errors='strict')
|
|
except UnicodeDecodeError:
|
|
pass
|
|
if text is None:
|
|
return None, ('encoding', None, None)
|
|
|
|
value = None
|
|
failure = None
|
|
loader = None
|
|
try:
|
|
loader = _StrictSafeLoader(text)
|
|
value = loader.get_single_data()
|
|
except _DuplicateKeyError as exc:
|
|
failure = ('duplicate_key', *_location(exc.mark))
|
|
except _DocumentSyntaxError as exc:
|
|
failure = ('syntax', *_location(exc.mark))
|
|
except Exception as exc:
|
|
mark = getattr(exc, 'problem_mark', None) or getattr(exc, 'context_mark', None)
|
|
failure = ('syntax', *_location(mark))
|
|
except BaseException:
|
|
if loader is not None:
|
|
try:
|
|
loader.dispose()
|
|
except Exception:
|
|
pass
|
|
loader = None
|
|
value = None
|
|
text = None
|
|
payload = None
|
|
raise
|
|
finally:
|
|
if loader is not None:
|
|
try:
|
|
loader.dispose()
|
|
except Exception:
|
|
failure = failure or ('syntax', None, None)
|
|
|
|
if failure is not None:
|
|
value = None
|
|
return value, failure
|
|
|
|
|
|
def load_yaml_document(payload, *, max_bytes):
|
|
if type(payload) is not bytes or type(max_bytes) is not int or max_bytes <= 0:
|
|
payload = None
|
|
max_bytes = None
|
|
raise RuntimeDocumentError('invalid_input')
|
|
if len(payload) > max_bytes:
|
|
payload = None
|
|
raise RuntimeDocumentError('size')
|
|
|
|
value = failure = None
|
|
try:
|
|
value, failure = _parse_yaml_document(payload)
|
|
finally:
|
|
payload = None
|
|
max_bytes = None
|
|
|
|
if failure is not None:
|
|
raise RuntimeDocumentError(*failure)
|
|
return value
|
|
|
|
|
|
def _preview_document(payload, max_bytes, document):
|
|
try:
|
|
value = load_yaml_document(payload, max_bytes=max_bytes)
|
|
except RuntimeDocumentError as exc:
|
|
return None, (
|
|
exc.category, exc.line, exc.column, document, exc.path,
|
|
)
|
|
finally:
|
|
payload = None
|
|
return value, None
|
|
|
|
|
|
def _field_path(path):
|
|
return '.'.join(path) if path else 'root'
|
|
|
|
|
|
def _failure(category, document, path):
|
|
return category, document, _field_path(path)
|
|
|
|
|
|
def _consume(state, count=1):
|
|
state['nodes'] += count
|
|
return state['nodes'] <= MAX_DOCUMENT_NODES
|
|
|
|
|
|
def _bounded_text(value, limit=MAX_SCALAR_BYTES, *, nonempty=False):
|
|
try:
|
|
if type(value) is not str or '\x00' in value:
|
|
return False
|
|
try:
|
|
size = len(value.encode('utf-8', errors='strict'))
|
|
except UnicodeError:
|
|
return False
|
|
return size <= limit and (not nonempty or bool(value))
|
|
finally:
|
|
value = None
|
|
|
|
|
|
def _bounded_name(value):
|
|
try:
|
|
return (
|
|
_bounded_text(value, MAX_NAME_BYTES, nonempty=True)
|
|
and value == value.strip()
|
|
)
|
|
finally:
|
|
value = None
|
|
|
|
|
|
def _normalize_string_list(value, path, state, *, max_items=MAX_SEQUENCE_ITEMS):
|
|
normalized = item = None
|
|
try:
|
|
if type(value) is not list:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > max_items:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = []
|
|
for item in value:
|
|
if not _consume(state) or not _bounded_text(item, nonempty=True):
|
|
return None, _failure('bounds', 'config', path + ('item',))
|
|
normalized.append(item)
|
|
return normalized, None
|
|
finally:
|
|
value = path = state = normalized = item = None
|
|
|
|
|
|
def _normalize_dynamic_config(value, path, state):
|
|
normalized = key = entry_name = profile_name = raw_profile = None
|
|
item_path = package_manifest = profile = sources = failure = item = None
|
|
query = raw_override = override = name = maximum = None
|
|
try:
|
|
if path == ('supervisor', 'worker_api', 'admin', 'managed_file_roots'):
|
|
try:
|
|
normalized, _registry = normalize_managed_file_roots(value)
|
|
except ManagedFileConfigurationError as exc:
|
|
return None, _failure(exc.category, 'config', path + exc.field)
|
|
return normalized, None
|
|
|
|
if path in (
|
|
('supervisor', 'worker_api', 'sources'),
|
|
('supervisor', 'defaults', 'extra_args'),
|
|
):
|
|
return _normalize_string_list(value, path, state)
|
|
|
|
if path == ('supervisor', 'worker_api', 'assignment_ttl_seconds_by_source'):
|
|
if type(value) is not dict:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > len(_CORE_SOURCES):
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for key, item in value.items():
|
|
item_path = path + (key,) if type(key) is str else path + ('field',)
|
|
if not _consume(state):
|
|
return None, _failure('bounds', 'config', item_path)
|
|
if key not in _CORE_SOURCES:
|
|
return None, _failure('unknown_key', 'config', item_path)
|
|
if type(item) is not int or not 60 <= item <= 7 * 24 * 60 * 60:
|
|
return None, _failure('bounds', 'config', item_path)
|
|
normalized[key] = item
|
|
return normalized, None
|
|
|
|
if path == ('supervisor', 'worker_api', 'auth_entries'):
|
|
if type(value) is not dict:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > len(_CORE_SOURCES) + 1:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for key, entry_name in value.items():
|
|
if not _consume(state) or not _bounded_name(key) or not _bounded_name(entry_name):
|
|
return None, _failure('bounds', 'config', path + ('item',))
|
|
normalized[key] = entry_name
|
|
return normalized, None
|
|
|
|
if path == ('supervisor', 'worker_api', 'compatibility_profiles'):
|
|
if type(value) is not dict:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > MAX_COMPATIBILITY_PROFILES:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for profile_name, raw_profile in value.items():
|
|
item_path = path + ('profile',)
|
|
if not _consume(state) or not _bounded_name(profile_name):
|
|
return None, _failure('bounds', 'config', item_path)
|
|
if type(raw_profile) is not dict:
|
|
return None, _failure('type', 'config', item_path)
|
|
if set(raw_profile) - {'package_manifest', 'sources'}:
|
|
return None, _failure('unknown_key', 'config', item_path)
|
|
package_manifest = raw_profile.get('package_manifest', _MISSING)
|
|
if package_manifest is _MISSING:
|
|
return None, _failure('schema', 'config', item_path + ('package_manifest',))
|
|
if not _consume(state) or not _bounded_text(package_manifest, MAX_PATH_BYTES, nonempty=True):
|
|
return None, _failure('bounds', 'config', item_path + ('package_manifest',))
|
|
profile = {'package_manifest': package_manifest}
|
|
if 'sources' in raw_profile:
|
|
sources, failure = _normalize_string_list(
|
|
raw_profile['sources'], item_path + ('sources',), state,
|
|
max_items=len(_CORE_SOURCES),
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
profile['sources'] = sources
|
|
normalized[profile_name] = profile
|
|
return normalized, None
|
|
|
|
if (
|
|
path == ('keychecks', 'env')
|
|
or len(path) == 4
|
|
and path[0] == 'supervisor'
|
|
and path[1] == 'sources'
|
|
and path[3] == 'env'
|
|
):
|
|
if type(value) is not dict:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > 128:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for key, item in value.items():
|
|
if (
|
|
not _consume(state)
|
|
or type(key) is not str
|
|
or _ENV_NAME.fullmatch(key) is None
|
|
or not _bounded_text(item)
|
|
):
|
|
return None, _failure('bounds', 'config', path + ('item',))
|
|
normalized[key] = item
|
|
return normalized, None
|
|
|
|
if (
|
|
len(path) == 3
|
|
and path[0] == 'sources'
|
|
and path[2] == 'query_overrides'
|
|
):
|
|
if type(value) is not dict:
|
|
return None, _failure('type', 'config', path)
|
|
if len(value) > MAX_MAPPING_ENTRIES:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for query, raw_override in value.items():
|
|
item_path = path + ('item',)
|
|
if not _consume(state) or not _bounded_text(query, 512, nonempty=True):
|
|
return None, _failure('bounds', 'config', item_path)
|
|
if type(raw_override) is not dict:
|
|
return None, _failure('type', 'config', item_path)
|
|
if set(raw_override) - {'pages', 'per_page', 'max_targets'}:
|
|
return None, _failure('unknown_key', 'config', item_path)
|
|
override = {}
|
|
for name, item in raw_override.items():
|
|
maximum = 100000 if name == 'max_targets' else 1000
|
|
if (
|
|
not _consume(state)
|
|
or type(item) is not int
|
|
or not 1 <= item <= maximum
|
|
):
|
|
return None, _failure('bounds', 'config', item_path + (name,))
|
|
override[name] = item
|
|
normalized[query] = override
|
|
return normalized, None
|
|
|
|
return _MISSING
|
|
finally:
|
|
value = path = state = normalized = key = entry_name = None
|
|
profile_name = raw_profile = item_path = package_manifest = profile = None
|
|
sources = failure = item = query = raw_override = override = None
|
|
name = maximum = None
|
|
|
|
|
|
def _normalize_scalar(value, template, path):
|
|
try:
|
|
if type(value) is not type(template):
|
|
return None, _failure('type', 'config', path)
|
|
if type(value) is int and not 0 <= value <= _MAX_INTEGER:
|
|
return None, _failure('bounds', 'config', path)
|
|
if type(value) is float and (not math.isfinite(value) or value < 0):
|
|
return None, _failure('bounds', 'config', path)
|
|
if type(value) is str and not _bounded_text(value):
|
|
return None, _failure('bounds', 'config', path)
|
|
if type(value) not in {type(None), bool, int, float, str}:
|
|
return None, _failure('type', 'config', path)
|
|
return value, None
|
|
finally:
|
|
value = template = path = None
|
|
|
|
|
|
def _normalize_config_shape(value, template, path, state, depth=0):
|
|
identity = dynamic = normalized = key = item = copied = failure = None
|
|
try:
|
|
if depth > MAX_DOCUMENT_DEPTH or not _consume(state):
|
|
return None, _failure('bounds', 'config', path)
|
|
|
|
if type(value) in (dict, list):
|
|
identity = id(value)
|
|
if identity in state['active']:
|
|
return None, _failure('bounds', 'config', path)
|
|
state['active'].add(identity)
|
|
try:
|
|
dynamic = _normalize_dynamic_config(value, path, state)
|
|
if dynamic is not _MISSING:
|
|
return dynamic
|
|
|
|
if type(template) is not type(value):
|
|
return None, _failure('type', 'config', path)
|
|
if type(value) is dict:
|
|
if len(value) > MAX_MAPPING_ENTRIES:
|
|
return None, _failure('bounds', 'config', path)
|
|
normalized = {}
|
|
for key, item in value.items():
|
|
if (
|
|
type(key) is not str
|
|
or not _bounded_text(key, MAX_MAPPING_KEY_BYTES, nonempty=True)
|
|
):
|
|
return None, _failure('type', 'config', path + ('field',))
|
|
if key not in template:
|
|
if (
|
|
len(path) == 2
|
|
and path[0] == 'sources'
|
|
and key in _SOURCE_PATH_FIELDS
|
|
):
|
|
copied, failure = _normalize_scalar(
|
|
item, '', path + (key,),
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized[key] = copied
|
|
continue
|
|
if (
|
|
len(path) == 3
|
|
and path[:2] == ('supervisor', 'sources')
|
|
and key == 'enabled'
|
|
):
|
|
copied, failure = _normalize_scalar(
|
|
item, False, path + (key,),
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized[key] = copied
|
|
continue
|
|
if (
|
|
len(path) == 3
|
|
and path[:2] == ('supervisor', 'sources')
|
|
and key == 'env'
|
|
):
|
|
copied, failure = _normalize_dynamic_config(
|
|
item, path + (key,), state,
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized[key] = copied
|
|
continue
|
|
return None, _failure('unknown_key', 'config', path + ('unknown',))
|
|
copied, failure = _normalize_config_shape(
|
|
item, template[key], path + (key,), state, depth + 1,
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized[key] = copied
|
|
return normalized, None
|
|
|
|
if len(value) > MAX_SEQUENCE_ITEMS:
|
|
return None, _failure('bounds', 'config', path)
|
|
if not template:
|
|
if value:
|
|
return None, _failure('type', 'config', path)
|
|
return [], None
|
|
normalized = []
|
|
for item in value:
|
|
copied, failure = _normalize_config_shape(
|
|
item, template[0], path + ('item',), state, depth + 1,
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized.append(copied)
|
|
return normalized, None
|
|
finally:
|
|
state['active'].remove(identity)
|
|
|
|
return _normalize_scalar(value, template, path)
|
|
finally:
|
|
value = template = path = state = identity = dynamic = None
|
|
normalized = key = item = copied = failure = None
|
|
|
|
|
|
def _dynamic_schema_kind(path):
|
|
if path == ('supervisor', 'worker_api', 'admin', 'managed_file_roots'):
|
|
return 'managed_file_roots'
|
|
if path in (
|
|
('supervisor', 'worker_api', 'sources'),
|
|
('supervisor', 'defaults', 'extra_args'),
|
|
):
|
|
return 'string_list'
|
|
if path == ('supervisor', 'worker_api', 'assignment_ttl_seconds_by_source'):
|
|
return 'assignment_ttl_seconds_by_source'
|
|
if path == ('supervisor', 'worker_api', 'auth_entries'):
|
|
return 'auth_entries'
|
|
if path == ('supervisor', 'worker_api', 'compatibility_profiles'):
|
|
return 'compatibility_profiles'
|
|
if (
|
|
path == ('keychecks', 'env')
|
|
or len(path) == 4
|
|
and path[0] == 'supervisor'
|
|
and path[1] == 'sources'
|
|
and path[3] == 'env'
|
|
):
|
|
return 'environment'
|
|
if (
|
|
len(path) == 3
|
|
and path[0] == 'sources'
|
|
and path[2] == 'query_overrides'
|
|
):
|
|
return 'query_overrides'
|
|
return None
|
|
|
|
|
|
def _config_template_schema_identity(value, path=()):
|
|
dynamic_kind = scalar_names = None
|
|
try:
|
|
dynamic_kind = _dynamic_schema_kind(path)
|
|
if dynamic_kind is not None:
|
|
return {'dynamic': dynamic_kind}
|
|
if type(value) is dict:
|
|
return {
|
|
'mapping': [
|
|
[key, _config_template_schema_identity(value[key], path + (key,))]
|
|
for key in sorted(value)
|
|
],
|
|
}
|
|
if type(value) is list:
|
|
return {
|
|
'sequence': (
|
|
_config_template_schema_identity(value[0], path + ('item',))
|
|
if value else None
|
|
),
|
|
}
|
|
scalar_names = {
|
|
type(None): 'null', bool: 'bool', int: 'int', float: 'float', str: 'str',
|
|
}
|
|
return {'scalar': scalar_names.get(type(value), 'unsupported')}
|
|
finally:
|
|
value = path = dynamic_kind = scalar_names = None
|
|
|
|
|
|
def _config_template_schema_hash(value):
|
|
encoded = None
|
|
try:
|
|
encoded = json.dumps(
|
|
_config_template_schema_identity(value),
|
|
ensure_ascii=True,
|
|
separators=(',', ':'),
|
|
sort_keys=True,
|
|
).encode('ascii')
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
finally:
|
|
value = encoded = None
|
|
|
|
|
|
def _normalize_secrets(value):
|
|
state = {'nodes': 0, 'active': set()}
|
|
pools = normalized = active = None
|
|
pool_name = raw_entries = raw_entry = entries = names = None
|
|
name = token = entry = username = None
|
|
try:
|
|
if type(value) is not dict:
|
|
return None, _failure('mapping_root', 'secrets', ())
|
|
if set(value) - {'auth_pools'}:
|
|
return None, _failure('unknown_key', 'secrets', ('unknown',))
|
|
pools = value.get('auth_pools', {})
|
|
if type(pools) is not dict:
|
|
return None, _failure('type', 'secrets', ('auth_pools',))
|
|
if len(pools) > MAX_AUTH_POOLS:
|
|
return None, _failure('bounds', 'secrets', ('auth_pools',))
|
|
|
|
normalized = {}
|
|
active = {id(value), id(pools)}
|
|
state['active'].update(active)
|
|
if not _consume(state, 2):
|
|
return None, _failure('bounds', 'secrets', ())
|
|
for pool_name, raw_entries in pools.items():
|
|
pool_path = ('auth_pools', 'pool')
|
|
if not _bounded_name(pool_name) or type(raw_entries) is not list:
|
|
return None, _failure('auth_pool', 'secrets', pool_path)
|
|
if id(raw_entries) in state['active'] or len(raw_entries) > MAX_AUTH_ENTRIES:
|
|
return None, _failure('bounds', 'secrets', pool_path)
|
|
state['active'].add(id(raw_entries))
|
|
raw_entries_id = id(raw_entries)
|
|
entries = []
|
|
names = set()
|
|
try:
|
|
for raw_entry in raw_entries:
|
|
entry_path = pool_path + ('entry',)
|
|
if not _consume(state) or type(raw_entry) is not dict:
|
|
return None, _failure('auth_pool', 'secrets', entry_path)
|
|
if id(raw_entry) in state['active']:
|
|
return None, _failure('bounds', 'secrets', entry_path)
|
|
state['active'].add(id(raw_entry))
|
|
try:
|
|
if set(raw_entry) not in ({'name', 'token'}, {'name', 'username', 'token'}):
|
|
return None, _failure('auth_pool', 'secrets', entry_path)
|
|
name = raw_entry.get('name')
|
|
token = raw_entry.get('token')
|
|
if not _bounded_name(name) or not _bounded_text(token, nonempty=True):
|
|
return None, _failure('auth_pool', 'secrets', entry_path)
|
|
if name in names:
|
|
return None, _failure('auth_pool', 'secrets', entry_path + ('name',))
|
|
names.add(name)
|
|
entry = {'name': name}
|
|
if 'username' in raw_entry:
|
|
username = raw_entry['username']
|
|
if not _bounded_text(username, MAX_NAME_BYTES, nonempty=True):
|
|
return None, _failure('auth_pool', 'secrets', entry_path + ('username',))
|
|
entry['username'] = username
|
|
entry['token'] = token
|
|
entries.append(entry)
|
|
finally:
|
|
state['active'].remove(id(raw_entry))
|
|
finally:
|
|
state['active'].remove(raw_entries_id)
|
|
normalized[pool_name] = entries
|
|
return {'auth_pools': normalized}, None
|
|
finally:
|
|
value = pools = normalized = active = None
|
|
pool_name = raw_entries = raw_entry = entries = names = None
|
|
name = token = entry = username = None
|
|
state = None
|
|
|
|
|
|
def _get_path(value, path):
|
|
current = name = None
|
|
try:
|
|
current = value
|
|
for name in path:
|
|
if type(current) is not dict or name not in current:
|
|
return _MISSING
|
|
current = current[name]
|
|
return current
|
|
finally:
|
|
value = path = current = name = None
|
|
|
|
|
|
def _validate_required_fields(config):
|
|
path = None
|
|
try:
|
|
for path in _REQUIRED_CONFIG_PATHS:
|
|
if _get_path(config, path) is _MISSING:
|
|
return _failure('schema', 'config', path)
|
|
return None
|
|
finally:
|
|
config = path = None
|
|
|
|
|
|
def _validate_auth_references(config, secrets):
|
|
pools = sources = worker_config = raw_worker_sources = None
|
|
worker_sources = auth_entries = source_config = matches = None
|
|
docker_pool = source_name = pool_name = entry_name = None
|
|
try:
|
|
pools = secrets['auth_pools']
|
|
sources = config['sources']
|
|
docker_pool = sources['dockerhub'].get('auth_pool') or ''
|
|
|
|
for source_name, source_config in sources.items():
|
|
pool_name = source_config.get('auth_pool')
|
|
if pool_name and pool_name not in pools:
|
|
return _failure('reference', 'config', ('sources', source_name, 'auth_pool'))
|
|
|
|
if docker_pool:
|
|
if any('username' not in entry for entry in pools[docker_pool]):
|
|
return _failure('auth_pool', 'secrets', ('auth_pools', 'pool', 'entry'))
|
|
|
|
worker_config = config['supervisor']['worker_api']
|
|
raw_worker_sources = worker_config['sources']
|
|
if len(raw_worker_sources) != len(set(raw_worker_sources)):
|
|
return _failure('core_profile', 'config', ('supervisor', 'worker_api', 'sources'))
|
|
if raw_worker_sources:
|
|
if any(source not in _CORE_SOURCES for source in raw_worker_sources):
|
|
return _failure('core_profile', 'config', ('supervisor', 'worker_api', 'sources'))
|
|
worker_sources = tuple(source for source in _CORE_SOURCES if source in raw_worker_sources)
|
|
else:
|
|
worker_sources = _CORE_SOURCES
|
|
worker_config['sources'] = list(worker_sources)
|
|
|
|
auth_entries = worker_config['auth_entries']
|
|
if set(auth_entries) - (set(worker_sources) | {'github'}):
|
|
return _failure('reference', 'config', ('supervisor', 'worker_api', 'auth_entries'))
|
|
if set(auth_entries) - {'github', 'gitlab'}:
|
|
return _failure('reference', 'config', ('supervisor', 'worker_api', 'auth_entries'))
|
|
for source_name, entry_name in auth_entries.items():
|
|
source_config = sources.get(source_name)
|
|
if type(source_config) is not dict or source_config.get('enabled') is not True:
|
|
return _failure('reference', 'config', ('supervisor', 'worker_api', 'auth_entries'))
|
|
pool_name = source_config.get('auth_pool')
|
|
if not pool_name or pool_name not in pools:
|
|
return _failure('reference', 'config', ('supervisor', 'worker_api', 'auth_entries'))
|
|
matches = [entry for entry in pools[pool_name] if entry['name'] == entry_name]
|
|
if len(matches) != 1:
|
|
return _failure('reference', 'config', ('supervisor', 'worker_api', 'auth_entries'))
|
|
return None
|
|
finally:
|
|
config = secrets = pools = sources = None
|
|
worker_config = raw_worker_sources = worker_sources = None
|
|
auth_entries = source_config = matches = None
|
|
docker_pool = source_name = pool_name = entry_name = None
|
|
|
|
|
|
def _normalize_capabilities(package_capabilities, profiles, worker_sources, enabled):
|
|
if package_capabilities is None:
|
|
package_capabilities = {}
|
|
if type(package_capabilities) is not dict:
|
|
return None, _failure('capability', 'package_capabilities', ())
|
|
if set(package_capabilities) != set(profiles):
|
|
return None, _failure('capability', 'package_capabilities', ('profiles',))
|
|
if enabled and not profiles:
|
|
return None, _failure(
|
|
'capability', 'config',
|
|
('supervisor', 'worker_api', 'compatibility_profiles'),
|
|
)
|
|
|
|
normalized_capabilities = {}
|
|
for profile_name, evidence in package_capabilities.items():
|
|
path = ('profiles', 'profile')
|
|
if (
|
|
not _bounded_name(profile_name)
|
|
or type(evidence) is not dict
|
|
or set(evidence) != {'package_manifest', 'capabilities'}
|
|
or evidence.get('package_manifest') != profiles[profile_name]['package_manifest']
|
|
):
|
|
return None, _failure('capability', 'package_capabilities', path)
|
|
raw_capabilities = evidence['capabilities']
|
|
if type(raw_capabilities) is not list:
|
|
return None, _failure('capability', 'package_capabilities', path)
|
|
if not 1 <= len(raw_capabilities) <= MAX_PACKAGE_CAPABILITIES:
|
|
return None, _failure('capability', 'package_capabilities', path)
|
|
seen = set()
|
|
capabilities = []
|
|
for raw_item in raw_capabilities:
|
|
item_path = path + ('capability',)
|
|
if type(raw_item) is not dict or set(raw_item) != {
|
|
'source', 'platform', 'planning_kind',
|
|
}:
|
|
return None, _failure('capability', 'package_capabilities', item_path)
|
|
if any(type(raw_item[name]) is not str for name in raw_item):
|
|
return None, _failure('capability', 'package_capabilities', item_path)
|
|
capability = (
|
|
raw_item['source'], raw_item['platform'], raw_item['planning_kind'],
|
|
)
|
|
if capability not in _KNOWN_CAPABILITIES or capability in seen:
|
|
return None, _failure('capability', 'package_capabilities', item_path)
|
|
seen.add(capability)
|
|
capabilities.append(capability)
|
|
normalized_capabilities[profile_name] = frozenset(capabilities)
|
|
|
|
covered = set()
|
|
for profile_name, profile in profiles.items():
|
|
capabilities = normalized_capabilities[profile_name]
|
|
raw_sources = profile.get('sources') or [item[0] for item in capabilities]
|
|
if len(raw_sources) != len(set(raw_sources)):
|
|
return None, _failure(
|
|
'capability', 'config',
|
|
('supervisor', 'worker_api', 'compatibility_profiles', 'profile', 'sources'),
|
|
)
|
|
profile_sources = tuple(source for source in _CORE_SOURCES if source in raw_sources)
|
|
if (
|
|
not profile_sources
|
|
or set(raw_sources) != set(profile_sources)
|
|
or not set(profile_sources) <= set(worker_sources)
|
|
or any(_SOURCE_CAPABILITIES[source] not in capabilities for source in profile_sources)
|
|
):
|
|
return None, _failure(
|
|
'capability', 'config',
|
|
('supervisor', 'worker_api', 'compatibility_profiles', 'profile', 'sources'),
|
|
)
|
|
profile['sources'] = list(profile_sources)
|
|
covered.update(profile_sources)
|
|
if enabled and covered != set(worker_sources):
|
|
return None, _failure(
|
|
'capability', 'config',
|
|
('supervisor', 'worker_api', 'compatibility_profiles'),
|
|
)
|
|
return normalized_capabilities, None
|
|
|
|
|
|
def _validate_worker_config(config, package_capabilities):
|
|
worker_config = enabled = address_text = address = integer_bounds = None
|
|
name = minimum = maximum = value = source = effective_ttl = minimum_ttl = None
|
|
ttl_overrides = None
|
|
admin_config = default = origin = marker = parsed = parsed_port = None
|
|
_normalized = failure = managed_file_roots = None
|
|
try:
|
|
worker_config = config['supervisor']['worker_api']
|
|
enabled = worker_config['enabled']
|
|
if type(enabled) is not bool:
|
|
return _failure('type', 'config', ('supervisor', 'worker_api', 'enabled'))
|
|
|
|
address_text = worker_config.get('address', '127.0.0.1')
|
|
try:
|
|
address = ipaddress.ip_address(address_text)
|
|
except (TypeError, ValueError):
|
|
return _failure('bounds', 'config', ('supervisor', 'worker_api', 'address'))
|
|
if address.is_unspecified or address.is_multicast or not (
|
|
address.is_loopback or address.is_private
|
|
):
|
|
return _failure('bounds', 'config', ('supervisor', 'worker_api', 'address'))
|
|
|
|
integer_bounds = {
|
|
'port': (1024, 65535),
|
|
'assignment_ttl_seconds': (60, 7 * 24 * 60 * 60),
|
|
'max_bundle_bytes': (
|
|
1024 * 1024,
|
|
min(MAX_RESULT_BUNDLE_BYTES, config['global'].get(
|
|
'result_bundle_max_event_bytes', MAX_RESULT_BUNDLE_BYTES,
|
|
)),
|
|
),
|
|
'reaper_interval_seconds': (5, 3600),
|
|
'reaper_batch_size': (1, 1000),
|
|
'limit_concurrency': (1, 1024),
|
|
'body_idle_timeout_seconds': (1, 120),
|
|
'json_body_timeout_seconds': (1, 300),
|
|
'bundle_body_timeout_seconds': (30, 86400),
|
|
}
|
|
for name, (minimum, maximum) in integer_bounds.items():
|
|
value = worker_config.get(name)
|
|
if type(value) is not int or not minimum <= value <= maximum:
|
|
return _failure('bounds', 'config', ('supervisor', 'worker_api', name))
|
|
|
|
ttl_overrides = worker_config.get('assignment_ttl_seconds_by_source', {})
|
|
for source in worker_config['sources']:
|
|
effective_ttl = ttl_overrides.get(
|
|
source, worker_config['assignment_ttl_seconds'],
|
|
)
|
|
minimum_ttl = (
|
|
config['sources'][source]['timeout']
|
|
+ worker_config['bundle_body_timeout_seconds']
|
|
+ 60
|
|
)
|
|
if effective_ttl < minimum_ttl:
|
|
return _failure(
|
|
'bounds', 'config',
|
|
(
|
|
'supervisor', 'worker_api',
|
|
'assignment_ttl_seconds_by_source', source,
|
|
) if source in ttl_overrides else (
|
|
'supervisor', 'worker_api', 'assignment_ttl_seconds',
|
|
),
|
|
)
|
|
|
|
admin_config = worker_config.get('admin') or {}
|
|
if type(admin_config) is not dict or type(admin_config.get('enabled', False)) is not bool:
|
|
return _failure('type', 'config', ('supervisor', 'worker_api', 'admin'))
|
|
for name, default, minimum, maximum in (
|
|
('max_body_bytes', 8192, 1024, 65536),
|
|
('snapshot_limit', 200, 1, 500),
|
|
('requeue_limit', 100, 1, 500),
|
|
):
|
|
value = admin_config.get(name, default)
|
|
if type(value) is not int or not minimum <= value <= maximum:
|
|
return _failure(
|
|
'bounds', 'config', ('supervisor', 'worker_api', 'admin', name),
|
|
)
|
|
try:
|
|
managed_file_roots = managed_file_root_registry_from_config(config)
|
|
except ManagedFileConfigurationError as exc:
|
|
return _failure(
|
|
exc.category, 'config',
|
|
('supervisor', 'worker_api', 'admin', 'managed_file_roots')
|
|
+ exc.field,
|
|
)
|
|
if admin_config.get('enabled') is True:
|
|
origin = admin_config.get('origin')
|
|
marker = admin_config.get('edge_marker')
|
|
try:
|
|
parsed = urlsplit(origin)
|
|
parsed_port = parsed.port
|
|
except (TypeError, ValueError):
|
|
parsed = None
|
|
parsed_port = None
|
|
if (
|
|
type(origin) is not str
|
|
or not 1 <= len(origin) <= 512
|
|
or any(character.isspace() for character in origin)
|
|
or parsed is None
|
|
or parsed.scheme != 'https'
|
|
or not parsed.hostname
|
|
or parsed.username is not None
|
|
or parsed.password is not None
|
|
or parsed.path
|
|
or parsed.query
|
|
or parsed.fragment
|
|
or parsed_port is not None and not 1 <= parsed_port <= 65535
|
|
):
|
|
return _failure(
|
|
'bounds', 'config', ('supervisor', 'worker_api', 'admin', 'origin'),
|
|
)
|
|
if (
|
|
type(marker) is not str
|
|
or not 32 <= len(marker) <= 512
|
|
or any(character.isspace() for character in marker)
|
|
):
|
|
return _failure(
|
|
'bounds', 'config', ('supervisor', 'worker_api', 'admin', 'edge_marker'),
|
|
)
|
|
|
|
_normalized, failure = _normalize_capabilities(
|
|
package_capabilities,
|
|
worker_config['compatibility_profiles'],
|
|
worker_config['sources'],
|
|
enabled,
|
|
)
|
|
return failure
|
|
finally:
|
|
config = package_capabilities = worker_config = enabled = address_text = None
|
|
address = integer_bounds = name = minimum = maximum = value = None
|
|
source = effective_ttl = minimum_ttl = ttl_overrides = None
|
|
admin_config = default = origin = marker = None
|
|
parsed = parsed_port = _normalized = failure = managed_file_roots = None
|
|
|
|
|
|
def _expand_path(raw_value, resolved_globals):
|
|
if not _bounded_text(raw_value, MAX_PATH_BYTES) or any(
|
|
marker in raw_value for marker in ('\\', '$', '~')
|
|
):
|
|
return None
|
|
references = _PLACEHOLDER.findall(raw_value)
|
|
if len(references) > 32:
|
|
return None
|
|
if any(reference not in resolved_globals for reference in references):
|
|
return None
|
|
expanded = _PLACEHOLDER.sub(lambda match: resolved_globals[match.group(1)], raw_value)
|
|
if '{' in expanded or '}' in expanded or not expanded:
|
|
return None
|
|
if not _bounded_text(expanded, MAX_PATH_BYTES) or not expanded.startswith('/'):
|
|
return None
|
|
if posixpath.normpath(expanded) != expanded:
|
|
return None
|
|
return expanded
|
|
|
|
|
|
def _resolve_global_deployment_paths(config):
|
|
global_config = resolved = active = name = raw_value = None
|
|
try:
|
|
global_config = config['global']
|
|
resolved = {}
|
|
active = set()
|
|
|
|
def resolve(name):
|
|
raw_value = references = reference = value = None
|
|
try:
|
|
if name in resolved:
|
|
return resolved[name]
|
|
if name in active or name not in global_config or name not in _GLOBAL_PATH_FIELDS:
|
|
return None
|
|
raw_value = global_config[name]
|
|
if type(raw_value) is not str:
|
|
return None
|
|
active.add(name)
|
|
references = _PLACEHOLDER.findall(raw_value)
|
|
for reference in references:
|
|
if resolve(reference) is None:
|
|
active.remove(name)
|
|
return None
|
|
value = _expand_path(raw_value, resolved)
|
|
active.remove(name)
|
|
if value is not None:
|
|
resolved[name] = value
|
|
return value
|
|
finally:
|
|
name = raw_value = references = reference = value = None
|
|
|
|
for name in _GLOBAL_PATH_FIELDS & set(global_config):
|
|
raw_value = global_config[name]
|
|
if raw_value == '':
|
|
resolved[name] = ''
|
|
continue
|
|
if resolve(name) is None:
|
|
return None, _failure('deployment_path', 'config', ('global', name))
|
|
|
|
return resolved, None
|
|
finally:
|
|
config = global_config = resolved = active = name = raw_value = None
|
|
|
|
|
|
def _resolve_package_manifest_path(config, raw_value):
|
|
resolved = failure = path = None
|
|
try:
|
|
try:
|
|
resolved, failure = _resolve_global_deployment_paths(config)
|
|
except Exception:
|
|
return None
|
|
if failure is not None:
|
|
return None
|
|
path = _expand_path(raw_value, resolved)
|
|
if path is None or not path.startswith('/data/worker-packages/'):
|
|
return None
|
|
return path
|
|
finally:
|
|
config = raw_value = resolved = failure = path = None
|
|
|
|
|
|
def _validate_deployment_paths(config):
|
|
global_config = resolved = failure = name = expected = runtime_root = None
|
|
value = trufflehog_config = supervisor = supervisor_paths = None
|
|
supervisor_roots = root = keychecks = keycheck_paths = None
|
|
source_name = source_config = source_paths = source_roots = None
|
|
profiles = profile = path = None
|
|
try:
|
|
global_config = config['global']
|
|
resolved, failure = _resolve_global_deployment_paths(config)
|
|
if failure is not None:
|
|
return failure
|
|
|
|
for name, expected in _FIXED_PATHS.items():
|
|
if resolved.get(name) != expected:
|
|
return _failure('deployment_path', 'config', ('global', name))
|
|
|
|
def contained(path, root):
|
|
return path == root or path.startswith(root.rstrip('/') + '/')
|
|
|
|
runtime_root = resolved['runtime_dir']
|
|
for name in (
|
|
'result_spool_dir', 'legacy_result_spool_dir', 'results_dir', 'queue_dir',
|
|
'state_dir', 'log_dir', 'keycheck_dir', 'postman_cache_dir',
|
|
'gharchive_cache_dir', 'database_path', 'state_file', 'api_proxy_file',
|
|
'download_proxy_file', 'dashboard_db_path', 'scan_limiter_db',
|
|
'dockerhub_tag_cache_path', 'proxy_file',
|
|
):
|
|
value = resolved.get(name)
|
|
if value and not contained(value, runtime_root):
|
|
return _failure('deployment_path', 'config', ('global', name))
|
|
trufflehog_config = resolved.get('trufflehog_config')
|
|
if trufflehog_config and not contained(trufflehog_config, resolved['project_dir']):
|
|
return _failure('deployment_path', 'config', ('global', 'trufflehog_config'))
|
|
|
|
def validate_section(section, fields, prefix):
|
|
values = name = raw_value = value = None
|
|
try:
|
|
values = {}
|
|
for name in fields & set(section):
|
|
raw_value = section[name]
|
|
if raw_value == '':
|
|
values[name] = ''
|
|
continue
|
|
value = _expand_path(raw_value, resolved)
|
|
if value is None:
|
|
return _failure('deployment_path', 'config', prefix + (name,))
|
|
values[name] = value
|
|
return values
|
|
finally:
|
|
section = fields = prefix = values = name = raw_value = value = None
|
|
|
|
supervisor = config['supervisor']
|
|
supervisor_paths = validate_section(
|
|
supervisor, _SUPERVISOR_PATH_FIELDS, ('supervisor',),
|
|
)
|
|
if type(supervisor_paths) is tuple:
|
|
return supervisor_paths
|
|
if supervisor_paths.get('control_dir') != _FIXED_PATHS['control_dir']:
|
|
return _failure('deployment_path', 'config', ('supervisor', 'control_dir'))
|
|
supervisor_roots = {
|
|
'log_dir': resolved.get('log_dir'),
|
|
'instance_file': resolved['control_dir'],
|
|
'lock_file': resolved['control_dir'],
|
|
'supervisor_log': resolved.get('log_dir'),
|
|
'status_file': resolved.get('log_dir'),
|
|
'dashboard_log': resolved.get('log_dir'),
|
|
'state_dir': resolved.get('state_dir'),
|
|
}
|
|
for name, root in supervisor_roots.items():
|
|
value = supervisor_paths.get(name)
|
|
if value and (not root or not contained(value, root)):
|
|
return _failure('deployment_path', 'config', ('supervisor', name))
|
|
|
|
keychecks = config.get('keychecks')
|
|
if type(keychecks) is dict:
|
|
keycheck_paths = validate_section(keychecks, _KEYCHECK_PATH_FIELDS, ('keychecks',))
|
|
if type(keycheck_paths) is tuple:
|
|
return keycheck_paths
|
|
for name, value in keycheck_paths.items():
|
|
root = resolved.get('proxy_file') if name == 'proxy_file' else resolved.get('keycheck_dir')
|
|
if value and (not root or not contained(value, root)):
|
|
return _failure('deployment_path', 'config', ('keychecks', name))
|
|
for source_name, source_config in config['sources'].items():
|
|
source_paths = validate_section(
|
|
source_config, _SOURCE_PATH_FIELDS, ('sources', source_name),
|
|
)
|
|
if type(source_paths) is tuple:
|
|
return source_paths
|
|
source_roots = {
|
|
'target_file': resolved.get('queue_dir'),
|
|
'trufflehog_config': resolved.get('trufflehog_config'),
|
|
'postman_cache_dir': resolved.get('postman_cache_dir'),
|
|
'gharchive_cache_dir': resolved.get('gharchive_cache_dir'),
|
|
}
|
|
for name, value in source_paths.items():
|
|
root = source_roots[name]
|
|
if value and (not root or not contained(value, root)):
|
|
return _failure(
|
|
'deployment_path', 'config', ('sources', source_name, name),
|
|
)
|
|
|
|
profiles = supervisor['worker_api']['compatibility_profiles']
|
|
for profile in profiles.values():
|
|
path = _resolve_package_manifest_path(config, profile['package_manifest'])
|
|
if path is None:
|
|
return _failure(
|
|
'deployment_path', 'config',
|
|
('supervisor', 'worker_api', 'compatibility_profiles', 'profile', 'package_manifest'),
|
|
)
|
|
return None
|
|
finally:
|
|
config = global_config = resolved = failure = name = expected = None
|
|
runtime_root = value = trufflehog_config = supervisor = supervisor_paths = None
|
|
supervisor_roots = root = keychecks = keycheck_paths = None
|
|
source_name = source_config = source_paths = source_roots = None
|
|
profiles = profile = path = None
|
|
|
|
|
|
def _validate_runtime_semantics(config, secrets, package_capabilities):
|
|
failure = enabled_sources = supervisor_sources = None
|
|
source_name = source_config = queries = overrides = None
|
|
try:
|
|
failure = _validate_required_fields(config)
|
|
if failure is not None:
|
|
return failure
|
|
|
|
enabled_sources = config['supervisor']['enabled_sources']
|
|
if (
|
|
type(enabled_sources) is not list
|
|
or len(enabled_sources) != len(_CORE_SOURCES)
|
|
or set(enabled_sources) != set(_CORE_SOURCES)
|
|
):
|
|
return _failure('core_profile', 'config', ('supervisor', 'enabled_sources'))
|
|
config['supervisor']['enabled_sources'] = list(_CORE_SOURCES)
|
|
if any(config['sources'][source]['enabled'] is not True for source in _CORE_SOURCES):
|
|
return _failure('core_profile', 'config', ('sources',))
|
|
|
|
try:
|
|
validate_remote_assignment_capacity(config['global'])
|
|
except ValueError as exc:
|
|
return _failure('bounds', 'config', ('global', str(exc)))
|
|
|
|
supervisor_sources = config['supervisor'].get('sources', {})
|
|
if any(source not in config['sources'] for source in supervisor_sources):
|
|
return _failure('reference', 'config', ('supervisor', 'sources'))
|
|
for source_name, source_config in config['sources'].items():
|
|
queries = source_config.get('queries', [])
|
|
overrides = source_config.get('query_overrides', {})
|
|
if any(query not in queries for query in overrides):
|
|
return _failure('reference', 'config', ('sources', source_name, 'query_overrides'))
|
|
if source_config.get('target_claim_order', 'oldest') not in (
|
|
'oldest', 'newest', 'balanced',
|
|
):
|
|
return _failure(
|
|
'core_profile', 'config',
|
|
('sources', source_name, 'target_claim_order'),
|
|
)
|
|
|
|
failure = _validate_auth_references(config, secrets)
|
|
if failure is not None:
|
|
return failure
|
|
failure = _validate_worker_config(config, package_capabilities)
|
|
if failure is not None:
|
|
return failure
|
|
failure = _validate_deployment_paths(config)
|
|
if failure is not None:
|
|
return failure
|
|
|
|
try:
|
|
validate_rejected_query_policy(config)
|
|
except QueryPolicyError:
|
|
return _failure('reference', 'config', ('query_policy',))
|
|
return None
|
|
finally:
|
|
config = secrets = package_capabilities = None
|
|
failure = enabled_sources = supervisor_sources = None
|
|
source_name = source_config = queries = overrides = None
|
|
|
|
|
|
def _validate_runtime_documents_inner(config, secrets, config_template, package_capabilities):
|
|
template = normalized_config = normalized_secrets = None
|
|
template_state = state = failure = None
|
|
try:
|
|
if type(config) is not dict:
|
|
return None, _failure('mapping_root', 'config', ())
|
|
if type(config_template) is not dict:
|
|
return None, _failure('mapping_root', 'schema', ())
|
|
|
|
template_state = {'nodes': 0, 'active': set()}
|
|
template, failure = _normalize_config_shape(
|
|
config_template, config_template, (), template_state,
|
|
)
|
|
if failure is not None:
|
|
return None, _failure('schema', 'schema', ())
|
|
if _config_template_schema_hash(template) != _CONFIG_TEMPLATE_SCHEMA_SHA256:
|
|
return None, _failure('schema', 'schema', ())
|
|
state = {'nodes': 0, 'active': set()}
|
|
normalized_config, failure = _normalize_config_shape(
|
|
config, template, (), state,
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
normalized_secrets, failure = _normalize_secrets(secrets)
|
|
if failure is not None:
|
|
return None, failure
|
|
failure = _validate_runtime_semantics(
|
|
normalized_config, normalized_secrets, package_capabilities,
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
return ValidatedRuntimeDocuments(
|
|
config=normalized_config,
|
|
secrets=normalized_secrets,
|
|
), None
|
|
except Exception:
|
|
return None, _failure('schema', 'schema', ())
|
|
except BaseException:
|
|
config = secrets = config_template = package_capabilities = None
|
|
template = normalized_config = normalized_secrets = None
|
|
template_state = state = failure = None
|
|
raise
|
|
|
|
|
|
def validate_runtime_documents(
|
|
config,
|
|
secrets,
|
|
*,
|
|
config_template,
|
|
package_capabilities=None,
|
|
):
|
|
validated = failure = None
|
|
try:
|
|
validated, failure = _validate_runtime_documents_inner(
|
|
config, secrets, config_template, package_capabilities,
|
|
)
|
|
finally:
|
|
config = None
|
|
secrets = None
|
|
config_template = None
|
|
package_capabilities = None
|
|
|
|
if failure is not None:
|
|
raise RuntimeDocumentError(
|
|
failure[0], document=failure[1], path=failure[2],
|
|
)
|
|
return validated
|
|
|
|
|
|
def _preview_runtime_documents_inner(
|
|
config_payload,
|
|
secrets_payload,
|
|
config_template_payload,
|
|
package_capabilities,
|
|
):
|
|
config = secrets = config_template = validated = None
|
|
try:
|
|
config, failure = _preview_document(
|
|
config_payload, MAX_CONFIG_DOCUMENT_BYTES, 'config',
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
secrets, failure = _preview_document(
|
|
secrets_payload, MAX_SECRETS_DOCUMENT_BYTES, 'secrets',
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
config_template, failure = _preview_document(
|
|
config_template_payload, MAX_CONFIG_DOCUMENT_BYTES, 'schema',
|
|
)
|
|
if failure is not None:
|
|
return None, failure
|
|
validated, failure = _validate_runtime_documents_inner(
|
|
config, secrets, config_template, package_capabilities,
|
|
)
|
|
if failure is not None:
|
|
return None, (failure[0], None, None, failure[1], failure[2])
|
|
return validated, None
|
|
except Exception:
|
|
return None, ('schema', None, None, 'schema', 'root')
|
|
finally:
|
|
config_payload = None
|
|
secrets_payload = None
|
|
config_template_payload = None
|
|
package_capabilities = None
|
|
config = None
|
|
secrets = None
|
|
config_template = None
|
|
validated = None
|
|
|
|
|
|
def preview_runtime_documents(
|
|
config_payload,
|
|
secrets_payload,
|
|
*,
|
|
config_template_payload,
|
|
package_capabilities=None,
|
|
):
|
|
validated = failure = None
|
|
try:
|
|
validated, failure = _preview_runtime_documents_inner(
|
|
config_payload,
|
|
secrets_payload,
|
|
config_template_payload,
|
|
package_capabilities,
|
|
)
|
|
finally:
|
|
config_payload = None
|
|
secrets_payload = None
|
|
config_template_payload = None
|
|
package_capabilities = None
|
|
|
|
if failure is not None:
|
|
raise RuntimeDocumentError(
|
|
failure[0], failure[1], failure[2],
|
|
document=failure[3], path=failure[4],
|
|
)
|
|
return validated
|