Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,265 @@
|
||||
"""Offline stdlib tests: python -I -S -B tests/test_container_import_config.py."""
|
||||
|
||||
from copy import deepcopy
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
HELPER = Path(__file__).resolve().parents[1] / 'app' / 'container_import_config.py'
|
||||
SPEC = importlib.util.spec_from_file_location('container_import_config', HELPER)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
translate_windows_config = MODULE.translate_windows_config
|
||||
|
||||
|
||||
class ContainerImportConfigTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.baseline = {
|
||||
'global': {
|
||||
'root_dir': '/opt/truf', 'project_dir': '{root_dir}/app',
|
||||
'runtime_dir': '/data/runtime-linux', 'postgres_data_dir': '/data/postgres-linux',
|
||||
'postgres_bin_dir': '/usr/lib/postgresql/16/bin',
|
||||
'result_bundle_dir': '/data/scanner-result-bundles', 'work_dir': '/data/scanner-work',
|
||||
'control_dir': '/run/truf/control', 'trufflehog_path': '/usr/local/bin/trufflehog',
|
||||
'proxy_file': '{runtime_dir}/proxy.txt', 'secrets_file': '/data/config/secrets.yaml',
|
||||
'trufflehog_config': '{project_dir}/trufflehog-custom-detectors.yaml',
|
||||
'max_active_scans': 1, 'opportunistic_scan_slots': 0,
|
||||
'trufflehog_job_memory_limit_bytes': 4294967296,
|
||||
'trufflehog_windows_job_cpu_weight': 2, 'trufflehog_windows_memory_priority': 4,
|
||||
'no_verification': False, 'api_proxy_enabled': False, 'min_free_gb': 20,
|
||||
'database_url': '', 'baseline_only_policy': True,
|
||||
},
|
||||
'supervisor': {
|
||||
'control_dir': '{control_dir}', 'instance_file': '{control_dir}/supervisor.instance.json',
|
||||
'lock_file': '{control_dir}/supervisor.lock', 'control_host': '127.0.0.1', 'control_port': 8765,
|
||||
'interactive': False, 'autostart': True, 'enabled_sources': ['dockerhub'],
|
||||
'dashboard': {'enabled': False, 'port': 5000},
|
||||
},
|
||||
'sources': {
|
||||
'dockerhub': {'enabled': False, 'trufflehog_job_memory_limit_bytes': 6442450944},
|
||||
'linux_only': {'enabled': True},
|
||||
},
|
||||
'keychecks': {'enabled': False, 'services': ['baseline-only']},
|
||||
'providers': {'baseline_only': {}}, 'query_policy': {'rejected': []},
|
||||
}
|
||||
self.original = {
|
||||
'global': {
|
||||
'root_dir': r'D:\truf', 'project_dir': '.', 'runtime_dir': r'{root_dir}\runtime',
|
||||
'postgres_data_dir': r'S:\postgres-data', 'result_bundle_dir': r'S:\scanner-result-bundles',
|
||||
'work_dir': r'S:\scanner-work', 'trufflehog_path': r'C:\Tools\trufflehog.exe',
|
||||
'control_dir': r'{runtime_dir}\control', 'proxy_file': r'{runtime_dir}\proxy.txt',
|
||||
'secrets_file': r'{project_dir}\secrets.yaml',
|
||||
'trufflehog_config': r'{project_dir}\trufflehog-custom-detectors.yaml',
|
||||
'max_active_scans': 3, 'opportunistic_scan_slots': 1,
|
||||
'trufflehog_job_memory_limit_bytes': 8589934592,
|
||||
'trufflehog_windows_job_cpu_weight': 9, 'trufflehog_windows_memory_priority': 5,
|
||||
'no_verification': True, 'api_proxy_enabled': True, 'download_proxy_enabled': False,
|
||||
'min_free_gb': 31, 'detectors': ['fixture'], 'drop_detectors': r'fixture\regex',
|
||||
'database_url': r'postgresql://fixture:fake\password@fixture.invalid/db',
|
||||
},
|
||||
'supervisor': {
|
||||
'control_dir': '{control_dir}', 'instance_file': r'{control_dir}\supervisor.instance.json',
|
||||
'lock_file': r'{control_dir}\supervisor.lock', 'control_host': 'localhost', 'control_port': 8888,
|
||||
'interactive': True, 'autostart': False, 'enabled_sources': ['github', 'keychecks'],
|
||||
'dashboard': {'enabled': True, 'port': 5444},
|
||||
'defaults': {'once': True, 'extra_args': ['--query', r'literal\query']},
|
||||
'sources': {'github': {'enabled': True, 'interval': 123}},
|
||||
},
|
||||
'sources': {
|
||||
'github': {
|
||||
'enabled': True, 'workers': 17,
|
||||
'queries': [r'path:/src\d+/ "sk-\w{6}"', r'C:\literal\query'],
|
||||
'query_overrides': {r'literal\query': {'pages': 7}},
|
||||
'auth_pool': [{'name': r'fixture\account', 'token': r'fake\token'}],
|
||||
'url': r'https://fixture.invalid/search?q=\d+&path=C:\literal',
|
||||
'regex': r'\bfixture[\\/]\w+\s*$',
|
||||
},
|
||||
'dockerhub': {'enabled': True, 'trufflehog_job_memory_limit_bytes': 12884901888},
|
||||
},
|
||||
'keychecks': {
|
||||
'enabled': True, 'services': ['fixture'], 'retry_valid': True,
|
||||
'service_args': {'fixture': ['--pattern', r'\bfixture\w+']},
|
||||
'env': {'FIXTURE_AUTH': r'fake\auth', 'FIXTURE_URL': r'https://fixture.invalid/\x'},
|
||||
},
|
||||
'providers': {'fixture': {'auth': r'fake\auth', 'path': r'C:\opaque\provider-value'}},
|
||||
'query_policy': {'rejected': [{'source': 'github', 'query': r'\bfixture\w+'}]},
|
||||
}
|
||||
|
||||
def test_preserves_queries_regex_urls_auth_and_policy_without_baseline_merge(self):
|
||||
config, adjusted = translate_windows_config(self.original, self.baseline)
|
||||
for key in ('providers', 'query_policy', 'keychecks'):
|
||||
self.assertEqual(config[key], self.original[key])
|
||||
self.assertEqual(config['sources']['github'], self.original['sources']['github'])
|
||||
self.assertTrue(config['sources']['dockerhub']['enabled'])
|
||||
self.assertNotIn('linux_only', config['sources'])
|
||||
self.assertNotIn('baseline_only_policy', config['global'])
|
||||
for key in ('no_verification', 'api_proxy_enabled', 'download_proxy_enabled',
|
||||
'min_free_gb', 'detectors', 'drop_detectors', 'database_url'):
|
||||
self.assertEqual(config['global'][key], self.original['global'][key])
|
||||
for key in ('enabled_sources', 'defaults', 'sources'):
|
||||
self.assertEqual(config['supervisor'][key], self.original['supervisor'][key])
|
||||
self.assertFalse(any(path.startswith(('providers.', 'query_policy.')) for path in adjusted))
|
||||
|
||||
def test_exact_fixed_paths_do_not_depend_on_import_config_directory(self):
|
||||
config, _ = translate_windows_config(self.original, self.baseline)
|
||||
expected = {
|
||||
'root_dir': '/opt/truf', 'project_dir': '/opt/truf/app',
|
||||
'runtime_dir': '/data/runtime-linux', 'postgres_data_dir': '/data/postgres-linux',
|
||||
'postgres_bin_dir': '/usr/lib/postgresql/16/bin',
|
||||
'result_bundle_dir': '/data/scanner-result-bundles', 'work_dir': '/data/scanner-work',
|
||||
'control_dir': '/run/truf/control', 'trufflehog_path': '/usr/local/bin/trufflehog',
|
||||
'proxy_file': '/data/runtime-linux/proxy.txt', 'secrets_file': '/data/config/secrets.yaml',
|
||||
'trufflehog_config': '/data/config/trufflehog-custom-detectors.yaml',
|
||||
}
|
||||
self.assertEqual({key: config['global'][key] for key in expected}, expected)
|
||||
self.assertEqual(config['supervisor']['control_dir'], '/run/truf/control')
|
||||
self.assertEqual(config['supervisor']['instance_file'], '/run/truf/control/supervisor.instance.json')
|
||||
self.assertEqual(config['supervisor']['lock_file'], '/run/truf/control/supervisor.lock')
|
||||
|
||||
def test_only_platform_and_headless_settings_follow_baseline(self):
|
||||
self.original['sources']['github']['trufflehog_windows_job_cpu_weight'] = 7
|
||||
config, _ = translate_windows_config(self.original, self.baseline)
|
||||
for key in ('max_active_scans', 'opportunistic_scan_slots', 'trufflehog_job_memory_limit_bytes',
|
||||
'trufflehog_windows_job_cpu_weight', 'trufflehog_windows_memory_priority'):
|
||||
self.assertEqual(config['global'][key], self.baseline['global'][key])
|
||||
self.assertEqual(config['sources']['dockerhub']['trufflehog_job_memory_limit_bytes'], 6442450944)
|
||||
self.assertEqual(config['sources']['github']['trufflehog_windows_job_cpu_weight'], 2)
|
||||
for key in ('interactive', 'autostart', 'control_host', 'control_port'):
|
||||
self.assertEqual(config['supervisor'][key], self.baseline['supervisor'][key])
|
||||
self.assertEqual(config['supervisor']['dashboard'], {'enabled': False, 'port': 5444})
|
||||
|
||||
def test_normalizes_only_known_path_fields_and_keeps_templates_and_custom_names(self):
|
||||
fields = {
|
||||
'global': {
|
||||
'result_spool_dir': r'{runtime_dir}\result_spool',
|
||||
'legacy_result_spool_dir': r'{runtime_dir}\result_spool',
|
||||
'results_dir': r'{runtime_dir}\results', 'queue_dir': r'{runtime_dir}\queues',
|
||||
'state_dir': r'{runtime_dir}\state', 'log_dir': r'{runtime_dir}\logs',
|
||||
'keycheck_dir': r'{runtime_dir}\keychecks', 'postman_cache_dir': r'{runtime_dir}\postman_cache',
|
||||
'gharchive_cache_dir': r'{state_dir}\gharchive_cache',
|
||||
'database_path': r'{results_dir}\scanner_active.db', 'dashboard_db_path': '{database_path}',
|
||||
'state_file': r'{state_dir}\custom-state.json', 'scan_limiter_db': r'{state_dir}\scan_limiter.db',
|
||||
'dockerhub_tag_cache_path': r'{state_dir}\dockerhub_tag_cache.sqlite',
|
||||
'api_proxy_file': '{proxy_file}', 'download_proxy_file': '',
|
||||
},
|
||||
'supervisor': {
|
||||
'log_dir': '{log_dir}', 'state_dir': '{state_dir}', 'supervisor_log': r'{log_dir}\supervisor.log',
|
||||
'status_file': r'{log_dir}\supervisor.status.txt', 'dashboard_log': r'{log_dir}\dashboard.log',
|
||||
},
|
||||
'keychecks': {
|
||||
'input': r'{results_dir}\found_secrets.jsonl', 'proxy_file': r'{runtime_dir}\proxy.txt',
|
||||
'keycheck_dir': r'{keycheck_dir}\fixture', 'summary_tsv': r'{keycheck_dir}\summary.tsv',
|
||||
'summary_json': r'{keycheck_dir}\summary.json', 'alive_summary_tsv': r'{keycheck_dir}\alive_summary.tsv',
|
||||
},
|
||||
}
|
||||
for section, values in fields.items():
|
||||
self.original[section].update(values)
|
||||
source_paths = {'target_file': r'inputs\fixture.txt', 'postman_cache_dir': r'{postman_cache_dir}\fixture',
|
||||
'gharchive_cache_dir': r'{state_dir}\gharchive_cache'}
|
||||
self.original['sources']['github'].update(source_paths)
|
||||
config, adjusted = translate_windows_config(self.original, self.baseline)
|
||||
for section, values in fields.items():
|
||||
for key, value in values.items():
|
||||
self.assertEqual(config[section][key], value.replace('\\', '/'))
|
||||
self.assertEqual(section + '.' + key in adjusted, '\\' in value)
|
||||
for key, value in source_paths.items():
|
||||
self.assertEqual(config['sources']['github'][key], value.replace('\\', '/'))
|
||||
|
||||
def test_known_copied_detector_and_proxy_references_use_imported_files(self):
|
||||
for detector in (r'{project_dir}\trufflehog-custom-detectors.yaml',
|
||||
r'D:\truf\app\trufflehog-custom-detectors.yaml', 'trufflehog-custom-detectors.yaml'):
|
||||
with self.subTest(detector=detector):
|
||||
self.original['sources']['github']['trufflehog_config'] = detector
|
||||
self.original['keychecks']['proxy_file'] = r'D:\truf\runtime\proxy.txt'
|
||||
config, _ = translate_windows_config(self.original, self.baseline)
|
||||
self.assertEqual(config['sources']['github']['trufflehog_config'],
|
||||
'/data/config/trufflehog-custom-detectors.yaml')
|
||||
self.assertEqual(config['keychecks']['proxy_file'], '/data/runtime-linux/proxy.txt')
|
||||
|
||||
def test_inputs_and_nested_values_are_independent_on_success(self):
|
||||
before = deepcopy((self.original, self.baseline))
|
||||
config, _ = translate_windows_config(self.original, self.baseline)
|
||||
self.assertEqual((self.original, self.baseline), before)
|
||||
config['sources']['github']['queries'].append('new query')
|
||||
config['sources']['github']['auth_pool'][0]['token'] = 'changed fake token'
|
||||
config['keychecks']['service_args']['fixture'].append('new argument')
|
||||
config['supervisor']['enabled_sources'].clear()
|
||||
self.assertEqual((self.original, self.baseline), before)
|
||||
|
||||
def test_adjustment_report_is_exact_sorted_key_paths_and_idempotent(self):
|
||||
config, adjusted = translate_windows_config(self.original, self.baseline)
|
||||
expected = ['global.' + key for key in (
|
||||
'root_dir', 'project_dir', 'runtime_dir', 'postgres_data_dir', 'postgres_bin_dir',
|
||||
'result_bundle_dir', 'work_dir', 'control_dir', 'trufflehog_path', 'proxy_file',
|
||||
'secrets_file', 'trufflehog_config', 'max_active_scans', 'opportunistic_scan_slots',
|
||||
'trufflehog_job_memory_limit_bytes', 'trufflehog_windows_job_cpu_weight', 'trufflehog_windows_memory_priority',
|
||||
)]
|
||||
expected += ['supervisor.' + key for key in (
|
||||
'control_dir', 'instance_file', 'lock_file', 'control_host', 'control_port',
|
||||
'interactive', 'autostart', 'dashboard.enabled',
|
||||
)]
|
||||
expected.append('sources.dockerhub.trufflehog_job_memory_limit_bytes')
|
||||
self.assertEqual(adjusted, sorted(expected))
|
||||
self.assertEqual(translate_windows_config(config, self.baseline), (config, []))
|
||||
|
||||
def test_rejects_unreviewed_absolute_executables_and_data_before_overriding(self):
|
||||
for key in ('root_dir', 'project_dir', 'runtime_dir', 'postgres_data_dir', 'postgres_bin_dir',
|
||||
'result_bundle_dir', 'work_dir', 'trufflehog_path', 'trufflehog_config',
|
||||
'proxy_file', 'secrets_file', 'database_path', 'api_proxy_file', 'download_proxy_file'):
|
||||
original = deepcopy(self.original)
|
||||
original['global'][key] = r'Z:\private-fixture\custom-path'
|
||||
before = deepcopy((original, self.baseline))
|
||||
with self.subTest(key=key), self.assertRaisesRegex(ValueError, 'global\\.' + key) as error:
|
||||
translate_windows_config(original, self.baseline)
|
||||
self.assertNotIn('private-fixture', str(error.exception))
|
||||
self.assertEqual((original, self.baseline), before)
|
||||
|
||||
def test_rejects_foreign_custom_inputs_including_unc_device_and_drive_relative_paths(self):
|
||||
paths = (r'C:\private-fixture\input.txt', 'C:/private-fixture/input.txt',
|
||||
r'\\server\private-fixture\input.txt', '//server/private-fixture/input.txt',
|
||||
r'\\?\C:\private-fixture\input.txt', r'\private-fixture\input.txt',
|
||||
r'C:private-fixture\input.txt', r'D:\truf\app\custom-input.txt')
|
||||
for value in paths:
|
||||
for prefix, mapping, key in (
|
||||
('keychecks', self.original['keychecks'], 'input'),
|
||||
('sources.github', self.original['sources']['github'], 'target_file'),
|
||||
('sources.github', self.original['sources']['github'], 'trufflehog_config'),
|
||||
('supervisor', self.original['supervisor'], 'instance_file'),
|
||||
):
|
||||
with self.subTest(prefix=prefix, key=key, value=value):
|
||||
previous = deepcopy(mapping)
|
||||
mapping[key] = value
|
||||
with self.assertRaises(ValueError) as error:
|
||||
translate_windows_config(self.original, self.baseline)
|
||||
self.assertIn(prefix + '.' + key, str(error.exception))
|
||||
self.assertNotIn('private-fixture', str(error.exception))
|
||||
mapping.clear()
|
||||
mapping.update(previous)
|
||||
|
||||
def test_rejects_relative_custom_executable_instead_of_substituting_a_different_binary(self):
|
||||
self.original['global']['trufflehog_path'] = r'custom-tools\private-fixture.exe'
|
||||
with self.assertRaisesRegex(ValueError, 'global.trufflehog_path'):
|
||||
translate_windows_config(self.original, self.baseline)
|
||||
|
||||
def test_rejects_baseline_escaping_the_fixed_storage_contract(self):
|
||||
for section, key, value in (
|
||||
('global', 'project_dir', '.'), ('global', 'root_dir', '/elsewhere'),
|
||||
('global', 'postgres_data_dir', '/elsewhere'), ('supervisor', 'control_dir', '/data/control'),
|
||||
):
|
||||
baseline = deepcopy(self.baseline)
|
||||
baseline[section][key] = value
|
||||
with self.subTest(section=section, key=key), self.assertRaisesRegex(ValueError, 'Invalid Linux baseline path'):
|
||||
translate_windows_config(self.original, baseline)
|
||||
|
||||
def test_translation_performs_no_io_or_environment_lookup(self):
|
||||
with mock.patch('builtins.open', side_effect=AssertionError('file IO forbidden')), \
|
||||
mock.patch('os.getenv', side_effect=AssertionError('environment lookup forbidden')), \
|
||||
mock.patch('os.environ', {}), mock.patch('builtins.print') as output:
|
||||
translate_windows_config(self.original, self.baseline)
|
||||
output.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user