Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,321 @@
|
||||
from pathlib import Path
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP_DIR = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP_DIR))
|
||||
|
||||
import console_runner
|
||||
import scanner
|
||||
|
||||
|
||||
class GitLabDiscoveryRetryTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def response(payload):
|
||||
response = mock.Mock(status_code=200, headers={}, text='')
|
||||
response.json.return_value = payload
|
||||
response.raise_for_status.return_value = None
|
||||
return response
|
||||
|
||||
def test_transient_timeout_retries_then_continues(self):
|
||||
timeout = scanner.requests.exceptions.ReadTimeout('fixture timeout')
|
||||
with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \
|
||||
mock.patch.object(
|
||||
scanner.requests, 'request',
|
||||
side_effect=[timeout, self.response([])],
|
||||
) as request:
|
||||
repos = scanner.fetch_gitlab_repo_items(
|
||||
'fixture', 1, request_attempts=3, retry_delay=0,
|
||||
)
|
||||
|
||||
self.assertEqual(repos, [])
|
||||
self.assertEqual(request.call_count, 2)
|
||||
|
||||
def test_transient_timeout_stops_at_attempt_budget(self):
|
||||
timeout = scanner.requests.exceptions.ReadTimeout('fixture timeout')
|
||||
with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \
|
||||
mock.patch.object(
|
||||
scanner.requests, 'request', side_effect=timeout,
|
||||
) as request:
|
||||
with self.assertRaisesRegex(scanner.GitLabDiscoveryTransportError, 'after 3 attempt'):
|
||||
scanner.fetch_gitlab_repo_items(
|
||||
'fixture', 1, request_attempts=3, retry_delay=0,
|
||||
)
|
||||
|
||||
self.assertEqual(request.call_count, 3)
|
||||
|
||||
def test_invalid_payload_is_not_misclassified_as_transport(self):
|
||||
with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \
|
||||
mock.patch.object(
|
||||
scanner.requests, 'request',
|
||||
return_value=self.response({'unexpected': 'object'}),
|
||||
):
|
||||
with self.assertRaises(scanner.ApiRequestError) as raised:
|
||||
scanner.fetch_gitlab_repo_items(
|
||||
'fixture', 1, request_attempts=3, retry_delay=0,
|
||||
)
|
||||
self.assertNotIsInstance(
|
||||
raised.exception, scanner.GitLabDiscoveryTransportError,
|
||||
)
|
||||
|
||||
def test_configured_attempts_flow_only_to_gitlab_discovery(self):
|
||||
config = console_runner.load_config(str(APP_DIR / 'config.yaml'))
|
||||
args = console_runner.build_args_from_source_config(
|
||||
'gitlab', config['sources']['gitlab'], config['global'], 'fixture',
|
||||
)
|
||||
self.assertEqual(args.gitlab_discovery_request_attempts, 3)
|
||||
self.assertEqual(args.gitlab_discovery_retry_delay, 5)
|
||||
|
||||
fetch_args = SimpleNamespace(
|
||||
platform='gitlab', mode='recent', token=None,
|
||||
max_repo_age_days=0, raise_rate_limit=False, query='fixture',
|
||||
recent_hours=1, pages=1, per_page=100,
|
||||
gitlab_visibility='public', stop_on_seen_pages=False,
|
||||
gitlab_discovery_request_attempts=3,
|
||||
gitlab_discovery_retry_delay=5,
|
||||
)
|
||||
with mock.patch.object(
|
||||
console_runner, 'fetch_recent_gitlab_repos', return_value=[],
|
||||
) as fetch:
|
||||
self.assertEqual(console_runner.fetch_targets(fetch_args), [])
|
||||
self.assertEqual(fetch.call_args.kwargs['request_attempts'], 3)
|
||||
self.assertEqual(fetch.call_args.kwargs['retry_delay'], 5)
|
||||
|
||||
|
||||
class GitLabDiscoveryCycleTests(unittest.TestCase):
|
||||
class Connection:
|
||||
is_postgres = True
|
||||
|
||||
def __init__(self, events):
|
||||
self.events = events
|
||||
|
||||
def rollback(self):
|
||||
self.events.append('rollback')
|
||||
|
||||
class DB:
|
||||
def __init__(self, events):
|
||||
self.conn = GitLabDiscoveryCycleTests.Connection(events)
|
||||
self.events = events
|
||||
self.finished = []
|
||||
|
||||
def start_source_cycle(self, *args, **kwargs):
|
||||
return 7
|
||||
|
||||
def finish_source_cycle(self, *args, **kwargs):
|
||||
self.events.append('finish')
|
||||
self.finished.append((args, kwargs))
|
||||
|
||||
@staticmethod
|
||||
def state(source):
|
||||
source_state = console_runner.default_source_state()
|
||||
source_state['auth_status'] = {
|
||||
'gl_1': {'status': 'ok', 'success_count': 4},
|
||||
}
|
||||
source_state['last_auth'] = 'gl_1'
|
||||
return {'sources': {source: source_state}}
|
||||
|
||||
def patches(self, args, error):
|
||||
return (
|
||||
mock.patch.object(console_runner, 'save_state'),
|
||||
mock.patch.object(
|
||||
console_runner, 'select_auth_entry',
|
||||
return_value={'name': 'gl_1', 'token': 'fixture'},
|
||||
),
|
||||
mock.patch.object(console_runner, 'refresh_auth_summary'),
|
||||
mock.patch.object(
|
||||
console_runner, 'build_args_from_source_config',
|
||||
return_value=args,
|
||||
),
|
||||
mock.patch.object(console_runner, 'configure_source_auth'),
|
||||
mock.patch.object(console_runner, 'queue_counts_for_args', return_value={}),
|
||||
mock.patch.object(console_runner, 'run_cycle', side_effect=error),
|
||||
)
|
||||
|
||||
def test_exhausted_gitlab_discovery_finishes_cycle_without_advancing(self):
|
||||
events = []
|
||||
db = self.DB(events)
|
||||
state = self.state('gitlab')
|
||||
original_auth = dict(state['sources']['gitlab']['auth_status']['gl_1'])
|
||||
args = SimpleNamespace(platform='gitlab', mode='recent')
|
||||
patches = self.patches(
|
||||
args, scanner.GitLabDiscoveryTransportError('bounded failure'),
|
||||
)
|
||||
|
||||
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]:
|
||||
metrics = console_runner.run_configured_source(
|
||||
'gitlab',
|
||||
{'global': {}, 'sources': {'gitlab': {'queries': ['first', 'second'], 'mode': 'recent'}}},
|
||||
state, 'state.json', {}, db, run_id=3,
|
||||
)
|
||||
|
||||
self.assertEqual(events, ['rollback', 'finish'])
|
||||
self.assertTrue(metrics['discovery_transport_failed'])
|
||||
self.assertEqual(state['sources']['gitlab']['query_index'], 0)
|
||||
self.assertEqual(state['sources']['gitlab']['last_status'], 'failed')
|
||||
self.assertEqual(state['sources']['gitlab']['auth_status']['gl_1'], original_auth)
|
||||
self.assertEqual(db.finished[0][0][1], 'failed')
|
||||
|
||||
def test_non_gitlab_api_error_keeps_fail_fast_behavior(self):
|
||||
events = []
|
||||
db = self.DB(events)
|
||||
state = self.state('github')
|
||||
args = SimpleNamespace(platform='github', mode='recent')
|
||||
patches = self.patches(args, scanner.ApiRequestError('bounded failure'))
|
||||
|
||||
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]:
|
||||
with self.assertRaisesRegex(scanner.ApiRequestError, 'bounded failure'):
|
||||
console_runner.run_configured_source(
|
||||
'github',
|
||||
{'global': {}, 'sources': {'github': {'queries': ['first'], 'mode': 'recent'}}},
|
||||
state, 'state.json', {}, db, run_id=3,
|
||||
)
|
||||
|
||||
self.assertEqual(events, ['rollback', 'finish'])
|
||||
|
||||
def test_gitlab_payload_error_keeps_fail_fast_behavior(self):
|
||||
events = []
|
||||
db = self.DB(events)
|
||||
state = self.state('gitlab')
|
||||
args = SimpleNamespace(platform='gitlab', mode='recent')
|
||||
patches = self.patches(
|
||||
args, scanner.ApiRequestError('invalid discovery payload'),
|
||||
)
|
||||
|
||||
with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]:
|
||||
with self.assertRaisesRegex(scanner.ApiRequestError, 'invalid discovery payload'):
|
||||
console_runner.run_configured_source(
|
||||
'gitlab',
|
||||
{'global': {}, 'sources': {'gitlab': {'queries': ['first'], 'mode': 'recent'}}},
|
||||
state, 'state.json', {}, db, run_id=3,
|
||||
)
|
||||
|
||||
self.assertEqual(events, ['rollback', 'finish'])
|
||||
|
||||
|
||||
class GitLabLifecycleCommandTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def message(value):
|
||||
return json.dumps({'level': 'info-0', 'msg': value})
|
||||
|
||||
def test_external_lifecycle_is_enabled_only_by_gitlab_config(self):
|
||||
completed = self.message('finished scanning')
|
||||
boundary = {'since_commit': None, 'skip': False, 'reason': ''}
|
||||
with mock.patch.object(scanner, 'recent_commit_boundary', return_value=boundary), \
|
||||
mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \
|
||||
mock.patch.object(
|
||||
scanner, 'run_command_streamed',
|
||||
return_value=scanner.streamed_output_from_text('', completed, 0),
|
||||
) as run:
|
||||
result = scanner.scan_git_repo(
|
||||
'https://gitlab.com/example/repo.git',
|
||||
provider='gitlab', external_trufflehog_lifecycle=True,
|
||||
)
|
||||
self.assertEqual(result['errors'], [])
|
||||
self.assertIn('--local-dev', run.call_args.args[0])
|
||||
self.assertIn('--no-update', run.call_args.args[0])
|
||||
|
||||
with mock.patch.object(scanner, 'recent_commit_boundary', return_value=boundary), \
|
||||
mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \
|
||||
mock.patch.object(
|
||||
scanner, 'run_command_streamed',
|
||||
return_value=scanner.streamed_output_from_text('', '', 0),
|
||||
) as run:
|
||||
unchanged = scanner.scan_git_repo(
|
||||
'https://github.com/example/repo.git', provider='github',
|
||||
)
|
||||
self.assertEqual(unchanged['errors'], [])
|
||||
self.assertNotIn('--local-dev', run.call_args.args[0])
|
||||
|
||||
config = console_runner.load_config(str(APP_DIR / 'config.yaml'))
|
||||
gitlab_args = console_runner.build_args_from_source_config(
|
||||
'gitlab', config['sources']['gitlab'], config['global'], 'fixture',
|
||||
)
|
||||
github_args = console_runner.build_args_from_source_config(
|
||||
'github', config['sources']['github'], config['global'], 'fixture',
|
||||
)
|
||||
with mock.patch('builtins.print'):
|
||||
gitlab_options = console_runner.prepare_scan_options(gitlab_args, 1)
|
||||
github_options = console_runner.prepare_scan_options(github_args, 1)
|
||||
self.assertIs(gitlab_options['external_trufflehog_lifecycle'], True)
|
||||
self.assertNotIn('external_trufflehog_lifecycle', github_options)
|
||||
|
||||
|
||||
class GitLabLifecyclePolicyTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def message(value):
|
||||
return json.dumps({'level': 'info-0', 'msg': value})
|
||||
|
||||
def apply(self, stderr, returncode, findings=None):
|
||||
return scanner.apply_trufflehog_diagnostics(
|
||||
{'findings': list(findings or []), 'errors': []},
|
||||
stderr, returncode, 'git', require_completion=True,
|
||||
)
|
||||
|
||||
def test_completion_and_incomplete_outcomes_are_explicit(self):
|
||||
complete = self.apply(self.message('finished scanning'), 0)
|
||||
self.assertEqual(complete['errors'], [])
|
||||
self.assertTrue(complete['scan_meta']['trufflehog_finished'])
|
||||
|
||||
incomplete = self.apply(self.message('running source'), 0)
|
||||
self.assertEqual(incomplete['error_class'], 'command_incomplete')
|
||||
self.assertTrue(incomplete['retryable'])
|
||||
self.assertFalse(incomplete['scan_meta']['trufflehog_finished'])
|
||||
|
||||
wrapper = self.apply(self.message('finished scanning'), 1)
|
||||
self.assertEqual(wrapper['error_class'], 'wrapper_exit')
|
||||
self.assertTrue(wrapper['retryable'])
|
||||
|
||||
def test_partial_findings_use_bounded_queue_policy(self):
|
||||
finding = {'DetectorName': 'Example', 'Raw': 'fixture-secret'}
|
||||
result = self.apply(self.message('running source'), 1, [finding])
|
||||
self.assertEqual(result['findings'], [finding])
|
||||
self.assertEqual(result['error_class'], 'command_incomplete')
|
||||
self.assertTrue(result['retryable'])
|
||||
|
||||
args = SimpleNamespace(
|
||||
target_retry_max_attempts=3,
|
||||
target_retry_base_delay_sec=60,
|
||||
target_retry_max_delay_sec=3600,
|
||||
target_timeout_retry_delay_sec=21600,
|
||||
)
|
||||
first = console_runner.queue_error_disposition(
|
||||
None, 'gitlab', 'gitlab', 'target', result, args, {'attempts': 1},
|
||||
)
|
||||
exhausted = console_runner.queue_error_disposition(
|
||||
None, 'gitlab', 'gitlab', 'target', result, args, {'attempts': 3},
|
||||
)
|
||||
self.assertEqual((first[0], first[2], first[3]), ('deferred', 1, 3))
|
||||
self.assertIsNotNone(first[1])
|
||||
self.assertEqual(exhausted, ('failed', None, 3, 3))
|
||||
|
||||
def test_partial_finding_identity_is_stable_within_scan_event(self):
|
||||
scan_result = {
|
||||
'findings': [{'DetectorName': 'Example', 'Raw': 'fixture-secret'}],
|
||||
'errors': [],
|
||||
}
|
||||
with mock.patch.object(
|
||||
scanner, 'scan_git_repo',
|
||||
side_effect=lambda *args, **kwargs: copy.deepcopy(scan_result),
|
||||
):
|
||||
first = scanner.scan_target_result(
|
||||
'https://gitlab.com/example/repo.git', 'gitlab', 'event-1',
|
||||
{'external_trufflehog_lifecycle': True},
|
||||
)
|
||||
second = scanner.scan_target_result(
|
||||
'https://gitlab.com/example/repo.git', 'gitlab', 'event-1',
|
||||
{'external_trufflehog_lifecycle': True},
|
||||
)
|
||||
self.assertEqual(
|
||||
first['findings'][0]['finding_uid'],
|
||||
second['findings'][0]['finding_uid'],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user