from pathlib import Path import copy import json import sys from types import SimpleNamespace import unittest from unittest import mock ROOT = Path(__file__).resolve().parents[1] APP_DIR = ROOT / 'app' sys.path.insert(0, str(APP_DIR)) import console_runner import scanner class GitLabDiscoveryRetryTests(unittest.TestCase): @staticmethod def response(payload): response = mock.Mock(status_code=200, headers={}, text='') response.json.return_value = payload response.raise_for_status.return_value = None return response def test_transient_timeout_retries_then_continues(self): timeout = scanner.requests.exceptions.ReadTimeout('fixture timeout') with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \ mock.patch.object( scanner.requests, 'request', side_effect=[timeout, self.response([])], ) as request: repos = scanner.fetch_gitlab_repo_items( 'fixture', 1, request_attempts=3, retry_delay=0, ) self.assertEqual(repos, []) self.assertEqual(request.call_count, 2) def test_transient_timeout_stops_at_attempt_budget(self): timeout = scanner.requests.exceptions.ReadTimeout('fixture timeout') with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \ mock.patch.object( scanner.requests, 'request', side_effect=timeout, ) as request: with self.assertRaisesRegex(scanner.GitLabDiscoveryTransportError, 'after 3 attempt'): scanner.fetch_gitlab_repo_items( 'fixture', 1, request_attempts=3, retry_delay=0, ) self.assertEqual(request.call_count, 3) def test_invalid_payload_is_not_misclassified_as_transport(self): with mock.patch.object(scanner.scan_config, 'api_proxy_enabled', False), \ mock.patch.object( scanner.requests, 'request', return_value=self.response({'unexpected': 'object'}), ): with self.assertRaises(scanner.ApiRequestError) as raised: scanner.fetch_gitlab_repo_items( 'fixture', 1, request_attempts=3, retry_delay=0, ) self.assertNotIsInstance( raised.exception, scanner.GitLabDiscoveryTransportError, ) def test_configured_attempts_flow_only_to_gitlab_discovery(self): config = console_runner.load_config(str(APP_DIR / 'config.yaml')) args = console_runner.build_args_from_source_config( 'gitlab', config['sources']['gitlab'], config['global'], 'fixture', ) self.assertEqual(args.gitlab_discovery_request_attempts, 3) self.assertEqual(args.gitlab_discovery_retry_delay, 5) fetch_args = SimpleNamespace( platform='gitlab', mode='recent', token=None, max_repo_age_days=0, raise_rate_limit=False, query='fixture', recent_hours=1, pages=1, per_page=100, gitlab_visibility='public', stop_on_seen_pages=False, gitlab_discovery_request_attempts=3, gitlab_discovery_retry_delay=5, ) with mock.patch.object( console_runner, 'fetch_recent_gitlab_repos', return_value=[], ) as fetch: self.assertEqual(console_runner.fetch_targets(fetch_args), []) self.assertEqual(fetch.call_args.kwargs['request_attempts'], 3) self.assertEqual(fetch.call_args.kwargs['retry_delay'], 5) class GitLabDiscoveryCycleTests(unittest.TestCase): class Connection: is_postgres = True def __init__(self, events): self.events = events def rollback(self): self.events.append('rollback') class DB: def __init__(self, events): self.conn = GitLabDiscoveryCycleTests.Connection(events) self.events = events self.finished = [] def start_source_cycle(self, *args, **kwargs): return 7 def finish_source_cycle(self, *args, **kwargs): self.events.append('finish') self.finished.append((args, kwargs)) @staticmethod def state(source): source_state = console_runner.default_source_state() source_state['auth_status'] = { 'gl_1': {'status': 'ok', 'success_count': 4}, } source_state['last_auth'] = 'gl_1' return {'sources': {source: source_state}} def patches(self, args, error): return ( mock.patch.object(console_runner, 'save_state'), mock.patch.object( console_runner, 'select_auth_entry', return_value={'name': 'gl_1', 'token': 'fixture'}, ), mock.patch.object(console_runner, 'refresh_auth_summary'), mock.patch.object( console_runner, 'build_args_from_source_config', return_value=args, ), mock.patch.object(console_runner, 'configure_source_auth'), mock.patch.object(console_runner, 'queue_counts_for_args', return_value={}), mock.patch.object(console_runner, 'run_cycle', side_effect=error), ) def test_exhausted_gitlab_discovery_finishes_cycle_without_advancing(self): events = [] db = self.DB(events) state = self.state('gitlab') original_auth = dict(state['sources']['gitlab']['auth_status']['gl_1']) args = SimpleNamespace(platform='gitlab', mode='recent') patches = self.patches( args, scanner.GitLabDiscoveryTransportError('bounded failure'), ) with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]: metrics = console_runner.run_configured_source( 'gitlab', {'global': {}, 'sources': {'gitlab': {'queries': ['first', 'second'], 'mode': 'recent'}}}, state, 'state.json', {}, db, run_id=3, ) self.assertEqual(events, ['rollback', 'finish']) self.assertTrue(metrics['discovery_transport_failed']) self.assertEqual(state['sources']['gitlab']['query_index'], 0) self.assertEqual(state['sources']['gitlab']['last_status'], 'failed') self.assertEqual(state['sources']['gitlab']['auth_status']['gl_1'], original_auth) self.assertEqual(db.finished[0][0][1], 'failed') def test_non_gitlab_api_error_keeps_fail_fast_behavior(self): events = [] db = self.DB(events) state = self.state('github') args = SimpleNamespace(platform='github', mode='recent') patches = self.patches(args, scanner.ApiRequestError('bounded failure')) with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]: with self.assertRaisesRegex(scanner.ApiRequestError, 'bounded failure'): console_runner.run_configured_source( 'github', {'global': {}, 'sources': {'github': {'queries': ['first'], 'mode': 'recent'}}}, state, 'state.json', {}, db, run_id=3, ) self.assertEqual(events, ['rollback', 'finish']) def test_gitlab_payload_error_keeps_fail_fast_behavior(self): events = [] db = self.DB(events) state = self.state('gitlab') args = SimpleNamespace(platform='gitlab', mode='recent') patches = self.patches( args, scanner.ApiRequestError('invalid discovery payload'), ) with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5], patches[6]: with self.assertRaisesRegex(scanner.ApiRequestError, 'invalid discovery payload'): console_runner.run_configured_source( 'gitlab', {'global': {}, 'sources': {'gitlab': {'queries': ['first'], 'mode': 'recent'}}}, state, 'state.json', {}, db, run_id=3, ) self.assertEqual(events, ['rollback', 'finish']) class GitLabLifecycleCommandTests(unittest.TestCase): @staticmethod def message(value): return json.dumps({'level': 'info-0', 'msg': value}) def test_external_lifecycle_is_enabled_only_by_gitlab_config(self): completed = self.message('finished scanning') boundary = {'since_commit': None, 'skip': False, 'reason': ''} with mock.patch.object(scanner, 'recent_commit_boundary', return_value=boundary), \ mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \ mock.patch.object( scanner, 'run_command_streamed', return_value=scanner.streamed_output_from_text('', completed, 0), ) as run: result = scanner.scan_git_repo( 'https://gitlab.com/example/repo.git', provider='gitlab', external_trufflehog_lifecycle=True, ) self.assertEqual(result['errors'], []) self.assertIn('--local-dev', run.call_args.args[0]) self.assertIn('--no-update', run.call_args.args[0]) with mock.patch.object(scanner, 'recent_commit_boundary', return_value=boundary), \ mock.patch.object(scanner, 'get_trufflehog_cmd', return_value='trufflehog'), \ mock.patch.object( scanner, 'run_command_streamed', return_value=scanner.streamed_output_from_text('', '', 0), ) as run: unchanged = scanner.scan_git_repo( 'https://github.com/example/repo.git', provider='github', ) self.assertEqual(unchanged['errors'], []) self.assertNotIn('--local-dev', run.call_args.args[0]) config = console_runner.load_config(str(APP_DIR / 'config.yaml')) gitlab_args = console_runner.build_args_from_source_config( 'gitlab', config['sources']['gitlab'], config['global'], 'fixture', ) github_args = console_runner.build_args_from_source_config( 'github', config['sources']['github'], config['global'], 'fixture', ) with mock.patch('builtins.print'): gitlab_options = console_runner.prepare_scan_options(gitlab_args, 1) github_options = console_runner.prepare_scan_options(github_args, 1) self.assertIs(gitlab_options['external_trufflehog_lifecycle'], True) self.assertNotIn('external_trufflehog_lifecycle', github_options) class GitLabLifecyclePolicyTests(unittest.TestCase): @staticmethod def message(value): return json.dumps({'level': 'info-0', 'msg': value}) def apply(self, stderr, returncode, findings=None): return scanner.apply_trufflehog_diagnostics( {'findings': list(findings or []), 'errors': []}, stderr, returncode, 'git', require_completion=True, ) def test_completion_and_incomplete_outcomes_are_explicit(self): complete = self.apply(self.message('finished scanning'), 0) self.assertEqual(complete['errors'], []) self.assertTrue(complete['scan_meta']['trufflehog_finished']) incomplete = self.apply(self.message('running source'), 0) self.assertEqual(incomplete['error_class'], 'command_incomplete') self.assertTrue(incomplete['retryable']) self.assertFalse(incomplete['scan_meta']['trufflehog_finished']) wrapper = self.apply(self.message('finished scanning'), 1) self.assertEqual(wrapper['error_class'], 'wrapper_exit') self.assertTrue(wrapper['retryable']) def test_partial_findings_use_bounded_queue_policy(self): finding = {'DetectorName': 'Example', 'Raw': 'fixture-secret'} result = self.apply(self.message('running source'), 1, [finding]) self.assertEqual(result['findings'], [finding]) self.assertEqual(result['error_class'], 'command_incomplete') self.assertTrue(result['retryable']) args = SimpleNamespace( target_retry_max_attempts=3, target_retry_base_delay_sec=60, target_retry_max_delay_sec=3600, target_timeout_retry_delay_sec=21600, ) first = console_runner.queue_error_disposition( None, 'gitlab', 'gitlab', 'target', result, args, {'attempts': 1}, ) exhausted = console_runner.queue_error_disposition( None, 'gitlab', 'gitlab', 'target', result, args, {'attempts': 3}, ) self.assertEqual((first[0], first[2], first[3]), ('deferred', 1, 3)) self.assertIsNotNone(first[1]) self.assertEqual(exhausted, ('failed', None, 3, 3)) def test_partial_finding_identity_is_stable_within_scan_event(self): scan_result = { 'findings': [{'DetectorName': 'Example', 'Raw': 'fixture-secret'}], 'errors': [], } with mock.patch.object( scanner, 'scan_git_repo', side_effect=lambda *args, **kwargs: copy.deepcopy(scan_result), ): first = scanner.scan_target_result( 'https://gitlab.com/example/repo.git', 'gitlab', 'event-1', {'external_trufflehog_lifecycle': True}, ) second = scanner.scan_target_result( 'https://gitlab.com/example/repo.git', 'gitlab', 'event-1', {'external_trufflehog_lifecycle': True}, ) self.assertEqual( first['findings'][0]['finding_uid'], second['findings'][0]['finding_uid'], ) if __name__ == '__main__': unittest.main()