import contextlib from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import shutil import subprocess import sys import tempfile import threading from types import SimpleNamespace from unittest import mock import pytest ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'app')) import scanner from test_docker_staging_bounds import command_harness FINISHED = '{"level":"info-0","msg":"finished scanning"}' SPACE = 'fixture/long-paths' @pytest.fixture def hf_scan(monkeypatch): state = SimpleNamespace( environment={}, stdout='', stderr=FINISHED, returncode=0, run=mock.Mock(), filters=mock.Mock(side_effect=lambda result, *a, **kw: result), ) state.platform = SimpleNamespace(name='nt', environ=state.environment, getenv=state.environment.get) state.run.side_effect = lambda *a, **kw: scanner.streamed_output_from_text( state.stdout, state.stderr, state.returncode, ) monkeypatch.setattr(scanner, 'os', state.platform) monkeypatch.setattr(scanner, 'get_trufflehog_cmd', lambda: 'fixture-trufflehog') monkeypatch.setattr(scanner, 'run_command_streamed', state.run) monkeypatch.setattr(scanner, 'apply_finding_filters', state.filters) monkeypatch.setattr(scanner.scan_config, 'trufflehog_config', '') return state @pytest.mark.parametrize('platform', ['nt', 'posix']) @pytest.mark.parametrize('token', [None, 'hf_synthetic_explicit_token']) def test_hf_child_config_is_windows_only_and_preserves_caller_environment(hf_scan, platform, token): state = hf_scan state.platform.name = platform state.environment.update({ 'GIT_CONFIG_COUNT': '2', 'GIT_CONFIG_KEY_0': 'http.version', 'GIT_CONFIG_VALUE_0': 'HTTP/1.1', 'GIT_CONFIG_KEY_1': 'core.longpaths', 'GIT_CONFIG_VALUE_1': 'false', 'GIT_CONFIG_PARAMETERS': "'color.ui=never'", 'GIT_ASKPASS': 'synthetic-askpass.cmd', 'GIT_TERMINAL_PROMPT': '0', 'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user', 'HUGGINGFACE_TOKEN': 'synthetic-inherited-hf', 'HF_TOKEN': 'synthetic-inherited-hf-alias', 'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'UNRELATED_SETTING': 'kept', }) before = dict(state.environment) state.stdout = json.dumps({'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'}) result = scanner.scan_huggingface_space( SPACE, 23, 'FixtureDetector', 'ExcludedFixture', True, 'fixture-policy.yaml', token, ) command, timeout, environment = state.run.call_args.args assert command == [ 'fixture-trufflehog', 'huggingface', '--space', SPACE, '--json', '--no-update', '--config', 'fixture-policy.yaml', '--include-detectors', 'FixtureDetector', '--exclude-detectors', 'ExcludedFixture', '--no-verification', ] expected = dict(before) if token: expected.update(HUGGINGFACE_TOKEN=token, HF_TOKEN=token) assert token not in command if platform == 'nt': expected.update(GIT_CONFIG_COUNT='3', GIT_CONFIG_KEY_2='core.longpaths', GIT_CONFIG_VALUE_2='true') assert environment == expected assert environment is not state.environment assert state.environment == before assert timeout == 23 assert state.run.call_args.kwargs == {} assert result['findings'] == [{'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'}] assert result['errors'] == [] assert result['scan_meta']['trufflehog_finished'] state.filters.assert_called_once_with(result, SPACE) @pytest.mark.parametrize('count', [None, '', '0', '002', '255']) def test_hf_appends_one_bounded_git_config_entry(hf_scan, count): environment = hf_scan.environment if count is not None: environment['GIT_CONFIG_COUNT'] = count existing = int(count or '0') for index in range(existing): environment[f'GIT_CONFIG_KEY_{index}'] = f'fixture.value{index}' environment[f'GIT_CONFIG_VALUE_{index}'] = str(index) before = dict(environment) scanner.scan_huggingface_space(SPACE) child = hf_scan.run.call_args.args[2] assert child['GIT_CONFIG_COUNT'] == str(existing + 1) assert child[f'GIT_CONFIG_KEY_{existing}'] == 'core.longpaths' assert child[f'GIT_CONFIG_VALUE_{existing}'] == 'true' assert all(child[key] == value for key, value in before.items() if key != 'GIT_CONFIG_COUNT') assert environment == before @pytest.mark.parametrize('count', ['-1', 'one', '1.0', ' 1', '256', '999', '9' * 5000, '\u0661'], ids=['negative', 'text', 'fraction', 'space', 'over-limit', 'large', 'oversized', 'unicode']) def test_hf_rejects_invalid_or_excessive_git_config_count_before_launch(hf_scan, count): hf_scan.environment['GIT_CONFIG_COUNT'] = count with pytest.raises(ValueError, match='HuggingFace GIT_CONFIG_COUNT'): scanner.scan_huggingface_space(SPACE) hf_scan.run.assert_not_called() assert hf_scan.environment['GIT_CONFIG_COUNT'] == count @pytest.mark.parametrize('cause', [ "error: unable to create file fixture.txt: Filename too long", "error: invalid path 'invalid:name.txt'", "error: invalid path 'con.txt'", 'fatal: unspecified checkout failure', ]) def test_hf_zero_exit_and_completion_never_hide_checkout_errors(hf_scan, cause): token = 'hf_synthetic_redacted_token' hf_scan.stderr = json.dumps({ 'level': 'error', 'msg': 'error processing repository', 'error': 'error executing git clone: exit status 128, ' + cause + '\nwarning: Clone succeeded, but checkout failed.\n' + token, }) + '\n' + FINISHED result = scanner.scan_huggingface_space(SPACE, token=token) assert len(result['errors']) == 1 assert cause in result['errors'][0] assert token not in json.dumps(result) assert result['error_class'] == 'trufflehog' assert result['source_failure'] is False assert result['retryable'] is True assert not result.get('skipped') assert result['scan_meta']['trufflehog_returncode'] == 0 assert result['scan_meta']['trufflehog_finished'] is True assert 'git_checkout_recovery' not in result['scan_meta'] hf_scan.run.assert_called_once() @pytest.mark.parametrize('stderr,returncode', [(FINISHED, 1), ('', 1), ('Command timed out', -1)]) def test_hf_keeps_nonzero_exit_and_timeout_errors(hf_scan, stderr, returncode): hf_scan.stderr, hf_scan.returncode = stderr, returncode result = scanner.scan_huggingface_space(SPACE) assert result['errors'] assert result['scan_meta']['trufflehog_returncode'] == returncode assert result['scan_meta']['trufflehog_finished'] == (stderr == FINISHED) hf_scan.run.assert_called_once() @pytest.mark.skipif(os.name != 'nt', reason='Windows owned-command environment contract') def test_hf_config_survives_runner_filter_without_losing_askpass_or_authority(command_harness, monkeypatch): state = command_harness state.completed = True environment = { 'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.protectNTFS', 'GIT_CONFIG_VALUE_0': 'true', 'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user', 'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'TRUF_POSTGRES_PASSWORD': 'synthetic-password', 'TRUF_MANAGED_POSTGRES_DSN': 'synthetic-dsn', } before = dict(environment) guard = mock.Mock() monkeypatch.setattr(scanner.os, 'environ', environment) monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority', guard) monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, state.slot)) monkeypatch.setattr(scanner, 'harden_private_file', lambda path: None) scanner.scan_huggingface_space(SPACE, token='hf_synthetic_token') child = state.options['env'] assert child['GIT_CONFIG_COUNT'] == '2' assert child['GIT_CONFIG_KEY_0'] == 'core.protectNTFS' assert child['GIT_CONFIG_VALUE_0'] == 'true' assert child['GIT_CONFIG_KEY_1'] == 'core.longpaths' assert child['GIT_CONFIG_VALUE_1'] == 'true' assert child['HF_TOKEN'] == child['HUGGINGFACE_TOKEN'] == 'hf_synthetic_token' assert child['TRUF_GIT_TOKEN'] == 'synthetic-git-token' assert child['TRUF_GIT_USERNAME'] == 'fixture-user' assert child['GIT_ASKPASS'] == str(state.command_dir / 'git-askpass.cmd') assert '%TRUF_GIT_TOKEN%' in (state.command_dir / 'git-askpass.cmd').read_text(encoding='ascii') assert child['GIT_TERMINAL_PROMPT'] == '0' assert child['TEMP'] == child['TMP'] == child['TMPDIR'] == str(state.command_dir) assert not {'TRUF_SUPERVISOR_TOKEN', 'TRUF_POSTGRES_PASSWORD', 'TRUF_MANAGED_POSTGRES_DSN'} & child.keys() assert environment == before assert state.options['job_memory_limit_bytes'] == 4 * 1024 ** 3 guard.assert_called_once() assert guard.call_args.args[0][1:4] == ['huggingface', '--space', SPACE] state.slot.set_child_pid.assert_called_once_with(41) state.slot.release.assert_not_called() def test_hf_does_not_bypass_launch_authority(command_harness, monkeypatch): launch = mock.Mock(side_effect=AssertionError('must not launch without authority')) monkeypatch.setattr(scanner, 'OwnedProcess', launch) monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority', mock.Mock(side_effect=RuntimeError('fixture authority denied'))) with pytest.raises(RuntimeError, match='fixture authority denied'): scanner.scan_huggingface_space(SPACE) launch.assert_not_called() @pytest.fixture def windows_git_fixture(): git_executable = shutil.which('git') if os.name != 'nt' or not git_executable: pytest.skip('requires installed Windows Git') # Python cleanup needs extended paths too; Git below still receives ordinary absolute paths. with tempfile.TemporaryDirectory(prefix='hf-long-', dir='\\\\?\\' + str(ROOT / 'tmp')) as extended_dir: temp_dir = extended_dir[4:] root = Path(temp_dir) home = root / 'home' home.mkdir() environment = {key: value for key, value in os.environ.items() if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'COMSPEC', 'PATHEXT'}} environment.update( TMP=temp_dir, TEMP=temp_dir, TMPDIR=temp_dir, HOME=str(home), USERPROFILE=str(home), XDG_CONFIG_HOME=str(home), GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_SYSTEM=os.devnull, GIT_ATTR_NOSYSTEM='1', GIT_TERMINAL_PROMPT='0', GIT_ALLOW_PROTOCOL='file', ) config = [('core.longpaths', 'false'), ('core.protectNTFS', 'true'), ('core.hooksPath', str(home)), ('init.templateDir', str(home))] environment['GIT_CONFIG_COUNT'] = str(len(config)) for index, (key, value) in enumerate(config): environment[f'GIT_CONFIG_KEY_{index}'] = key environment[f'GIT_CONFIG_VALUE_{index}'] = value def git(*args, env=None, data=None, check=True): completed = subprocess.run( [git_executable, *map(str, args)], input=data, capture_output=True, env=environment if env is None else env, cwd=root, timeout=30, ) if check: assert completed.returncode == 0, completed.stderr.decode('utf-8', errors='replace') return completed path208 = '/'.join(['a' * 64, 'b' * 63, 'c' * 75 + '.txt']) path240 = '/'.join(['d' * 49, 'e' * 51, 'f' * 134 + '.txt']) deeper = '/'.join(['depth_' + 'g' * 18] * 12 + ['deep.txt']) paths = ['README.txt', path208, path240, deeper] markers = dict(zip(paths, [ 'HFLONGPATHFIXTURE_SHORTAAA_1001', 'HFLONGPATHFIXTURE_LONGTWOA_1002', 'HFLONGPATHFIXTURE_LONGTWOB_1003', 'HFLONGPATHFIXTURE_DEEPFILE_1004', ])) assert (len(path208), max(map(len, path208.split('/')))) == (208, 79) assert (len(path240), max(map(len, path240.split('/')))) == (240, 138) assert all(max(map(len, path.split('/'))) < 255 for path in paths) repositories = {} for name, members in [('long', paths), ('short', paths[:1])]: repository = root / (name + '.git') git('init', '--bare', '--quiet', f'--template={home}', repository) stream = bytearray(b'commit refs/heads/main\ncommitter Fixture ' b'1700000000 +0000\ndata 7\nfixture\n') for path in members: content = (markers[path] + '\n').encode('ascii') stream.extend(f'M 100644 inline {json.dumps(path)}\ndata {len(content)}\n'.encode('ascii')) stream.extend(content + b'\n') git('-C', repository, 'fast-import', '--quiet', data=bytes(stream) + b'\ndone\n') git('-C', repository, 'symbolic-ref', 'HEAD', 'refs/heads/main') repositories[name] = repository yield SimpleNamespace(root=root, home=home, env=environment, git=git, paths=paths, markers=markers, repositories=repositories, executable=git_executable) def hf_child_environment(environment): with mock.patch.object(scanner.os, 'environ', environment), \ mock.patch.object(scanner, 'run_command_streamed', return_value=scanner.streamed_output_from_text('', FINISHED, 0)) as command, \ mock.patch.object(scanner, 'apply_finding_filters', side_effect=lambda result, *a: result): scanner.scan_huggingface_space(SPACE) return command.call_args.args[2] def test_native_windows_git_clone_checkout_and_deeper_files(windows_git_fixture): fixture = windows_git_fixture fixed_env = hf_child_environment(fixture.env) for enabled, environment in [(False, fixture.env), (True, fixed_env)]: destination = fixture.root / (('enabled-' if enabled else 'disabled-') + 'x' * 32) read_root = Path('\\\\?\\' + str(destination)) assert all(len(str(destination / path)) > 260 for path in fixture.paths[1:]) clone = fixture.git('clone', '--', fixture.repositories['long'], destination, env=environment, check=False) stderr = clone.stderr.decode('utf-8', errors='replace') assert (read_root / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n' if enabled: assert clone.returncode == 0, stderr assert 'Filename too long' not in stderr for path, marker in fixture.markers.items(): assert (read_root / path).read_text(encoding='ascii') == marker + '\n' tracked = fixture.git('-C', destination, 'ls-files', env=environment).stdout.decode().splitlines() assert set(tracked) == set(fixture.paths) else: assert clone.returncode != 0 assert 'Filename too long' in stderr assert 'Clone succeeded, but checkout failed' in stderr print('native clone', json.dumps({'longpaths': enabled, 'returncode': clone.returncode, 'relative_lengths': list(map(len, fixture.paths)), 'absolute_lengths': [len(str(destination / path)) for path in fixture.paths]})) destination = fixture.root / ('checkout-' + 'y' * 32) fixture.git('clone', '--no-checkout', '--', fixture.repositories['long'], destination) failed = fixture.git('-C', destination, 'checkout', '--force', 'HEAD', check=False) assert failed.returncode != 0 assert b'Filename too long' in failed.stderr fixture.git('-C', destination, 'checkout', '--force', 'HEAD', env=fixed_env) read_root = Path('\\\\?\\' + str(destination)) for path, marker in fixture.markers.items(): assert (read_root / path).read_text(encoding='ascii') == marker + '\n' assert fixture.git('config', '--get', 'core.longpaths').stdout.strip() == b'false' assert fixture.git('config', '--get', 'core.longpaths', env=fixed_env).stdout.strip() == b'true' assert fixture.git('config', '--get', 'core.protectNTFS', env=fixed_env).stdout.strip() == b'true' @pytest.mark.parametrize('enabled', [False, True]) def test_native_windows_git_short_normal_path_is_unchanged(windows_git_fixture, enabled): fixture = windows_git_fixture environment = hf_child_environment(fixture.env) if enabled else fixture.env destination = fixture.root / 'short-checkout' assert len(str(destination / 'README.txt')) < 260 fixture.git('clone', '--', fixture.repositories['short'], destination, env=environment) assert (destination / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n' @pytest.mark.parametrize('enabled', [False, True]) def test_installed_hf_uses_native_git_and_reads_all_fixture_paths(windows_git_fixture, monkeypatch, enabled): fixture = windows_git_fixture executable = Path(r'C:\Tools\trufflehog.exe') if not executable.is_file(): pytest.skip('requires installed TruffleHog at C:\\Tools\\trufflehog.exe') requests = [] class FixtureAPI(BaseHTTPRequestHandler): def do_GET(self): requests.append(self.path) if self.path.split('?', 1)[0] != '/api/spaces/' + SPACE: self.send_error(404) return body = json.dumps({'id': SPACE, 'private': False, 'gated': False, 'disabled': False, 'siblings': [{'rfilename': path} for path in fixture.paths]}).encode('ascii') self.send_response(200) self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass server = ThreadingHTTPServer(('127.0.0.1', 0), FixtureAPI) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: endpoint = f'http://127.0.0.1:{server.server_port}' environment = dict(fixture.env) count = int(environment['GIT_CONFIG_COUNT']) for suffix in ('', '.git'): environment[f'GIT_CONFIG_KEY_{count}'] = f'url.{fixture.repositories["long"].as_uri()}.insteadOf' environment[f'GIT_CONFIG_VALUE_{count}'] = endpoint + '/spaces/' + SPACE + suffix count += 1 environment['GIT_CONFIG_COUNT'] = str(count) trace = fixture.root / 'git-trace.jsonl' environment['GIT_TRACE2_EVENT'] = str(trace) environment['GIT_TRACE2_CONFIG_PARAMS'] = 'core.longpaths,core.protectNTFS' policy = fixture.root / 'marker.yaml' policy.write_text( 'detectors:\n - name: OfflineHFLongPaths\n keywords: [HFLONGPATHFIXTURE]\n' " regex:\n marker: 'HFLONGPATHFIXTURE_[A-Z]{8}_[0-9]{4}'\n", encoding='ascii', ) command_root = fixture.root / 'commands' scanner.ensure_private_directory(str(command_root)) metadata = {'code_manifest': {'executables': { 'git': {'path': fixture.executable}, 'trufflehog': {'path': str(executable)}, }, 'assets': {str(policy): {'path': str(policy)}}}} # Only bootstrap authority/lease are synthetic; runner, argv guard and OwnedProcess stay real. monkeypatch.setattr(scanner, 'require_active_supervisor_child', lambda **kw: metadata) monkeypatch.setattr(scanner.os, 'environ', environment) monkeypatch.setattr(scanner, '_runtime_initialized', True) monkeypatch.setattr(scanner.scan_config, 'work_dir', str(command_root)) monkeypatch.setattr(scanner.scan_config, 'trufflehog_path', str(executable)) monkeypatch.setattr(scanner.scan_config, 'min_free_gb', 0) monkeypatch.setattr(scanner.scan_config, 'trufflehog_job_memory_limit_bytes', 2 * 1024 ** 3) slot = mock.Mock(releasable=True) monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, slot)) original = scanner.run_command_streamed @contextlib.contextmanager def local_command(command, timeout, env): child_env = dict(env) if not enabled: # Negative control: ignore the appended setting, retaining inherited core.longpaths=false. child_env['GIT_CONFIG_COUNT'] = environment['GIT_CONFIG_COUNT'] with original([*command, '--endpoint', endpoint, '--concurrency', '1'], timeout, child_env) as output: yield output monkeypatch.setattr(scanner, 'run_command_streamed', local_command) before = dict(environment) result = scanner.scan_huggingface_space(SPACE, 60, no_verification=True, trufflehog_config=str(policy)) assert environment == before events = [json.loads(line) for line in trace.read_text(encoding='utf-8').splitlines()] clones = [event for event in events if event.get('event') == 'start' and 'clone' in event.get('argv', [])] assert clones, 'installed HF must delegate cloning to native Git' assert all('--no-checkout' not in event['argv'] for event in clones) assert any(event.get('param') == 'core.longpaths' and event.get('value') == str(enabled).lower() for event in events) seen = {path for path, marker in fixture.markers.items() if any(marker in str(finding.get('Raw', '')) + str(finding.get('RawV2', '')) for finding in result['findings'])} assert result['scan_meta']['trufflehog_returncode'] == 0 assert result['scan_meta']['trufflehog_finished'] is True if enabled: assert result['errors'] == [], result['errors'] assert seen == set(fixture.paths) else: assert result['errors'] assert 'Filename too long' in '\n'.join(result['errors']) assert 'Clone succeeded, but checkout failed' in '\n'.join(result['errors']) assert result['source_failure'] is False assert not result.get('skipped') slot.set_child_pid.assert_called_once() slot.release.assert_not_called() assert not list(command_root.iterdir()), 'owned HF command directory must be cleaned' print('installed HF', json.dumps({'longpaths': enabled, 'native_clones': len(clones), 'finished': result['scan_meta']['trufflehog_finished'], 'returncode': 0, 'marker_files': len(seen), 'errors': len(result['errors']), 'loopback_requests': requests})) finally: server.shutdown() server.server_close() thread.join(timeout=5)