sashatrask/truf-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Initial server source import

Browse Source
This commit is contained in:
sashatrask
2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_huggingface_long_paths.py
+427
View File
@@ -0,0 +1,427 @@
import contextlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import threading
from types import SimpleNamespace
from unittest import mock
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'app'))
import scanner
from test_docker_staging_bounds import command_harness
FINISHED = '{"level":"info-0","msg":"finished scanning"}'
SPACE = 'fixture/long-paths'
@pytest.fixture
def hf_scan(monkeypatch):
state = SimpleNamespace(
environment={}, stdout='', stderr=FINISHED, returncode=0,
run=mock.Mock(), filters=mock.Mock(side_effect=lambda result, *a, **kw: result),
)
state.platform = SimpleNamespace(name='nt', environ=state.environment, getenv=state.environment.get)
state.run.side_effect = lambda *a, **kw: scanner.streamed_output_from_text(
state.stdout, state.stderr, state.returncode,
)
monkeypatch.setattr(scanner, 'os', state.platform)
monkeypatch.setattr(scanner, 'get_trufflehog_cmd', lambda: 'fixture-trufflehog')
monkeypatch.setattr(scanner, 'run_command_streamed', state.run)
monkeypatch.setattr(scanner, 'apply_finding_filters', state.filters)
monkeypatch.setattr(scanner.scan_config, 'trufflehog_config', '')
return state
@pytest.mark.parametrize('platform', ['nt', 'posix'])
@pytest.mark.parametrize('token', [None, 'hf_synthetic_explicit_token'])
def test_hf_child_config_is_windows_only_and_preserves_caller_environment(hf_scan, platform, token):
state = hf_scan
state.platform.name = platform
state.environment.update({
'GIT_CONFIG_COUNT': '2',
'GIT_CONFIG_KEY_0': 'http.version', 'GIT_CONFIG_VALUE_0': 'HTTP/1.1',
'GIT_CONFIG_KEY_1': 'core.longpaths', 'GIT_CONFIG_VALUE_1': 'false',
'GIT_CONFIG_PARAMETERS': "'color.ui=never'",
'GIT_ASKPASS': 'synthetic-askpass.cmd', 'GIT_TERMINAL_PROMPT': '0',
'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user',
'HUGGINGFACE_TOKEN': 'synthetic-inherited-hf', 'HF_TOKEN': 'synthetic-inherited-hf-alias',
'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'UNRELATED_SETTING': 'kept',
})
before = dict(state.environment)
state.stdout = json.dumps({'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'})
result = scanner.scan_huggingface_space(
SPACE, 23, 'FixtureDetector', 'ExcludedFixture', True, 'fixture-policy.yaml', token,
)
command, timeout, environment = state.run.call_args.args
assert command == [
'fixture-trufflehog', 'huggingface', '--space', SPACE, '--json', '--no-update',
'--config', 'fixture-policy.yaml', '--include-detectors', 'FixtureDetector',
'--exclude-detectors', 'ExcludedFixture', '--no-verification',
]
expected = dict(before)
if token:
expected.update(HUGGINGFACE_TOKEN=token, HF_TOKEN=token)
assert token not in command
if platform == 'nt':
expected.update(GIT_CONFIG_COUNT='3', GIT_CONFIG_KEY_2='core.longpaths', GIT_CONFIG_VALUE_2='true')
assert environment == expected
assert environment is not state.environment
assert state.environment == before
assert timeout == 23
assert state.run.call_args.kwargs == {}
assert result['findings'] == [{'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'}]
assert result['errors'] == []
assert result['scan_meta']['trufflehog_finished']
state.filters.assert_called_once_with(result, SPACE)
@pytest.mark.parametrize('count', [None, '', '0', '002', '255'])
def test_hf_appends_one_bounded_git_config_entry(hf_scan, count):
environment = hf_scan.environment
if count is not None:
environment['GIT_CONFIG_COUNT'] = count
existing = int(count or '0')
for index in range(existing):
environment[f'GIT_CONFIG_KEY_{index}'] = f'fixture.value{index}'
environment[f'GIT_CONFIG_VALUE_{index}'] = str(index)
before = dict(environment)
scanner.scan_huggingface_space(SPACE)
child = hf_scan.run.call_args.args[2]
assert child['GIT_CONFIG_COUNT'] == str(existing + 1)
assert child[f'GIT_CONFIG_KEY_{existing}'] == 'core.longpaths'
assert child[f'GIT_CONFIG_VALUE_{existing}'] == 'true'
assert all(child[key] == value for key, value in before.items() if key != 'GIT_CONFIG_COUNT')
assert environment == before
@pytest.mark.parametrize('count', ['-1', 'one', '1.0', ' 1', '256', '999', '9' * 5000, '\u0661'],
ids=['negative', 'text', 'fraction', 'space', 'over-limit', 'large', 'oversized', 'unicode'])
def test_hf_rejects_invalid_or_excessive_git_config_count_before_launch(hf_scan, count):
hf_scan.environment['GIT_CONFIG_COUNT'] = count
with pytest.raises(ValueError, match='HuggingFace GIT_CONFIG_COUNT'):
scanner.scan_huggingface_space(SPACE)
hf_scan.run.assert_not_called()
assert hf_scan.environment['GIT_CONFIG_COUNT'] == count
@pytest.mark.parametrize('cause', [
"error: unable to create file fixture.txt: Filename too long",
"error: invalid path 'invalid:name.txt'",
"error: invalid path 'con.txt'",
'fatal: unspecified checkout failure',
])
def test_hf_zero_exit_and_completion_never_hide_checkout_errors(hf_scan, cause):
token = 'hf_synthetic_redacted_token'
hf_scan.stderr = json.dumps({
'level': 'error', 'msg': 'error processing repository',
'error': 'error executing git clone: exit status 128, ' + cause
+ '\nwarning: Clone succeeded, but checkout failed.\n' + token,
}) + '\n' + FINISHED
result = scanner.scan_huggingface_space(SPACE, token=token)
assert len(result['errors']) == 1
assert cause in result['errors'][0]
assert token not in json.dumps(result)
assert result['error_class'] == 'trufflehog'
assert result['source_failure'] is False
assert result['retryable'] is True
assert not result.get('skipped')
assert result['scan_meta']['trufflehog_returncode'] == 0
assert result['scan_meta']['trufflehog_finished'] is True
assert 'git_checkout_recovery' not in result['scan_meta']
hf_scan.run.assert_called_once()
@pytest.mark.parametrize('stderr,returncode', [(FINISHED, 1), ('', 1), ('Command timed out', -1)])
def test_hf_keeps_nonzero_exit_and_timeout_errors(hf_scan, stderr, returncode):
hf_scan.stderr, hf_scan.returncode = stderr, returncode
result = scanner.scan_huggingface_space(SPACE)
assert result['errors']
assert result['scan_meta']['trufflehog_returncode'] == returncode
assert result['scan_meta']['trufflehog_finished'] == (stderr == FINISHED)
hf_scan.run.assert_called_once()
@pytest.mark.skipif(os.name != 'nt', reason='Windows owned-command environment contract')
def test_hf_config_survives_runner_filter_without_losing_askpass_or_authority(command_harness, monkeypatch):
state = command_harness
state.completed = True
environment = {
'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.protectNTFS', 'GIT_CONFIG_VALUE_0': 'true',
'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user',
'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'TRUF_POSTGRES_PASSWORD': 'synthetic-password',
'TRUF_MANAGED_POSTGRES_DSN': 'synthetic-dsn',
}
before = dict(environment)
guard = mock.Mock()
monkeypatch.setattr(scanner.os, 'environ', environment)
monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority', guard)
monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, state.slot))
monkeypatch.setattr(scanner, 'harden_private_file', lambda path: None)
scanner.scan_huggingface_space(SPACE, token='hf_synthetic_token')
child = state.options['env']
assert child['GIT_CONFIG_COUNT'] == '2'
assert child['GIT_CONFIG_KEY_0'] == 'core.protectNTFS'
assert child['GIT_CONFIG_VALUE_0'] == 'true'
assert child['GIT_CONFIG_KEY_1'] == 'core.longpaths'
assert child['GIT_CONFIG_VALUE_1'] == 'true'
assert child['HF_TOKEN'] == child['HUGGINGFACE_TOKEN'] == 'hf_synthetic_token'
assert child['TRUF_GIT_TOKEN'] == 'synthetic-git-token'
assert child['TRUF_GIT_USERNAME'] == 'fixture-user'
assert child['GIT_ASKPASS'] == str(state.command_dir / 'git-askpass.cmd')
assert '%TRUF_GIT_TOKEN%' in (state.command_dir / 'git-askpass.cmd').read_text(encoding='ascii')
assert child['GIT_TERMINAL_PROMPT'] == '0'
assert child['TEMP'] == child['TMP'] == child['TMPDIR'] == str(state.command_dir)
assert not {'TRUF_SUPERVISOR_TOKEN', 'TRUF_POSTGRES_PASSWORD', 'TRUF_MANAGED_POSTGRES_DSN'} & child.keys()
assert environment == before
assert state.options['job_memory_limit_bytes'] == 4 * 1024 ** 3
guard.assert_called_once()
assert guard.call_args.args[0][1:4] == ['huggingface', '--space', SPACE]
state.slot.set_child_pid.assert_called_once_with(41)
state.slot.release.assert_not_called()
def test_hf_does_not_bypass_launch_authority(command_harness, monkeypatch):
launch = mock.Mock(side_effect=AssertionError('must not launch without authority'))
monkeypatch.setattr(scanner, 'OwnedProcess', launch)
monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority',
mock.Mock(side_effect=RuntimeError('fixture authority denied')))
with pytest.raises(RuntimeError, match='fixture authority denied'):
scanner.scan_huggingface_space(SPACE)
launch.assert_not_called()
@pytest.fixture
def windows_git_fixture():
git_executable = shutil.which('git')
if os.name != 'nt' or not git_executable:
pytest.skip('requires installed Windows Git')
# Python cleanup needs extended paths too; Git below still receives ordinary absolute paths.
with tempfile.TemporaryDirectory(prefix='hf-long-', dir='\\\\?\\' + str(ROOT / 'tmp')) as extended_dir:
temp_dir = extended_dir[4:]
root = Path(temp_dir)
home = root / 'home'
home.mkdir()
environment = {key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'COMSPEC', 'PATHEXT'}}
environment.update(
TMP=temp_dir, TEMP=temp_dir, TMPDIR=temp_dir, HOME=str(home), USERPROFILE=str(home),
XDG_CONFIG_HOME=str(home), GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull,
GIT_CONFIG_SYSTEM=os.devnull, GIT_ATTR_NOSYSTEM='1', GIT_TERMINAL_PROMPT='0',
GIT_ALLOW_PROTOCOL='file',
)
config = [('core.longpaths', 'false'), ('core.protectNTFS', 'true'),
('core.hooksPath', str(home)), ('init.templateDir', str(home))]
environment['GIT_CONFIG_COUNT'] = str(len(config))
for index, (key, value) in enumerate(config):
environment[f'GIT_CONFIG_KEY_{index}'] = key
environment[f'GIT_CONFIG_VALUE_{index}'] = value
def git(*args, env=None, data=None, check=True):
completed = subprocess.run(
[git_executable, *map(str, args)], input=data, capture_output=True,
env=environment if env is None else env, cwd=root, timeout=30,
)
if check:
assert completed.returncode == 0, completed.stderr.decode('utf-8', errors='replace')
return completed
path208 = '/'.join(['a' * 64, 'b' * 63, 'c' * 75 + '.txt'])
path240 = '/'.join(['d' * 49, 'e' * 51, 'f' * 134 + '.txt'])
deeper = '/'.join(['depth_' + 'g' * 18] * 12 + ['deep.txt'])
paths = ['README.txt', path208, path240, deeper]
markers = dict(zip(paths, [
'HFLONGPATHFIXTURE_SHORTAAA_1001', 'HFLONGPATHFIXTURE_LONGTWOA_1002',
'HFLONGPATHFIXTURE_LONGTWOB_1003', 'HFLONGPATHFIXTURE_DEEPFILE_1004',
]))
assert (len(path208), max(map(len, path208.split('/')))) == (208, 79)
assert (len(path240), max(map(len, path240.split('/')))) == (240, 138)
assert all(max(map(len, path.split('/'))) < 255 for path in paths)
repositories = {}
for name, members in [('long', paths), ('short', paths[:1])]:
repository = root / (name + '.git')
git('init', '--bare', '--quiet', f'--template={home}', repository)
stream = bytearray(b'commit refs/heads/main\ncommitter Fixture <fixture@example.test> '
b'1700000000 +0000\ndata 7\nfixture\n')
for path in members:
content = (markers[path] + '\n').encode('ascii')
stream.extend(f'M 100644 inline {json.dumps(path)}\ndata {len(content)}\n'.encode('ascii'))
stream.extend(content + b'\n')
git('-C', repository, 'fast-import', '--quiet', data=bytes(stream) + b'\ndone\n')
git('-C', repository, 'symbolic-ref', 'HEAD', 'refs/heads/main')
repositories[name] = repository
yield SimpleNamespace(root=root, home=home, env=environment, git=git, paths=paths,
markers=markers, repositories=repositories, executable=git_executable)
def hf_child_environment(environment):
with mock.patch.object(scanner.os, 'environ', environment), \
mock.patch.object(scanner, 'run_command_streamed',
return_value=scanner.streamed_output_from_text('', FINISHED, 0)) as command, \
mock.patch.object(scanner, 'apply_finding_filters', side_effect=lambda result, *a: result):
scanner.scan_huggingface_space(SPACE)
return command.call_args.args[2]
def test_native_windows_git_clone_checkout_and_deeper_files(windows_git_fixture):
fixture = windows_git_fixture
fixed_env = hf_child_environment(fixture.env)
for enabled, environment in [(False, fixture.env), (True, fixed_env)]:
destination = fixture.root / (('enabled-' if enabled else 'disabled-') + 'x' * 32)
read_root = Path('\\\\?\\' + str(destination))
assert all(len(str(destination / path)) > 260 for path in fixture.paths[1:])
clone = fixture.git('clone', '--', fixture.repositories['long'], destination, env=environment, check=False)
stderr = clone.stderr.decode('utf-8', errors='replace')
assert (read_root / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n'
if enabled:
assert clone.returncode == 0, stderr
assert 'Filename too long' not in stderr
for path, marker in fixture.markers.items():
assert (read_root / path).read_text(encoding='ascii') == marker + '\n'
tracked = fixture.git('-C', destination, 'ls-files', env=environment).stdout.decode().splitlines()
assert set(tracked) == set(fixture.paths)
else:
assert clone.returncode != 0
assert 'Filename too long' in stderr
assert 'Clone succeeded, but checkout failed' in stderr
print('native clone', json.dumps({'longpaths': enabled, 'returncode': clone.returncode,
'relative_lengths': list(map(len, fixture.paths)),
'absolute_lengths': [len(str(destination / path)) for path in fixture.paths]}))
destination = fixture.root / ('checkout-' + 'y' * 32)
fixture.git('clone', '--no-checkout', '--', fixture.repositories['long'], destination)
failed = fixture.git('-C', destination, 'checkout', '--force', 'HEAD', check=False)
assert failed.returncode != 0
assert b'Filename too long' in failed.stderr
fixture.git('-C', destination, 'checkout', '--force', 'HEAD', env=fixed_env)
read_root = Path('\\\\?\\' + str(destination))
for path, marker in fixture.markers.items():
assert (read_root / path).read_text(encoding='ascii') == marker + '\n'
assert fixture.git('config', '--get', 'core.longpaths').stdout.strip() == b'false'
assert fixture.git('config', '--get', 'core.longpaths', env=fixed_env).stdout.strip() == b'true'
assert fixture.git('config', '--get', 'core.protectNTFS', env=fixed_env).stdout.strip() == b'true'
@pytest.mark.parametrize('enabled', [False, True])
def test_native_windows_git_short_normal_path_is_unchanged(windows_git_fixture, enabled):
fixture = windows_git_fixture
environment = hf_child_environment(fixture.env) if enabled else fixture.env
destination = fixture.root / 'short-checkout'
assert len(str(destination / 'README.txt')) < 260
fixture.git('clone', '--', fixture.repositories['short'], destination, env=environment)
assert (destination / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n'
@pytest.mark.parametrize('enabled', [False, True])
def test_installed_hf_uses_native_git_and_reads_all_fixture_paths(windows_git_fixture, monkeypatch, enabled):
fixture = windows_git_fixture
executable = Path(r'C:\Tools\trufflehog.exe')
if not executable.is_file():
pytest.skip('requires installed TruffleHog at C:\\Tools\\trufflehog.exe')
requests = []
class FixtureAPI(BaseHTTPRequestHandler):
def do_GET(self):
requests.append(self.path)
if self.path.split('?', 1)[0] != '/api/spaces/' + SPACE:
self.send_error(404)
return
body = json.dumps({'id': SPACE, 'private': False, 'gated': False, 'disabled': False,
'siblings': [{'rfilename': path} for path in fixture.paths]}).encode('ascii')
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
server = ThreadingHTTPServer(('127.0.0.1', 0), FixtureAPI)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
endpoint = f'http://127.0.0.1:{server.server_port}'
environment = dict(fixture.env)
count = int(environment['GIT_CONFIG_COUNT'])
for suffix in ('', '.git'):
environment[f'GIT_CONFIG_KEY_{count}'] = f'url.{fixture.repositories["long"].as_uri()}.insteadOf'
environment[f'GIT_CONFIG_VALUE_{count}'] = endpoint + '/spaces/' + SPACE + suffix
count += 1
environment['GIT_CONFIG_COUNT'] = str(count)
trace = fixture.root / 'git-trace.jsonl'
environment['GIT_TRACE2_EVENT'] = str(trace)
environment['GIT_TRACE2_CONFIG_PARAMS'] = 'core.longpaths,core.protectNTFS'
policy = fixture.root / 'marker.yaml'
policy.write_text(
'detectors:\n - name: OfflineHFLongPaths\n keywords: [HFLONGPATHFIXTURE]\n'
" regex:\n marker: 'HFLONGPATHFIXTURE_[A-Z]{8}_[0-9]{4}'\n", encoding='ascii',
)
command_root = fixture.root / 'commands'
scanner.ensure_private_directory(str(command_root))
metadata = {'code_manifest': {'executables': {
'git': {'path': fixture.executable}, 'trufflehog': {'path': str(executable)},
}, 'assets': {str(policy): {'path': str(policy)}}}}
# Only bootstrap authority/lease are synthetic; runner, argv guard and OwnedProcess stay real.
monkeypatch.setattr(scanner, 'require_active_supervisor_child', lambda **kw: metadata)
monkeypatch.setattr(scanner.os, 'environ', environment)
monkeypatch.setattr(scanner, '_runtime_initialized', True)
monkeypatch.setattr(scanner.scan_config, 'work_dir', str(command_root))
monkeypatch.setattr(scanner.scan_config, 'trufflehog_path', str(executable))
monkeypatch.setattr(scanner.scan_config, 'min_free_gb', 0)
monkeypatch.setattr(scanner.scan_config, 'trufflehog_job_memory_limit_bytes', 2 * 1024 ** 3)
slot = mock.Mock(releasable=True)
monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, slot))
original = scanner.run_command_streamed
@contextlib.contextmanager
def local_command(command, timeout, env):
child_env = dict(env)
if not enabled:
# Negative control: ignore the appended setting, retaining inherited core.longpaths=false.
child_env['GIT_CONFIG_COUNT'] = environment['GIT_CONFIG_COUNT']
with original([*command, '--endpoint', endpoint, '--concurrency', '1'], timeout, child_env) as output:
yield output
monkeypatch.setattr(scanner, 'run_command_streamed', local_command)
before = dict(environment)
result = scanner.scan_huggingface_space(SPACE, 60, no_verification=True, trufflehog_config=str(policy))
assert environment == before
events = [json.loads(line) for line in trace.read_text(encoding='utf-8').splitlines()]
clones = [event for event in events if event.get('event') == 'start' and 'clone' in event.get('argv', [])]
assert clones, 'installed HF must delegate cloning to native Git'
assert all('--no-checkout' not in event['argv'] for event in clones)
assert any(event.get('param') == 'core.longpaths' and event.get('value') == str(enabled).lower()
for event in events)
seen = {path for path, marker in fixture.markers.items()
if any(marker in str(finding.get('Raw', '')) + str(finding.get('RawV2', ''))
for finding in result['findings'])}
assert result['scan_meta']['trufflehog_returncode'] == 0
assert result['scan_meta']['trufflehog_finished'] is True
if enabled:
assert result['errors'] == [], result['errors']
assert seen == set(fixture.paths)
else:
assert result['errors']
assert 'Filename too long' in '\n'.join(result['errors'])
assert 'Clone succeeded, but checkout failed' in '\n'.join(result['errors'])
assert result['source_failure'] is False
assert not result.get('skipped')
slot.set_child_pid.assert_called_once()
slot.release.assert_not_called()
assert not list(command_root.iterdir()), 'owned HF command directory must be cleaned'
print('installed HF', json.dumps({'longpaths': enabled, 'native_clones': len(clones),
'finished': result['scan_meta']['trufflehog_finished'], 'returncode': 0,
'marker_files': len(seen), 'errors': len(result['errors']), 'loopback_requests': requests}))
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)