Initial server source import

tests/test_discovery_request_budgets.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+import sys
+from unittest import mock
+
+import pytest
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'app'))
+import scanner
+
+
+@pytest.fixture(params=['gitlab', 'huggingface_metadata', 'huggingface_legacy'])
+def discovery(request):
+    source = request.param
+    if source == 'gitlab':
+        fetch = lambda **kwargs: scanner.fetch_gitlab_repo_items('fixture', 2, **kwargs)
+        payloads = [[{'http_url_to_repo': 'https://fixture.invalid/repo.git'}], []]
+    else:
+        metadata = source == 'huggingface_metadata'
+        fetch = lambda **kwargs: scanner.fetch_huggingface_spaces(pages=2, return_metadata=metadata, **kwargs)
+        spaces = [{'id': 'fixture/space'}]
+        payloads = [spaces, []] if metadata else [{'spaces': spaces}, {'spaces': []}]
+    responses = []
+    for index, payload in enumerate(payloads):
+        response = mock.Mock(status_code=200, headers={}, text='')
+        response.json.return_value = payload
+        response.links = {'next': {'url': 'https://fixture.invalid/page2'}} if index == 0 else {}
+        responses.append(response)
+    return source, fetch, responses
+
+
+def test_each_discovery_page_has_one_shared_attempt_deadline(discovery, monkeypatch):
+    source, fetch, responses = discovery
+    clock = [100.0]
+    calls = []
+
+    def request(*args, **kwargs):
+        calls.append(kwargs)
+        clock[0] += 10
+        return responses.pop(0)
+
+    monkeypatch.setattr(scanner.time, 'monotonic', lambda: clock[0])
+    monkeypatch.setattr(scanner, 'api_request', request)
+    assert len(fetch(request_attempts=3, retry_delay=5)) == 1
+    budget = 100 if source == 'gitlab' else 55
+    assert [call['deadline'] for call in calls] == [100 + budget, 110 + budget]
+    assert all(call['max_retries'] == 3 and call['retry_delay'] == 5 for call in calls)
+    assert all(call['timeout'] == (30 if source == 'gitlab' else 15) for call in calls)
+
+
+def test_discovery_proxy_uses_source_read_timeout_and_at_most_three_attempts(discovery, monkeypatch):
+    source, fetch, responses = discovery
+    clock = [0.0]
+    calls = []
+    waits = []
+    proxy = {'https': 'http://configured.invalid:8080'}
+
+    def request(*args, **kwargs):
+        calls.append(kwargs)
+        raise scanner.requests.exceptions.ReadTimeout('offline fixture')
+
+    def wait(seconds):
+        waits.append(seconds)
+        clock[0] += seconds
+
+    monkeypatch.setattr(scanner.scan_config, 'api_proxy_enabled', True)
+    monkeypatch.setattr(scanner.scan_config, 'api_proxy_timeout', 5)
+    monkeypatch.setattr(scanner.scan_config, 'api_proxy_max_retries', 100)
+    monkeypatch.setattr(scanner, 'next_api_proxy', lambda: proxy)
+    monkeypatch.setattr(scanner.time, 'monotonic', lambda: clock[0])
+    monkeypatch.setattr(scanner.requests, 'request', request)
+    monkeypatch.setattr(scanner, '_wait_or_raise_scan_slot_fatal', wait)
+    error_type = scanner.GitLabDiscoveryTransportError if source == 'gitlab' else scanner.ApiRequestError
+    with pytest.raises(error_type, match='after 3 attempt'):
+        fetch(request_attempts=3, retry_delay=5)
+    assert len(calls) == 3
+    assert all(call['timeout'] == (5, 30 if source == 'gitlab' else 15) for call in calls)
+    assert all(call['proxies'] == proxy for call in calls)
+    assert waits == [5, 5]
+
+
+def test_late_discovery_success_fails_closed_without_advancing_page(discovery, monkeypatch):
+    source, fetch, responses = discovery
+    clock = [0.0]
+    calls = []
+    first_response = responses[0]
+
+    def request(*args, **kwargs):
+        calls.append(kwargs)
+        clock[0] = 101.0
+        return first_response
+
+    monkeypatch.setattr(scanner.scan_config, 'api_proxy_enabled', False)
+    monkeypatch.setattr(scanner.time, 'monotonic', lambda: clock[0])
+    monkeypatch.setattr(scanner.requests, 'request', request)
+    error_type = scanner.GitLabDiscoveryTransportError if source == 'gitlab' else scanner.ApiRequestError
+    with pytest.raises(error_type, match='deadline expired after response'):
+        fetch(request_attempts=3, retry_delay=5)
+    assert len(calls) == 1
+    first_response.json.assert_not_called()
+    first_response.close.assert_called_once()