sashatrask
958 lines
48 KiB
Python
958 lines
48 KiB
Python
import hashlib
|
|
import contextlib
|
|
import builtins
|
|
from datetime import datetime
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import sys
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
APP_DIR = ROOT / 'app'
|
|
sys.path.insert(0, str(APP_DIR))
|
|
|
|
import scanner
|
|
from keycheckers import keycheck_common
|
|
from scanner_db import normalize_target
|
|
|
|
|
|
class ResultSinkOrderingTests(unittest.TestCase):
|
|
def test_scan_result_and_batch_fallback_timestamps_are_aware_utc(self):
|
|
with mock.patch.object(
|
|
scanner, 'scan_git_repo', return_value={'findings': [], 'errors': []},
|
|
):
|
|
result = scanner.scan_target_result(
|
|
'https://example.invalid/repo', 'git', 'event-id', {},
|
|
)
|
|
for name in ('scan_started_at', 'timestamp'):
|
|
parsed = datetime.fromisoformat(result[name])
|
|
self.assertIsNotNone(parsed.tzinfo)
|
|
self.assertEqual(parsed.utcoffset().total_seconds(), 0)
|
|
|
|
with mock.patch.object(
|
|
scanner, 'scan_target_result', side_effect=RuntimeError('fixture'),
|
|
):
|
|
fallback = scanner.scan_targets_batch(
|
|
['https://example.invalid/repo'], 'git', max_workers=1,
|
|
persist_results=False,
|
|
)[0]
|
|
for name in ('scan_started_at', 'timestamp'):
|
|
parsed = datetime.fromisoformat(fallback[name])
|
|
self.assertIsNotNone(parsed.tzinfo)
|
|
self.assertEqual(parsed.utcoffset().total_seconds(), 0)
|
|
|
|
def test_each_sink_precedes_completed_progress_and_failures_are_collected(self):
|
|
events = []
|
|
lock = threading.Lock()
|
|
|
|
def fake_scan(target, provider=None, **kwargs):
|
|
if target == 'slow':
|
|
time.sleep(0.04)
|
|
elif target == 'middle':
|
|
time.sleep(0.02)
|
|
return {'findings': [], 'errors': []}
|
|
|
|
def sink(result):
|
|
with lock:
|
|
events.append(('sink', result['target'], result['scan_event_id']))
|
|
if result['target'] in ('fast', 'slow'):
|
|
raise RuntimeError(f"sink failed: {result['target']}")
|
|
|
|
def log(message):
|
|
with lock:
|
|
events.append(('log', str(message), None))
|
|
|
|
def progress(completed, total, target):
|
|
with lock:
|
|
events.append(('progress', target, None))
|
|
|
|
with mock.patch.object(scanner, 'scan_git_repo', side_effect=fake_scan), \
|
|
mock.patch.object(scanner.logger, 'info', side_effect=log):
|
|
with self.assertRaises(scanner.ResultSinkError) as caught:
|
|
scanner.scan_targets_batch(
|
|
['slow', 'fast', 'middle'], 'git', progress_callback=progress,
|
|
max_workers=3, persist_results=False, result_sink=sink,
|
|
)
|
|
|
|
self.assertEqual({target for target, _ in caught.exception.failures}, {'fast', 'slow'})
|
|
sink_events = [item for item in events if item[0] == 'sink']
|
|
self.assertEqual({item[1] for item in sink_events}, {'slow', 'fast', 'middle'})
|
|
self.assertEqual(len({item[2] for item in sink_events}), 3)
|
|
for target in ('slow', 'fast', 'middle'):
|
|
sink_index = next(index for index, item in enumerate(events) if item[0] == 'sink' and item[1] == target)
|
|
if target == 'middle':
|
|
log_index = next(index for index, item in enumerate(events) if item[0] == 'log' and f': {target} ' in item[1])
|
|
progress_index = next(index for index, item in enumerate(events) if item[0] == 'progress' and item[1] == target)
|
|
self.assertLess(sink_index, log_index)
|
|
self.assertLess(sink_index, progress_index)
|
|
else:
|
|
self.assertFalse(any(item[0] == 'log' and f': {target} ' in item[1] for item in events))
|
|
self.assertFalse(any(item[0] == 'progress' and item[1] == target for item in events))
|
|
|
|
def test_file_sink_false_is_a_result_sink_error_and_not_completed(self):
|
|
progress = []
|
|
with mock.patch.object(scanner, 'scan_git_repo', return_value={'findings': [], 'errors': []}), \
|
|
mock.patch.object(scanner, 'save_scan_result', return_value=False), \
|
|
mock.patch.object(scanner.logger, 'info') as info:
|
|
with self.assertRaises(scanner.ResultSinkError):
|
|
scanner.scan_targets_batch(
|
|
['target'], 'git', progress_callback=lambda *value: progress.append(value),
|
|
max_workers=1, persist_results=True,
|
|
)
|
|
self.assertFalse(any(value[2] == 'target' for value in progress))
|
|
self.assertFalse(any('Completed' in str(call.args[0]) for call in info.call_args_list))
|
|
|
|
|
|
class IdempotentProjectionTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
|
|
|
|
@staticmethod
|
|
def json_rows(path):
|
|
rows = []
|
|
for candidate in scanner.projection_segment_paths(path):
|
|
with open(candidate, 'r', encoding='utf-8') as handle:
|
|
rows.extend(json.loads(line) for line in handle if line.strip())
|
|
return rows
|
|
|
|
def test_retry_after_ambiguity_at_each_append_stage_writes_each_projection_once(self):
|
|
for failure_stage in range(1, 5):
|
|
with self.subTest(failure_stage=failure_stage), tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
results_dir = os.path.join(temp_dir, 'results')
|
|
scanner.ensure_private_directory(results_dir, reject_reparse=True)
|
|
result = {
|
|
'scan_event_id': str(uuid.uuid4()),
|
|
'target': 'https://example.invalid/repo',
|
|
'scan_type': 'github',
|
|
'timestamp': '2026-07-19T00:00:00+00:00',
|
|
'findings': [
|
|
{'DetectorName': 'OpenAI', 'Raw': 'fixture-one'},
|
|
{'DetectorName': 'Anthropic', 'Raw': 'fixture-two'},
|
|
],
|
|
'errors': ['first error', 'second error'],
|
|
}
|
|
original_jsonl = scanner.append_rotating_jsonl_once
|
|
original_errors = scanner.append_scan_errors_once
|
|
stage = {'value': 0}
|
|
|
|
def ambiguous_jsonl(*args, **kwargs):
|
|
ok = original_jsonl(*args, **kwargs)
|
|
stage['value'] += 1
|
|
return False if stage['value'] == failure_stage else ok
|
|
|
|
def ambiguous_errors(*args, **kwargs):
|
|
ok = original_errors(*args, **kwargs)
|
|
stage['value'] += 1
|
|
return False if stage['value'] == failure_stage else ok
|
|
|
|
with mock.patch.object(scanner.scan_config, 'results_dir', results_dir), \
|
|
mock.patch.object(scanner.scan_config, 'jsonl_rotation_enabled', False), \
|
|
mock.patch.object(scanner.scan_config, 'scan_errors_max_mb', 1), \
|
|
mock.patch.object(scanner, 'write_foundry_keycheck_candidates_from_findings', return_value=0), \
|
|
mock.patch.object(scanner, 'append_rotating_jsonl_once', side_effect=ambiguous_jsonl), \
|
|
mock.patch.object(scanner, 'append_scan_errors_once', side_effect=ambiguous_errors):
|
|
self.assertFalse(scanner.save_scan_result(result))
|
|
with mock.patch.object(scanner.scan_config, 'results_dir', results_dir), \
|
|
mock.patch.object(scanner.scan_config, 'jsonl_rotation_enabled', False), \
|
|
mock.patch.object(scanner.scan_config, 'scan_errors_max_mb', 1), \
|
|
mock.patch.object(scanner, 'write_foundry_keycheck_candidates_from_findings', return_value=0):
|
|
self.assertTrue(scanner.save_scan_result(result))
|
|
|
|
scan_rows = self.json_rows(os.path.join(results_dir, 'scan_results.jsonl'))
|
|
finding_rows = self.json_rows(os.path.join(results_dir, 'found_secrets.jsonl'))
|
|
error_lines = []
|
|
for candidate in scanner.projection_segment_paths(os.path.join(results_dir, 'scan_errors.log')):
|
|
error_lines.extend(Path(candidate).read_text(encoding='utf-8').splitlines())
|
|
self.assertEqual(sum(row.get('scan_event_id') == result['scan_event_id'] for row in scan_rows), 1)
|
|
self.assertEqual(len(finding_rows), 2)
|
|
self.assertEqual(len({row['finding_uid'] for row in finding_rows}), 2)
|
|
self.assertEqual(len(error_lines), 2)
|
|
self.assertEqual(len({line.split('\t', 1)[0] for line in error_lines}), 2)
|
|
|
|
def test_scan_error_log_segments_and_retention_are_bounded(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_errors.log')
|
|
result = {
|
|
'scan_event_id': str(uuid.uuid4()),
|
|
'timestamp': '2026-07-19T00:00:00+00:00',
|
|
'scan_type': 'fixture',
|
|
'target': 'fixture',
|
|
'errors': ['x' * (2 * 1024 * 1024) for _ in range(4)],
|
|
}
|
|
self.assertTrue(scanner.append_scan_errors_once(path, result, max_mb=1, keep=1))
|
|
logs = scanner.projection_segment_paths(path)
|
|
self.assertLessEqual(len(logs), 2)
|
|
self.assertTrue(all(os.path.getsize(candidate) <= 1024 * 1024 for candidate in logs))
|
|
|
|
def test_offline_projection_reconciliation_is_resumable_and_preserves_history(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_results.jsonl')
|
|
original = b''.join(
|
|
json.dumps({'scan_event_id': f'event-{index}', 'value': index}).encode('utf-8') + b'\n'
|
|
for index in range(3)
|
|
)
|
|
Path(path).write_bytes(original)
|
|
scanner.harden_private_file(path)
|
|
first = scanner.reconcile_projection_ledger_batch(
|
|
path, 'scan_event_id', max_rows=1, max_bytes=1024, max_seconds=5,
|
|
)
|
|
self.assertFalse(first['complete'])
|
|
second = scanner.reconcile_projection_ledger_batch(
|
|
path, 'scan_event_id', max_rows=10, max_bytes=1024 * 1024, max_seconds=5,
|
|
)
|
|
self.assertTrue(second['complete'])
|
|
self.assertEqual(second['indexed_rows'], 3)
|
|
self.assertEqual(Path(path).read_bytes(), original)
|
|
with mock.patch.object(
|
|
scanner, 'projection_segment_paths', side_effect=AssertionError('history rescanned'),
|
|
):
|
|
done = scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
self.assertTrue(done['complete'])
|
|
|
|
def test_reviewed_corrupt_legacy_record_preserves_history_and_resume_cursor(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_results.jsonl')
|
|
first = b'{"scan_event_id":"event-1"}\n'
|
|
corrupt = b'1, "legacy":"DO_NOT_LEAK"}\n'
|
|
last = b'{"scan_event_id":"event-2"}\n'
|
|
original = first + corrupt + last
|
|
Path(path).write_bytes(original)
|
|
scanner.harden_private_file(path)
|
|
digest = hashlib.sha256(corrupt).hexdigest()
|
|
with self.assertRaises(scanner.JsonlProjectionReconciliationRequired) as raised:
|
|
scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
message = str(raised.exception)
|
|
self.assertIn(f'offset={len(first)}', message)
|
|
self.assertIn(f'sha256={digest}', message)
|
|
self.assertIn('classification=legacy_numeric_prefix_corrupt_json', message)
|
|
self.assertNotIn('DO_NOT_LEAK', message)
|
|
ledger_path = scanner.jsonl_ledger_path(path)
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
issue = ledger.execute(
|
|
'''SELECT byte_offset, byte_length, record_sha256, classification, status
|
|
FROM reconciliation_issue'''
|
|
).fetchone()
|
|
columns = {row[1] for row in ledger.execute('PRAGMA table_info(reconciliation_issue)')}
|
|
cursor = ledger.execute(
|
|
'''SELECT value FROM publication_meta
|
|
WHERE key = 'offline-reconcile:scan_event_id:offset' ''',
|
|
).fetchone()
|
|
self.assertEqual(issue, (
|
|
len(first), len(corrupt), digest, 'legacy_numeric_prefix_corrupt_json', 'pending',
|
|
))
|
|
self.assertEqual(cursor, (str(len(first)),))
|
|
self.assertFalse(any('payload' in column or 'raw' in column for column in columns))
|
|
with self.assertRaises(scanner.JsonlProjectionReconciliationRequired):
|
|
scanner.approve_projection_reconciliation_issue(
|
|
path, 'scan_event_id', os.path.basename(path), len(first), '0' * 64,
|
|
)
|
|
approval = scanner.approve_projection_reconciliation_issue(
|
|
path, 'scan_event_id', os.path.basename(path), len(first), digest,
|
|
)
|
|
self.assertEqual(approval['status'], 'resolved')
|
|
report = scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
self.assertTrue(report['complete'])
|
|
self.assertEqual(report['indexed_rows'], 2)
|
|
self.assertEqual(Path(path).read_bytes(), original)
|
|
|
|
def test_concatenated_legacy_json_objects_are_indexed_individually(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_results.jsonl')
|
|
original = b'{"scan_event_id":"event-1"}{"scan_event_id":"event-2"}\n'
|
|
Path(path).write_bytes(original)
|
|
scanner.harden_private_file(path)
|
|
report = scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
self.assertTrue(report['complete'])
|
|
self.assertEqual(report['indexed_rows'], 2)
|
|
self.assertEqual(Path(path).read_bytes(), original)
|
|
|
|
def test_offline_projection_reconciliation_streams_reviewed_oversized_record(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_results.jsonl')
|
|
original = b'{"scan_event_id":"event-1","value":"' + (b'x' * (1024 * 1024)) + b'"}\n'
|
|
Path(path).write_bytes(original)
|
|
scanner.harden_private_file(path)
|
|
digest = hashlib.sha256(original).hexdigest()
|
|
with self.assertRaisesRegex(
|
|
scanner.JsonlProjectionReconciliationRequired, 'classification=oversized_record',
|
|
):
|
|
scanner.reconcile_projection_ledger_batch(
|
|
path, 'scan_event_id', max_record_bytes=1024 * 1024,
|
|
)
|
|
approval = scanner.approve_projection_reconciliation_issues(
|
|
path,
|
|
'scan_event_id',
|
|
[{
|
|
'file': os.path.basename(path),
|
|
'offset': 0,
|
|
'length': len(original),
|
|
'sha256': digest,
|
|
'classification': 'oversized_record',
|
|
}],
|
|
max_record_bytes=1024 * 1024,
|
|
)
|
|
self.assertEqual(approval['resolved_count'], 1)
|
|
report = scanner.reconcile_projection_ledger_batch(
|
|
path, 'scan_event_id', max_record_bytes=1024 * 1024,
|
|
)
|
|
self.assertTrue(report['complete'])
|
|
self.assertEqual(Path(path).read_bytes(), original)
|
|
|
|
def test_historical_payload_variants_require_review_and_future_variant_fails(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_results.jsonl')
|
|
identity = 'event-1'
|
|
first_payload = {'scan_event_id': identity, 'value': 'first'}
|
|
second_payload = {'scan_event_id': identity, 'value': 'second'}
|
|
third_payload = {'scan_event_id': identity, 'value': 'future'}
|
|
first = (json.dumps(first_payload) + '\n').encode('utf-8')
|
|
second = (json.dumps(second_payload) + '\n').encode('utf-8')
|
|
original = first + second
|
|
Path(path).write_bytes(original)
|
|
scanner.harden_private_file(path)
|
|
with self.assertRaisesRegex(
|
|
scanner.JsonlProjectionReconciliationRequired,
|
|
'historical projection payload variant requires explicit review',
|
|
):
|
|
scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
ledger_path = scanner.jsonl_ledger_path(path)
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
issue = ledger.execute(
|
|
'''SELECT identity_sha256, payload_sha256, status
|
|
FROM reconciliation_variant_issue'''
|
|
).fetchone()
|
|
self.assertEqual(issue, (
|
|
hashlib.sha256(identity.encode()).hexdigest(),
|
|
hashlib.sha256(second).hexdigest(),
|
|
'pending',
|
|
))
|
|
review = scanner.approve_projection_conflict_variants(
|
|
path,
|
|
'scan_event_id',
|
|
[{
|
|
'file': os.path.basename(path),
|
|
'offset': len(first),
|
|
'length': len(second),
|
|
'identity_sha256': hashlib.sha256(identity.encode()).hexdigest(),
|
|
'payload_sha256': hashlib.sha256(second).hexdigest(),
|
|
'field_name_set_sha256': scanner._projection_field_name_set_sha256(second_payload),
|
|
'classification': 'historical_payload_variant',
|
|
}],
|
|
)
|
|
self.assertEqual(review['resolved_variant_count'], 1)
|
|
completed = scanner.reconcile_projection_ledger_batch(path, 'scan_event_id')
|
|
self.assertTrue(completed['complete'])
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
variants = ledger.execute(
|
|
'''SELECT payload_sha256 FROM publication_identity_variant
|
|
WHERE identity_key = 'scan_event_id' AND identity_value = ?''',
|
|
(identity,),
|
|
).fetchall()
|
|
self.assertEqual({row[0] for row in variants}, {
|
|
hashlib.sha256(first).hexdigest(), hashlib.sha256(second).hexdigest(),
|
|
})
|
|
before = Path(path).read_bytes()
|
|
self.assertTrue(scanner.append_rotating_jsonl_once(
|
|
path, first_payload, 'scan_event_id', identity, max_mb=0,
|
|
))
|
|
self.assertTrue(scanner.append_rotating_jsonl_once(
|
|
path, second_payload, 'scan_event_id', identity, max_mb=0,
|
|
))
|
|
self.assertFalse(scanner.append_rotating_jsonl_once(
|
|
path, third_payload, 'scan_event_id', identity, max_mb=0,
|
|
))
|
|
self.assertEqual(Path(path).read_bytes(), before)
|
|
|
|
def test_identical_historical_duplicate_uses_one_exact_variant(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'found_secrets.jsonl')
|
|
payload = {'finding_uid': 'finding-1', 'value': 'same'}
|
|
row = (json.dumps(payload) + '\n').encode('utf-8')
|
|
Path(path).write_bytes(row + row)
|
|
scanner.harden_private_file(path)
|
|
report = scanner.reconcile_projection_ledger_batch(path, 'finding_uid')
|
|
self.assertTrue(report['complete'])
|
|
ledger_path = scanner.jsonl_ledger_path(path)
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
variants = ledger.execute(
|
|
'''SELECT COUNT(*) FROM publication_identity_variant
|
|
WHERE identity_key = 'finding_uid' AND identity_value = 'finding-1' '''
|
|
).fetchone()[0]
|
|
conflicts = ledger.execute('SELECT COUNT(*) FROM reconciliation_variant_issue').fetchone()[0]
|
|
self.assertEqual(variants, 1)
|
|
self.assertEqual(conflicts, 0)
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
ledger.execute('DELETE FROM publication_identity_variant')
|
|
ledger.commit()
|
|
reopened = scanner._open_projection_ledger(path)
|
|
try:
|
|
backfilled = reopened.execute(
|
|
'SELECT COUNT(*) FROM publication_identity_variant'
|
|
).fetchone()[0]
|
|
finally:
|
|
reopened.close()
|
|
self.assertEqual(backfilled, 1)
|
|
|
|
def test_conflict_manifest_verification_opens_each_segment_once(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_errors.log')
|
|
rows = []
|
|
reviewed = []
|
|
offset = 0
|
|
for index in range(200):
|
|
identity = f'identity-{index // 2}'
|
|
value = {'variant': index % 2, 'index': index}
|
|
row = f'{identity}\twhen\tgithub\t{json.dumps(value)}\n'.encode()
|
|
rows.append(row)
|
|
reviewed.append({
|
|
'file': os.path.basename(path),
|
|
'offset': offset,
|
|
'length': len(row),
|
|
'identity_sha256': hashlib.sha256(identity.encode()).hexdigest(),
|
|
'payload_sha256': hashlib.sha256(row).hexdigest(),
|
|
'field_name_set_sha256': scanner._projection_field_name_set_sha256(value),
|
|
'classification': 'historical_payload_variant',
|
|
})
|
|
offset += len(row)
|
|
Path(path).write_bytes(b''.join(rows))
|
|
scanner.harden_private_file(path)
|
|
real_open = builtins.open
|
|
counts = {'opens': 0, 'readlines': 0}
|
|
|
|
class CountedFile:
|
|
def __init__(self, handle):
|
|
self.handle = handle
|
|
|
|
def __enter__(self):
|
|
self.handle.__enter__()
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return self.handle.__exit__(*args)
|
|
|
|
def readline(self, *args):
|
|
counts['readlines'] += 1
|
|
return self.handle.readline(*args)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.handle, name)
|
|
|
|
def counted_open(name, *args, **kwargs):
|
|
handle = real_open(name, *args, **kwargs)
|
|
if os.path.abspath(os.fspath(name)) == os.path.abspath(path) and 'rb' in str(args[0] if args else kwargs.get('mode', 'r')):
|
|
counts['opens'] += 1
|
|
return CountedFile(handle)
|
|
return handle
|
|
|
|
with mock.patch('builtins.open', side_effect=counted_open):
|
|
report = scanner.approve_projection_conflict_variants(
|
|
path, 'error_row_id', reviewed, commit_batch_size=50,
|
|
)
|
|
self.assertEqual(report['resolved_variant_count'], 200)
|
|
self.assertEqual(counts, {'opens': 1, 'readlines': 200})
|
|
|
|
def test_conflict_manifest_apply_resumes_after_committed_interruption(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
path = os.path.join(temp_dir, 'scan_errors.log')
|
|
rows = []
|
|
reviewed = []
|
|
offset = 0
|
|
for index in range(30):
|
|
identity = f'identity-{index // 2}'
|
|
value = {'variant': index % 2, 'index': index}
|
|
row = f'{identity}\twhen\tgithub\t{json.dumps(value)}\n'.encode()
|
|
rows.append(row)
|
|
reviewed.append({
|
|
'file': os.path.basename(path),
|
|
'offset': offset,
|
|
'length': len(row),
|
|
'identity_sha256': hashlib.sha256(identity.encode()).hexdigest(),
|
|
'payload_sha256': hashlib.sha256(row).hexdigest(),
|
|
'field_name_set_sha256': scanner._projection_field_name_set_sha256(value),
|
|
'classification': 'historical_payload_variant',
|
|
})
|
|
offset += len(row)
|
|
Path(path).write_bytes(b''.join(rows))
|
|
scanner.harden_private_file(path)
|
|
|
|
def interrupt_after_commit(progress):
|
|
if progress['newly_resolved'] == 10:
|
|
raise RuntimeError('injected interruption')
|
|
|
|
with self.assertRaisesRegex(RuntimeError, 'injected interruption'):
|
|
scanner.approve_projection_conflict_variants(
|
|
path,
|
|
'error_row_id',
|
|
reviewed,
|
|
commit_batch_size=10,
|
|
progress_callback=interrupt_after_commit,
|
|
)
|
|
real_apply = scanner._apply_reviewed_projection_conflict_variant
|
|
with mock.patch.object(
|
|
scanner, '_apply_reviewed_projection_conflict_variant', wraps=real_apply,
|
|
) as apply_entry:
|
|
report = scanner.approve_projection_conflict_variants(
|
|
path, 'error_row_id', reviewed, commit_batch_size=10,
|
|
)
|
|
self.assertEqual(report['already_resolved'], 10)
|
|
self.assertEqual(report['newly_resolved'], 20)
|
|
self.assertEqual(apply_entry.call_count, 20)
|
|
ledger_path = scanner.jsonl_ledger_path(path)
|
|
with contextlib.closing(sqlite3.connect(ledger_path)) as ledger:
|
|
resolved = ledger.execute(
|
|
"SELECT COUNT(*) FROM reconciliation_variant_issue WHERE status = 'resolved'"
|
|
).fetchone()[0]
|
|
self.assertEqual(resolved, 30)
|
|
|
|
|
|
class ScannerTempPlacementTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
|
|
|
|
def test_command_and_docker_temp_never_fall_back_to_system_temp(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
scanner.ensure_private_directory(work_dir, reject_reparse=True)
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner.scan_config, 'min_free_gb', 0), \
|
|
mock.patch.object(scanner.tempfile, 'gettempdir', side_effect=AssertionError('system temp used')):
|
|
command_dir = scanner.create_command_work_dir()
|
|
docker_dir = scanner.create_docker_config_dir()
|
|
self.assertTrue(command_dir.startswith(work_dir))
|
|
self.assertTrue(docker_dir.startswith(os.path.join(work_dir, 'docker-config')))
|
|
scanner.cleanup_command_work_dir(command_dir)
|
|
scanner.cleanup_command_work_dir(docker_dir)
|
|
|
|
def test_configured_work_dir_failure_has_no_mkdtemp_fallback(self):
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', r'D:\unavailable-work'), \
|
|
mock.patch.object(scanner, 'require_private_directory', side_effect=OSError('denied')), \
|
|
mock.patch.object(scanner.tempfile, 'mkdtemp') as mkdtemp:
|
|
with self.assertRaisesRegex(RuntimeError, 'configured|private|work_dir'):
|
|
scanner.create_command_work_dir()
|
|
with self.assertRaisesRegex(RuntimeError, 'configured|private|work_dir'):
|
|
scanner.create_docker_config_dir()
|
|
mkdtemp.assert_not_called()
|
|
|
|
def test_trufflehog_output_files_use_command_work_directory(self):
|
|
class Process:
|
|
pid = 12345
|
|
returncode = 0
|
|
job_membership_verified = True
|
|
payload_identity = {
|
|
'pid': pid,
|
|
'creation_time': 'fixture-12345',
|
|
'executable': sys.executable,
|
|
}
|
|
|
|
def poll(self):
|
|
return 0
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
scanner.ensure_private_directory(work_dir, reject_reparse=True)
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner.scan_config, 'min_free_gb', 0), \
|
|
mock.patch.object(scanner, 'require_trufflehog_launch_authority'), \
|
|
mock.patch.object(scanner, 'OwnedProcess', return_value=Process()), \
|
|
mock.patch.object(scanner.tempfile, 'TemporaryFile', wraps=tempfile.TemporaryFile) as temporary_file, \
|
|
mock.patch.object(scanner.tempfile, 'gettempdir', side_effect=AssertionError('system temp used')):
|
|
stdout, stderr, returncode = scanner.run_command(['fixture'], 1)
|
|
self.assertEqual((stdout, stderr, returncode), ('', '', 0))
|
|
self.assertEqual(temporary_file.call_count, 2)
|
|
self.assertTrue(all(call.kwargs.get('dir') for call in temporary_file.call_args_list))
|
|
|
|
def test_source_side_persisted_cleanup_is_retired(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
external = os.path.join(temp_dir, 'external', 'trufflehog-run-outside')
|
|
unapproved = os.path.join(work_dir, 'user-data')
|
|
approved = os.path.join(work_dir, 'trufflehog-run-orphan')
|
|
for path in (work_dir, os.path.dirname(external), external, unapproved, approved):
|
|
scanner.ensure_private_directory(path, reject_reparse=True)
|
|
scanner.write_temp_owner(external, ['fixture'], owner_pid=99999991)
|
|
scanner.write_temp_owner(unapproved, ['fixture'], owner_pid=99999992)
|
|
scanner.write_temp_owner(approved, ['fixture'], owner_pid=99999993)
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner, 'temp_dir_active', return_value=False):
|
|
scanner.pending_temp_dirs.clear()
|
|
scanner.atomic_write_private_json(
|
|
scanner.get_pending_temp_file(),
|
|
{
|
|
'schema': scanner.PENDING_TEMP_SCHEMA,
|
|
'paths': [external, unapproved, approved],
|
|
},
|
|
)
|
|
removed = scanner.cleanup_pending_command_work_dirs(log_failures=True)
|
|
scanner.pending_temp_dirs.clear()
|
|
self.assertEqual(removed, 0)
|
|
self.assertTrue(os.path.isdir(external))
|
|
self.assertTrue(os.path.isdir(unapproved))
|
|
self.assertTrue(os.path.isdir(approved))
|
|
|
|
def test_periodic_source_cleanup_never_deletes_any_tree(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
markerless = os.path.join(work_dir, 'tmp-user-data')
|
|
owned = os.path.join(work_dir, 'trufflehog-run-owned')
|
|
for path in (work_dir, markerless, owned):
|
|
scanner.ensure_private_directory(path, reject_reparse=True)
|
|
scanner.write_temp_owner(owned, ['fixture'], owner_pid=99999991)
|
|
old = time.time() - 7200
|
|
os.utime(markerless, (old, old))
|
|
os.utime(owned, (old, old))
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner, 'temp_dir_active', return_value=False):
|
|
removed = scanner.cleanup_stale_temp_dirs(age_minutes=1, log=False)
|
|
self.assertEqual(removed, 0)
|
|
self.assertTrue(os.path.isdir(markerless))
|
|
self.assertTrue(os.path.isdir(owned))
|
|
|
|
def test_dependency_probe_uses_only_configured_work_volume_temp(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
scanner.ensure_private_directory(work_dir, reject_reparse=True)
|
|
completed = SimpleNamespace(returncode=0, stdout=b'version', stderr=b'')
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner, 'require_trufflehog_launch_authority'), \
|
|
mock.patch.object(scanner, 'run_owned', return_value=completed) as run:
|
|
self.assertTrue(scanner.check_dependencies())
|
|
options = run.call_args.kwargs
|
|
self.assertTrue(options['cwd'].startswith(work_dir))
|
|
self.assertEqual(options['env']['TEMP'], options['cwd'])
|
|
self.assertEqual(options['env']['TMP'], options['cwd'])
|
|
self.assertEqual(options['env']['TMPDIR'], options['cwd'])
|
|
self.assertEqual(options['timeout'], 30)
|
|
|
|
def test_supervised_dependency_probe_reports_bounded_timeout_reason(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
work_dir = os.path.join(temp_dir, 'work')
|
|
scanner.ensure_private_directory(work_dir, reject_reparse=True)
|
|
timeout = subprocess.TimeoutExpired(['trufflehog', '--version', '--no-update'], 30)
|
|
with mock.patch.object(scanner.scan_config, 'work_dir', work_dir), \
|
|
mock.patch.object(scanner, 'require_trufflehog_launch_authority'), \
|
|
mock.patch.object(scanner, 'run_owned', side_effect=timeout), \
|
|
mock.patch.dict(os.environ, {'SCANNER_SUPERVISED': '1'}, clear=False), \
|
|
self.assertLogs(scanner.logger, level='ERROR') as captured:
|
|
self.assertFalse(scanner.check_dependencies())
|
|
self.assertTrue(any('timed out after 30s' in line for line in captured.output))
|
|
self.assertFalse(any('Please install' in line for line in captured.output))
|
|
|
|
def test_dependency_probe_fails_closed_when_configured_work_dir_is_unavailable(self):
|
|
with mock.patch.object(scanner, 'get_work_dir', side_effect=RuntimeError('D: unavailable')), \
|
|
mock.patch.object(scanner, 'run_owned') as run:
|
|
self.assertFalse(scanner.check_dependencies())
|
|
run.assert_not_called()
|
|
|
|
def test_postman_semantic_identity_ignores_origin_metadata(self):
|
|
digest = 'a' * 64
|
|
first = json.dumps({
|
|
'source': 'github_code', 'sha256': digest, 'cache_path': 'one.json',
|
|
'origin': {'repo': 'one/repo'},
|
|
})
|
|
second = json.dumps({
|
|
'source': 'npm_package', 'sha256': digest, 'cache_path': 'two.json',
|
|
'origin': {'package_name': 'different'},
|
|
})
|
|
self.assertEqual(scanner.postman_target_identity(first), scanner.postman_target_identity(second))
|
|
self.assertEqual(normalize_target(first, 'postman'), normalize_target(second, 'postman'))
|
|
|
|
def test_postman_scan_rejects_cache_escape_and_hash_mismatch_before_command(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
cache_root = os.path.join(temp_dir, 'cache')
|
|
scanner.ensure_private_directory(cache_root, reject_reparse=True)
|
|
inside = os.path.join(cache_root, 'artifact.json')
|
|
outside = os.path.join(temp_dir, 'outside.json')
|
|
Path(inside).write_text('{"inside":true}', encoding='ascii')
|
|
Path(outside).write_text('{"outside":true}', encoding='ascii')
|
|
scanner.harden_private_file(inside)
|
|
scanner.harden_private_file(outside)
|
|
with mock.patch.object(scanner.scan_config, 'runtime_dir', temp_dir), \
|
|
mock.patch.object(scanner.scan_config, 'postman_cache_dir', cache_root), \
|
|
mock.patch.object(scanner, 'run_command_streamed') as run:
|
|
escaped = scanner.scan_postman_target(json.dumps({
|
|
'source': 'cache', 'cache_path': outside,
|
|
'sha256': hashlib.sha256(Path(outside).read_bytes()).hexdigest(),
|
|
}))
|
|
mismatched = scanner.scan_postman_target(json.dumps({
|
|
'source': 'cache', 'cache_path': inside, 'sha256': '0' * 64,
|
|
}))
|
|
self.assertIn('escapes', escaped['errors'][0])
|
|
self.assertIn('mismatch', mismatched['errors'][0])
|
|
run.assert_not_called()
|
|
|
|
def test_projection_lock_release_never_unlinks_or_releases_a_new_generation(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
scanner.ensure_private_directory(temp_dir, reject_reparse=True)
|
|
scanner_lock_path = os.path.join(temp_dir, 'scanner.jsonl.lock')
|
|
old = scanner.acquire_file_lock(scanner_lock_path, timeout_sec=1)
|
|
scanner.release_file_lock(old, scanner_lock_path)
|
|
current = scanner.acquire_file_lock(scanner_lock_path, timeout_sec=1)
|
|
scanner.release_file_lock(old, scanner_lock_path)
|
|
self.assertTrue(current.acquired)
|
|
self.assertTrue(os.path.exists(scanner_lock_path))
|
|
scanner.release_file_lock(current, scanner_lock_path)
|
|
self.assertTrue(os.path.exists(scanner_lock_path))
|
|
|
|
keycheck_lock_path = os.path.join(temp_dir, 'keycheck.jsonl.lock')
|
|
old = keycheck_common.acquire_file_lock(keycheck_lock_path, timeout_sec=1)
|
|
keycheck_common.release_file_lock(old, keycheck_lock_path)
|
|
current = keycheck_common.acquire_file_lock(keycheck_lock_path, timeout_sec=1)
|
|
keycheck_common.release_file_lock(old, keycheck_lock_path)
|
|
self.assertTrue(current.acquired)
|
|
self.assertTrue(os.path.exists(keycheck_lock_path))
|
|
keycheck_common.release_file_lock(current, keycheck_lock_path)
|
|
|
|
def test_command_output_limits_fit_the_configured_event_budget(self):
|
|
with mock.patch.object(scanner.scan_config, 'result_bundle_max_event_bytes', 8 * 1024 * 1024), \
|
|
mock.patch.dict(os.environ, {
|
|
'TRUFFLEHOG_STDOUT_MAX_MB': '128',
|
|
'TRUFFLEHOG_STDERR_MAX_MB': '16',
|
|
}):
|
|
stdout_limit, stderr_limit = scanner.command_output_limits()
|
|
self.assertLess(stdout_limit + stderr_limit, 8 * 1024 * 1024)
|
|
|
|
def test_command_output_limits_use_bounded_runtime_defaults(self):
|
|
with mock.patch.dict(os.environ, {}, clear=False), \
|
|
mock.patch.object(scanner.scan_config, 'trufflehog_stdout_max_mb', 32), \
|
|
mock.patch.object(scanner.scan_config, 'trufflehog_stderr_max_mb', 8):
|
|
os.environ.pop('TRUFFLEHOG_STDOUT_MAX_MB', None)
|
|
os.environ.pop('TRUFFLEHOG_STDERR_MAX_MB', None)
|
|
stdout_limit, stderr_limit = scanner.command_output_limits()
|
|
self.assertEqual(stdout_limit, 32 * 1024 * 1024)
|
|
self.assertEqual(stderr_limit, 8 * 1024 * 1024)
|
|
|
|
def test_docker_search_caps_requests_at_accessible_page_window(self):
|
|
requested_pages = []
|
|
|
|
class Response:
|
|
status_code = 200
|
|
headers = {}
|
|
|
|
def __init__(self, page):
|
|
self.page = page
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return {
|
|
'count': 300,
|
|
'results': [{'repo_name': f'owner/repo-{self.page}'}],
|
|
}
|
|
|
|
def request(_method, url, **_kwargs):
|
|
page = int(_kwargs['params']['page'])
|
|
requested_pages.append(page)
|
|
return Response(page)
|
|
|
|
with mock.patch.object(scanner, 'api_request', side_effect=request):
|
|
images = scanner.fetch_dockerhub_images(
|
|
'fixture', pages=30, per_page=10, fetch_workers=5, resolve_tags=False,
|
|
)
|
|
|
|
self.assertEqual(len(images), 30)
|
|
self.assertEqual(sorted(requested_pages), list(range(1, 31)))
|
|
|
|
def test_docker_resolver_rotates_account_after_429_without_global_backoff(self):
|
|
class Response:
|
|
def __init__(self, status_code, payload=None, headers=None):
|
|
self.status_code = status_code
|
|
self.payload = payload or {}
|
|
self.headers = headers or {}
|
|
self.content = json.dumps(self.payload).encode('utf-8')
|
|
|
|
def raise_for_status(self):
|
|
if self.status_code >= 400:
|
|
raise RuntimeError(f'HTTP {self.status_code}')
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
manager = scanner.DockerTokenManager()
|
|
manager.accounts = [
|
|
scanner.DockerAccount('account-a', 'user-a', 'secret-a', ''),
|
|
scanner.DockerAccount('account-b', 'user-b', 'secret-b', ''),
|
|
]
|
|
manager.explicit_pool = True
|
|
responses = [
|
|
Response(200, {'access_token': 'bearer-a', 'expires_in': 600}),
|
|
Response(429, headers={'Retry-After': '300'}),
|
|
Response(200, {'access_token': 'bearer-b', 'expires_in': 600}),
|
|
Response(200, {'results': []}),
|
|
]
|
|
|
|
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
|
|
mock.patch.object(
|
|
scanner, 'dockerhub_tag_rate_limit_state',
|
|
return_value={'active': False, 'retry_at': None},
|
|
), \
|
|
mock.patch.object(scanner, 'put_dockerhub_tags_rate_limit') as put_backoff, \
|
|
mock.patch.object(scanner, 'docker_token_manager', manager), \
|
|
mock.patch.object(scanner, 'api_request', side_effect=responses) as request, \
|
|
mock.patch.object(scanner.time, 'sleep') as sleep:
|
|
tags, status = scanner.fetch_dockerhub_tags(
|
|
'owner/repo', retry_count=0,
|
|
return_status=True,
|
|
)
|
|
self.assertEqual((tags, status), ([], 'empty'))
|
|
self.assertEqual(request.call_count, 4)
|
|
put_backoff.assert_not_called()
|
|
sleep.assert_not_called()
|
|
events = manager.drain_status_events()
|
|
self.assertEqual(
|
|
{event['name']: event['category'] for event in events},
|
|
{'account-a': 'rate_limit', 'account-b': 'ok'},
|
|
)
|
|
self.assertNotIn('secret-a', repr(events))
|
|
self.assertNotIn('secret-b', repr(events))
|
|
self.assertEqual(
|
|
scanner.dockerhub_retry_after_seconds(
|
|
Response(429, headers={'Retry-After': '999999'}),
|
|
),
|
|
3600,
|
|
)
|
|
|
|
def test_docker_resolver_sets_shared_backoff_only_after_pool_exhaustion(self):
|
|
class Response:
|
|
def __init__(self, status_code, payload=None, headers=None):
|
|
self.status_code = status_code
|
|
self.payload = payload or {}
|
|
self.headers = headers or {}
|
|
self.content = json.dumps(self.payload).encode('utf-8')
|
|
|
|
def raise_for_status(self):
|
|
if self.status_code >= 400:
|
|
raise RuntimeError(f'HTTP {self.status_code}')
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
manager = scanner.DockerTokenManager()
|
|
manager.accounts = [
|
|
scanner.DockerAccount('account-a', 'user-a', 'secret-a', ''),
|
|
scanner.DockerAccount('account-b', 'user-b', 'secret-b', ''),
|
|
]
|
|
manager.explicit_pool = True
|
|
responses = [
|
|
Response(200, {'access_token': 'bearer-a', 'expires_in': 600}),
|
|
Response(429, headers={'Retry-After': '300'}),
|
|
Response(200, {'access_token': 'bearer-b', 'expires_in': 600}),
|
|
Response(429, headers={'Retry-After': '300'}),
|
|
]
|
|
retry_at = '2026-08-29T12:00:00+00:00'
|
|
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
|
|
mock.patch.object(
|
|
scanner, 'dockerhub_tag_rate_limit_state',
|
|
return_value={'active': False, 'retry_at': None},
|
|
), \
|
|
mock.patch.object(
|
|
scanner, 'put_dockerhub_tags_rate_limit', return_value=retry_at,
|
|
) as put_backoff, \
|
|
mock.patch.object(scanner, 'docker_token_manager', manager), \
|
|
mock.patch.object(scanner, 'api_request', side_effect=responses):
|
|
outcome = scanner.fetch_dockerhub_tags(
|
|
'owner/repo', retry_count=0, return_outcome=True,
|
|
)
|
|
self.assertEqual(outcome.status, 'rate_limited')
|
|
self.assertTrue(outcome.remote_attempted)
|
|
self.assertEqual(outcome.retry_at, retry_at)
|
|
self.assertEqual(put_backoff.call_count, 1)
|
|
self.assertNotIn('secret-a', repr(outcome))
|
|
self.assertNotIn('secret-b', repr(outcome))
|
|
|
|
def test_docker_explicit_empty_pool_fails_without_anonymous_request_or_shared_429(self):
|
|
manager = scanner.DockerTokenManager()
|
|
manager.setup_accounts([], explicit_pool=True)
|
|
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
|
|
mock.patch.object(
|
|
scanner, 'dockerhub_tag_rate_limit_state',
|
|
return_value={'active': False, 'retry_at': None},
|
|
), \
|
|
mock.patch.object(scanner, 'put_dockerhub_tags_rate_limit') as put_backoff, \
|
|
mock.patch.object(scanner, 'docker_token_manager', manager), \
|
|
mock.patch.object(scanner, 'api_request') as request:
|
|
outcome = scanner.fetch_dockerhub_tags(
|
|
'owner/repo', retry_count=0, return_outcome=True,
|
|
)
|
|
self.assertEqual(outcome.status, 'auth_failed')
|
|
self.assertFalse(outcome.remote_attempted)
|
|
self.assertIsNone(outcome.retry_at)
|
|
request.assert_not_called()
|
|
put_backoff.assert_not_called()
|
|
|
|
def test_docker_repository_403_rotates_without_globally_cooling_account(self):
|
|
class Response:
|
|
def __init__(self, status_code, payload=None):
|
|
self.status_code = status_code
|
|
self.payload = payload or {}
|
|
self.headers = {}
|
|
self.content = json.dumps(self.payload).encode('utf-8')
|
|
|
|
def raise_for_status(self):
|
|
if self.status_code >= 400:
|
|
raise RuntimeError(f'HTTP {self.status_code}')
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
manager = scanner.DockerTokenManager()
|
|
manager.accounts = [
|
|
scanner.DockerAccount('account-a', 'user-a', 'secret-a', ''),
|
|
scanner.DockerAccount('account-b', 'user-b', 'secret-b', ''),
|
|
]
|
|
manager.explicit_pool = True
|
|
responses = [
|
|
Response(200, {'access_token': 'bearer-a'}),
|
|
Response(403),
|
|
Response(200, {'access_token': 'bearer-b'}),
|
|
Response(200, {'results': []}),
|
|
]
|
|
with mock.patch.object(scanner, 'get_dockerhub_tag_cache', return_value=None), \
|
|
mock.patch.object(
|
|
scanner, 'dockerhub_tag_rate_limit_state',
|
|
return_value={'active': False, 'retry_at': None},
|
|
), \
|
|
mock.patch.object(scanner, 'put_dockerhub_tags_rate_limit') as put_backoff, \
|
|
mock.patch.object(scanner, 'docker_token_manager', manager), \
|
|
mock.patch.object(scanner, 'api_request', side_effect=responses):
|
|
outcome = scanner.fetch_dockerhub_tags(
|
|
'owner/private-repo', retry_count=0, return_outcome=True,
|
|
)
|
|
self.assertEqual(outcome.status, 'empty')
|
|
self.assertFalse(manager.all_unavailable('hub_tags'))
|
|
self.assertEqual(
|
|
{event['name']: event['category'] for event in manager.drain_status_events()},
|
|
{'account-a': 'ok', 'account-b': 'ok'},
|
|
)
|
|
put_backoff.assert_not_called()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|