sashatrask
1775 lines
79 KiB
Python
1775 lines
79 KiB
Python
import os
|
|
import copy
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from contextlib import nullcontext
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest import mock
|
|
|
|
|
|
APP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'app'))
|
|
if APP_DIR not in sys.path:
|
|
sys.path.insert(0, APP_DIR)
|
|
|
|
import worker_assignment_runner as runner
|
|
import remote_worker_client
|
|
import scan_execution
|
|
from remote_worker_client import WorkerClientError, WorkerSlot
|
|
from result_bundle import (
|
|
BundleReservation, ResultBundleReader, ResultBundleWriter, bundle_ready_path,
|
|
)
|
|
from runtime_security import atomic_write_private_json, ensure_private_directory
|
|
from scanner import StagedResult
|
|
from test_worker_api import assignment, build_compatibility
|
|
|
|
|
|
def live_assignment():
|
|
value = assignment()
|
|
issued = datetime.now(timezone.utc).replace(microsecond=0)
|
|
deadline = issued + timedelta(days=1)
|
|
issued_text = issued.isoformat(timespec='seconds')
|
|
deadline_text = deadline.isoformat(timespec='seconds')
|
|
value['reservation']['remote_issued_at'] = issued_text
|
|
value['reservation']['remote_expires_at'] = deadline_text
|
|
value['deadlines'].update({
|
|
'assignment_issued_at': issued_text,
|
|
'assignment_deadline_at': deadline_text,
|
|
'assignment_ttl_seconds': 86400,
|
|
})
|
|
return value
|
|
|
|
|
|
def package_runtime():
|
|
return {
|
|
'build_compatibility': build_compatibility(),
|
|
'code_manifest': {},
|
|
'code_manifest_sha256': '9' * 64,
|
|
'trufflehog_path': 'trufflehog',
|
|
'git_path': 'git',
|
|
'detector_policy_path': 'detectors.yaml',
|
|
'capabilities': (
|
|
('github', 'github', 'exact_git_v1'),
|
|
('gitlab', 'gitlab', 'exact_git_v1'),
|
|
),
|
|
}
|
|
|
|
|
|
class WorkerAssignmentRunnerProtocolTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self._scanner_paths = (
|
|
runner.scanner.scan_config.trufflehog_path,
|
|
runner.scanner.scan_config.trufflehog_config,
|
|
runner.scanner.scan_config.work_dir,
|
|
)
|
|
|
|
def tearDown(self):
|
|
(
|
|
runner.scanner.scan_config.trufflehog_path,
|
|
runner.scanner.scan_config.trufflehog_config,
|
|
runner.scanner.scan_config.work_dir,
|
|
) = self._scanner_paths
|
|
|
|
def _root(
|
|
self, temporary, generation='a' * 32, release_gate=True,
|
|
deadline_seconds=300,
|
|
):
|
|
work = ensure_private_directory(os.path.join(temporary, 'work'), reject_reparse=True)
|
|
pending = live_assignment()
|
|
started = datetime.now(timezone.utc)
|
|
value = runner.build_runner_input(
|
|
pending,
|
|
generation=generation,
|
|
slot_id=0,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=deadline_seconds)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
)
|
|
name = runner.runner_root_name(0, 17, generation)
|
|
paths, digest = runner.create_runner_root(work, name, value)
|
|
if release_gate:
|
|
identity = runner.serialize_process_identity(runner.current_process_identity())
|
|
runner.publish_start_gate(
|
|
paths['start'], generation=generation, input_sha256=digest,
|
|
host=identity, payload=identity,
|
|
)
|
|
return work, pending, paths, digest
|
|
|
|
def test_runner_does_not_begin_permit_or_scan_work_before_identity_gate(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(
|
|
temporary, release_gate=False,
|
|
)
|
|
entered = threading.Event()
|
|
result = []
|
|
|
|
def execute(*_args, **_kwargs):
|
|
entered.set()
|
|
raise AssertionError('fixture stops after proving gate release')
|
|
|
|
def run():
|
|
result.append(runner.run_assignment(paths['root'], package_runtime()))
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
thread = threading.Thread(target=run)
|
|
thread.start()
|
|
self.assertFalse(entered.wait(0.2))
|
|
identity = runner.serialize_process_identity(
|
|
runner.current_process_identity(),
|
|
)
|
|
runner.publish_start_gate(
|
|
paths['start'], generation='a' * 32,
|
|
input_sha256=digest, host=identity, payload=identity,
|
|
)
|
|
self.assertTrue(entered.wait(2))
|
|
thread.join(5)
|
|
self.assertEqual(result, [1])
|
|
|
|
def test_input_is_closed_canonical_and_contains_no_worker_api_token(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _assignment, paths, digest = self._root(temporary)
|
|
value, loaded_digest = runner.load_runner_input(paths['input'])
|
|
self.assertEqual(loaded_digest, digest)
|
|
self.assertEqual(set(value), {
|
|
'schema', 'generation', 'slot_id', 'created_at',
|
|
'scan_started_at', 'scan_deadline_at', 'operation',
|
|
'watchdog_deadline_at', 'timeout_phase', 'assignment',
|
|
})
|
|
payload = open(paths['input'], 'r', encoding='utf-8').read()
|
|
self.assertNotIn('worker-api-secret', payload)
|
|
value['unexpected'] = True
|
|
with self.assertRaises(runner.RunnerProtocolError):
|
|
runner.validate_runner_input(value)
|
|
|
|
def test_subsecond_budget_keeps_validated_policy_and_never_passes_fractional_timeout(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, pending, paths, _digest = self._root(
|
|
temporary, deadline_seconds=0.5,
|
|
)
|
|
validated_kwargs = scan_execution.validate_scan_kwargs(
|
|
pending['reservation']['platform'], pending['scan_kwargs'],
|
|
)
|
|
self.assertEqual(validated_kwargs['timeout_sec'], 60.0)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim',
|
|
side_effect=AssertionError('fractional timeout reached execution validator'),
|
|
) as execute:
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 1)
|
|
execute.assert_not_called()
|
|
terminal, _digest = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=runner.load_runner_input(paths['input'])[1],
|
|
)
|
|
self.assertEqual(terminal['decision'], 'timed_out')
|
|
self.assertEqual(terminal['reason'], 'scan_stage_deadline')
|
|
|
|
def test_real_runner_boundaries_have_deterministic_phase_failures(self):
|
|
phases = (
|
|
'preparing', 'waiting_permit', 'scanning', 'filtering',
|
|
'cleaning', 'bundling',
|
|
)
|
|
for failed_phase in phases:
|
|
with self.subTest(phase=failed_phase), tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
|
|
def inject(phase):
|
|
if phase == failed_phase:
|
|
raise RuntimeError('injected phase failure')
|
|
|
|
def execute(_validated, _bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
for phase in phases[1:]:
|
|
phase_callback(phase)
|
|
raise AssertionError('phase injection did not stop the runner')
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(
|
|
runner.run_assignment(
|
|
paths['root'], package_runtime(), fault=inject,
|
|
),
|
|
1,
|
|
)
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32,
|
|
input_sha256=digest, operation='execute',
|
|
)
|
|
self.assertEqual(events[-1]['phase'], failed_phase)
|
|
terminal, _ = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest,
|
|
)
|
|
self.assertEqual(terminal['outcome']['status'], 'failed')
|
|
self.assertEqual(terminal['outcome']['final_phase'], failed_phase)
|
|
|
|
def test_source_failure_is_reported_in_the_honest_encompassing_scan_phase(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
|
|
def execute(_validated, _bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
phase_callback('waiting_permit')
|
|
phase_callback('scanning', {'operation': 'provider_and_scanner'})
|
|
raise RuntimeError('injected source failure')
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 1)
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32,
|
|
input_sha256=digest, operation='execute',
|
|
)
|
|
self.assertEqual(events[-1]['phase'], 'scanning')
|
|
self.assertNotIn('downloading', [item['phase'] for item in events])
|
|
self.assertNotIn('cloning', [item['phase'] for item in events])
|
|
|
|
def test_each_real_phase_stall_is_closed_by_the_terminal_barrier(self):
|
|
phases = (
|
|
'preparing', 'waiting_permit', 'scanning', 'filtering',
|
|
'cleaning', 'bundling',
|
|
)
|
|
for stalled_phase in phases:
|
|
with self.subTest(phase=stalled_phase), tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
result = []
|
|
|
|
def stall(phase):
|
|
if phase == stalled_phase:
|
|
entered.set()
|
|
self.assertTrue(release.wait(5))
|
|
|
|
def execute(_validated, _bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
for phase in phases[1:]:
|
|
phase_callback(phase)
|
|
raise AssertionError('phase stall did not close the runner')
|
|
|
|
def run():
|
|
result.append(runner.run_assignment(
|
|
paths['root'], package_runtime(), fault=stall,
|
|
))
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
thread = threading.Thread(target=run)
|
|
thread.start()
|
|
self.assertTrue(entered.wait(5))
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='scan_stage_deadline',
|
|
)
|
|
release.set()
|
|
thread.join(5)
|
|
self.assertEqual(result, [1])
|
|
|
|
def test_real_bundle_fsync_and_publish_stalls_lose_terminal_race(self):
|
|
for stage in ('before_fsync', 'before_rename'):
|
|
with self.subTest(stage=stage), tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
result = []
|
|
|
|
def stall(current, _writer):
|
|
if current == stage:
|
|
entered.set()
|
|
self.assertTrue(release.wait(5))
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, bundle_fault=None, **_kwargs):
|
|
return self._stage_fixture(
|
|
validated, bundle_root, phase_callback, bundle_fault,
|
|
)
|
|
|
|
def run():
|
|
result.append(runner.run_assignment(
|
|
paths['root'], package_runtime(), bundle_fault=stall,
|
|
))
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
thread = threading.Thread(target=run)
|
|
thread.start()
|
|
self.assertTrue(entered.wait(5))
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='scan_stage_deadline',
|
|
)
|
|
release.set()
|
|
thread.join(5)
|
|
self.assertEqual(result, [1])
|
|
terminal, _ = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest,
|
|
)
|
|
self.assertEqual(terminal['decision'], 'timed_out')
|
|
|
|
def test_real_permit_block_loses_to_terminal_timeout_barrier(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
waiting = threading.Event()
|
|
release = threading.Event()
|
|
result = []
|
|
|
|
class BlockedPermit:
|
|
def __enter__(self):
|
|
waiting.set()
|
|
self_test.assertTrue(release.wait(5))
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
self_test = self
|
|
|
|
def run():
|
|
result.append(runner.run_assignment(paths['root'], package_runtime()))
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
scan_execution, 'scan_slot_scope', return_value=BlockedPermit(),
|
|
):
|
|
thread = threading.Thread(target=run)
|
|
thread.start()
|
|
self.assertTrue(waiting.wait(5))
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32,
|
|
input_sha256=digest, operation='execute',
|
|
)
|
|
self.assertEqual(
|
|
[item['phase'] for item in events],
|
|
['preparing', 'waiting_permit'],
|
|
)
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='scan_stage_deadline',
|
|
)
|
|
release.set()
|
|
thread.join(5)
|
|
self.assertEqual(result, [1])
|
|
|
|
def test_successful_outcome_adopts_only_validated_identity_matching_bundle(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
work, pending, paths, digest = self._root(temporary)
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
for phase in (
|
|
'waiting_permit', 'scanning',
|
|
'filtering', 'cleaning', 'bundling',
|
|
):
|
|
phase_callback(phase)
|
|
reservation = validated['reservation']
|
|
with ResultBundleWriter.open(bundle_root, reservation) as writer:
|
|
commit = writer.finish({'queue_status': 'done'})
|
|
return StagedResult(
|
|
target=reservation.target,
|
|
scan_event_id=commit.scan_event_id,
|
|
bundle_id=commit.bundle_id,
|
|
reservation_id=commit.reservation_id,
|
|
scan_event_hash=commit.scan_event_hash,
|
|
actual_bytes=commit.actual_bytes,
|
|
relative_path=commit.relative_path,
|
|
frame_count=commit.frame_count,
|
|
finding_count=commit.finding_count,
|
|
error_count=commit.error_count,
|
|
candidate_count=commit.candidate_count,
|
|
queue_status='done',
|
|
source_failure=False,
|
|
source_failure_category='',
|
|
source_failure_auth_related=False,
|
|
first_error='',
|
|
)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 0)
|
|
terminal, outcome_digest = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
outcome = terminal['outcome']
|
|
self.assertEqual(outcome['status'], 'succeeded')
|
|
self.assertEqual(len(outcome_digest), 64)
|
|
destination = ensure_private_directory(
|
|
os.path.join(temporary, 'bundles'), reject_reparse=True,
|
|
)
|
|
for name in ('tmp', 'ready', 'quarantine'):
|
|
ensure_private_directory(os.path.join(destination, name), reject_reparse=True)
|
|
adopted = runner.adopt_runner_bundle(
|
|
work, os.path.basename(paths['root']), outcome, pending, destination,
|
|
)
|
|
self.assertEqual(adopted.header, BundleReservation.from_mapping(
|
|
pending['reservation'],
|
|
).header())
|
|
self.assertTrue(os.path.isfile(bundle_ready_path(destination, 'd' * 32)))
|
|
|
|
stale = dict(outcome)
|
|
stale['generation'] = 'b' * 32
|
|
with self.assertRaises(runner.RunnerProtocolError):
|
|
runner.adopt_runner_bundle(
|
|
work, os.path.basename(paths['root']), stale, pending, destination,
|
|
)
|
|
|
|
@staticmethod
|
|
def _stage_fixture(validated, bundle_root, phase_callback, bundle_fault=None):
|
|
for phase in (
|
|
'waiting_permit', 'scanning', 'filtering', 'cleaning', 'bundling',
|
|
):
|
|
phase_callback(phase)
|
|
reservation = validated['reservation']
|
|
with ResultBundleWriter.open(
|
|
bundle_root, reservation, fault=bundle_fault,
|
|
) as writer:
|
|
commit = writer.finish({'queue_status': 'done'})
|
|
return StagedResult(
|
|
target=reservation.target,
|
|
scan_event_id=commit.scan_event_id,
|
|
bundle_id=commit.bundle_id,
|
|
reservation_id=commit.reservation_id,
|
|
scan_event_hash=commit.scan_event_hash,
|
|
actual_bytes=commit.actual_bytes,
|
|
relative_path=commit.relative_path,
|
|
frame_count=commit.frame_count,
|
|
finding_count=commit.finding_count,
|
|
error_count=commit.error_count,
|
|
candidate_count=commit.candidate_count,
|
|
queue_status='done', source_failure=False,
|
|
source_failure_category='', source_failure_auth_related=False,
|
|
first_error='',
|
|
)
|
|
|
|
def test_timeout_terminal_decision_wins_race_against_late_success(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
staged = threading.Event()
|
|
release = threading.Event()
|
|
result = []
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
value = self._stage_fixture(validated, bundle_root, phase_callback)
|
|
staged.set()
|
|
self.assertTrue(release.wait(5))
|
|
return value
|
|
|
|
def run():
|
|
result.append(runner.run_assignment(paths['root'], package_runtime()))
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
thread = threading.Thread(target=run)
|
|
thread.start()
|
|
self.assertTrue(staged.wait(5))
|
|
terminal, won = runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='scan_stage_deadline',
|
|
)
|
|
self.assertTrue(won)
|
|
self.assertEqual(terminal['decision'], 'timed_out')
|
|
release.set()
|
|
thread.join(5)
|
|
self.assertEqual(result, [1])
|
|
terminal, _ = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
self.assertEqual(terminal['decision'], 'timed_out')
|
|
self.assertTrue(os.path.isfile(bundle_ready_path(paths['bundle_root'], 'd' * 32)))
|
|
|
|
def test_completed_terminal_decision_cannot_be_replaced_by_timeout(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
return self._stage_fixture(validated, bundle_root, phase_callback)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 0)
|
|
terminal, won = runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='scan_stage_deadline',
|
|
)
|
|
self.assertFalse(won)
|
|
self.assertEqual(terminal['decision'], 'completed')
|
|
|
|
def test_terminal_semantics_reject_phase_sequence_and_duration_tampering(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
return self._stage_fixture(validated, bundle_root, phase_callback)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 0)
|
|
terminal, _ = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
runner_input, _ = runner.load_runner_input(paths['input'])
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32, input_sha256=digest,
|
|
operation='execute',
|
|
)
|
|
wrong_phase = copy.deepcopy(terminal)
|
|
wrong_phase['outcome']['final_phase'] = 'cleaning'
|
|
with self.assertRaisesRegex(runner.RunnerProtocolError, 'journal'):
|
|
runner.validate_terminal_against_journal(
|
|
wrong_phase, events, runner_input,
|
|
)
|
|
wrong_duration = copy.deepcopy(terminal)
|
|
wrong_duration['outcome']['phase_durations']['idle'] = 1.0
|
|
with self.assertRaisesRegex(runner.RunnerProtocolError, 'journal'):
|
|
runner.validate_terminal_against_journal(
|
|
wrong_duration, events, runner_input,
|
|
)
|
|
|
|
def test_operation_requires_its_exact_first_phase(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'], 'a' * 32, digest,
|
|
)
|
|
journal.emit('bundling')
|
|
with self.assertRaisesRegex(runner.RunnerProtocolError, 'wrong operation phase'):
|
|
runner.read_runner_events(
|
|
paths['events'], generation='a' * 32,
|
|
input_sha256=digest, operation='execute',
|
|
)
|
|
|
|
def test_event_journal_clamps_backward_wall_clock(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'], 'a' * 32, digest,
|
|
)
|
|
with mock.patch.object(runner, 'utc_now', side_effect=(
|
|
'2026-09-26T08:59:23.344Z',
|
|
'2026-09-26T08:59:23.300Z',
|
|
'2026-09-26T08:59:23.250Z',
|
|
)):
|
|
first = journal.emit('preparing')
|
|
second = journal.emit('waiting_permit')
|
|
completed_at = journal.current_timestamp()
|
|
|
|
self.assertEqual(second['timestamp'], first['timestamp'])
|
|
self.assertEqual(second['phase_started_at'], first['timestamp'])
|
|
self.assertEqual(completed_at, first['timestamp'])
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32,
|
|
input_sha256=digest, operation='execute',
|
|
)
|
|
self.assertEqual(len(events), 2)
|
|
self.assertEqual(
|
|
runner.phase_durations_from_events(events, completed_at),
|
|
{'preparing': 0.0, 'waiting_permit': 0.0},
|
|
)
|
|
|
|
def test_malformed_and_incomplete_event_tail_is_never_interpreted_as_progress(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'], 'a' * 32, digest,
|
|
)
|
|
journal.emit('waiting_permit')
|
|
with open(paths['events'], 'ab') as handle:
|
|
handle.write(b'{"partial":')
|
|
events = runner.read_runner_events(
|
|
paths['events'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
self.assertEqual([item['phase'] for item in events], ['waiting_permit'])
|
|
with open(paths['events'], 'ab') as handle:
|
|
handle.write(b'false}\n')
|
|
with self.assertRaises(runner.RunnerProtocolError):
|
|
runner.read_runner_events(
|
|
paths['events'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
|
|
def test_fence_blocks_late_bundle_publication_fault_boundary(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation='a' * 32,
|
|
input_sha256=digest, decision='timed_out',
|
|
reason='controller_timeout',
|
|
)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'], 'a' * 32, digest,
|
|
)
|
|
with self.assertRaises(runner.RunnerFencedError):
|
|
journal.emit('waiting_permit')
|
|
|
|
def test_janitor_retains_active_slot_root_then_removes_abandoned_root(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
work, _pending, paths, _digest = self._root(temporary)
|
|
name = os.path.basename(paths['root'])
|
|
with mock.patch('janitor.exact_process_identity_state', return_value='dead'):
|
|
retained = runner.cleanup_abandoned_runner_roots(
|
|
work, minimum_age_sec=0, active_root_names=(name,),
|
|
)
|
|
self.assertEqual(retained['removed'], 0)
|
|
self.assertTrue(os.path.isdir(paths['root']))
|
|
removed = runner.cleanup_abandoned_runner_roots(
|
|
work, minimum_age_sec=0,
|
|
)
|
|
self.assertEqual(removed['removed'], 1)
|
|
self.assertFalse(os.path.exists(paths['root']))
|
|
|
|
def test_abandoned_tree_transfer_is_durable_validated_and_janitor_visible(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
work, _pending, paths, _digest = self._root(temporary)
|
|
reference = runner.transfer_runner_to_janitor(
|
|
work, os.path.basename(paths['root']), 'a' * 32,
|
|
)
|
|
destination = os.path.join(work, *reference.split('/'))
|
|
self.assertFalse(os.path.exists(paths['root']))
|
|
self.assertTrue(os.path.isdir(destination))
|
|
marker = runner.read_private_json(
|
|
os.path.join(destination, '.scanner-owner.json'),
|
|
)
|
|
intent = runner.read_private_json(
|
|
os.path.join(destination, '.janitor-transfer.json'),
|
|
)
|
|
self.assertEqual(marker['relative_path'], reference)
|
|
self.assertEqual(intent['status'], 'transferred')
|
|
with mock.patch('janitor.exact_process_identity_state', return_value='dead'):
|
|
report = runner.cleanup_abandoned_runner_roots(
|
|
work, minimum_age_sec=0,
|
|
)
|
|
self.assertEqual(report['removed'], 1)
|
|
self.assertFalse(os.path.exists(destination))
|
|
|
|
def test_bundle_fsync_fault_publishes_failed_outcome_not_ready_bundle(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
_work, _pending, paths, digest = self._root(temporary)
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, bundle_fault=None, **_kwargs):
|
|
return self._stage_fixture(
|
|
validated, bundle_root, phase_callback, bundle_fault,
|
|
)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
code = runner.run_assignment(
|
|
paths['root'], package_runtime(),
|
|
bundle_fault=lambda stage, _writer: (
|
|
(_ for _ in ()).throw(OSError('injected fsync failure'))
|
|
if stage == 'before_fsync' else None
|
|
),
|
|
)
|
|
self.assertEqual(code, 1)
|
|
terminal, _ = runner.load_generation_terminal(
|
|
paths['terminal'], generation='a' * 32, input_sha256=digest,
|
|
)
|
|
outcome = terminal['outcome']
|
|
self.assertEqual(outcome['status'], 'failed')
|
|
self.assertEqual(outcome['final_phase'], 'bundling')
|
|
self.assertIsNone(outcome['bundle'])
|
|
|
|
|
|
class NonExitingProcess:
|
|
def __init__(self):
|
|
executable = os.path.abspath(sys.executable)
|
|
self.host_identity = {
|
|
'pid': 101, 'creation_time': 'windows-filetime:1',
|
|
'creation_time_unix': 1.0, 'executable': executable, 'in_job': True,
|
|
}
|
|
self.payload_identity = {
|
|
'pid': 102, 'creation_time': 'windows-filetime:2',
|
|
'creation_time_unix': 1.0, 'executable': executable, 'in_job': True,
|
|
}
|
|
self.job_membership_verified = True
|
|
self.kills = 0
|
|
|
|
def poll(self):
|
|
return None
|
|
|
|
def kill(self):
|
|
self.kills += 1
|
|
|
|
def wait(self, timeout=None):
|
|
raise subprocess.TimeoutExpired('runner', timeout)
|
|
|
|
|
|
class CrashedProcess(NonExitingProcess):
|
|
def __init__(self, ordinal):
|
|
super().__init__()
|
|
self.host_identity['pid'] += ordinal * 10
|
|
self.payload_identity['pid'] += ordinal * 10
|
|
|
|
def poll(self):
|
|
return 7
|
|
|
|
def wait(self, timeout=None):
|
|
return 7
|
|
|
|
|
|
class KillableProcess(NonExitingProcess):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.alive = True
|
|
self.killed_at = None
|
|
|
|
def poll(self):
|
|
return None if self.alive else 1
|
|
|
|
def kill(self):
|
|
super().kill()
|
|
self.alive = False
|
|
self.killed_at = time.monotonic()
|
|
|
|
def wait(self, timeout=None):
|
|
self.alive = False
|
|
return 1
|
|
|
|
|
|
class WorkerSlotRunnerTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self._scanner_paths = (
|
|
runner.scanner.scan_config.trufflehog_path,
|
|
runner.scanner.scan_config.trufflehog_config,
|
|
runner.scanner.scan_config.work_dir,
|
|
)
|
|
|
|
def tearDown(self):
|
|
(
|
|
runner.scanner.scan_config.trufflehog_path,
|
|
runner.scanner.scan_config.trufflehog_config,
|
|
runner.scanner.scan_config.work_dir,
|
|
) = self._scanner_paths
|
|
|
|
def _slot(self, temporary, api=object(), process_factory=None):
|
|
state = ensure_private_directory(os.path.join(temporary, 'state'), reject_reparse=True)
|
|
bundles = ensure_private_directory(os.path.join(temporary, 'bundles'), reject_reparse=True)
|
|
work = ensure_private_directory(os.path.join(temporary, 'work'), reject_reparse=True)
|
|
for name in ('tmp', 'ready', 'quarantine'):
|
|
ensure_private_directory(os.path.join(bundles, name), reject_reparse=True)
|
|
return WorkerSlot(
|
|
0, api, build_compatibility(), state, bundles, work_root=work,
|
|
process_factory=process_factory or remote_worker_client.OwnedProcess,
|
|
package_runtime={
|
|
'code_manifest': {}, 'code_manifest_sha256': '9' * 64,
|
|
'trufflehog_path': 'trufflehog', 'git_path': 'git',
|
|
'detector_policy_path': 'detectors.yaml',
|
|
'capabilities': (
|
|
('github', 'github', 'exact_git_v1'),
|
|
('gitlab', 'gitlab', 'exact_git_v1'),
|
|
),
|
|
'bootstrap_path': os.path.join(APP_DIR, 'remote_worker_bootstrap.py'),
|
|
},
|
|
)
|
|
|
|
def test_assignment_runner_launch_hides_windows_console(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
process = NonExitingProcess()
|
|
slot = self._slot(temporary)
|
|
for platform, expected in (('nt', 0x08000000), ('posix', 0)):
|
|
captured = {}
|
|
|
|
def factory(*_args, **kwargs):
|
|
captured.update(kwargs)
|
|
return process
|
|
|
|
slot._process_factory = factory
|
|
with self.subTest(platform=platform), mock.patch.object(
|
|
remote_worker_client.os, 'name', platform,
|
|
), mock.patch.object(
|
|
remote_worker_client.subprocess,
|
|
'CREATE_NO_WINDOW', 0x08000000, create=True,
|
|
):
|
|
launched = slot._launch_process_bounded(
|
|
{}, {'root': slot.work_root}, 1.0,
|
|
)
|
|
self.assertIs(launched, process)
|
|
self.assertEqual(captured['creationflags'], expected)
|
|
|
|
def test_permit_starvation_nonexit_keeps_containment_pending(self):
|
|
process = NonExitingProcess()
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary, process_factory=lambda *_args, **_kwargs: process)
|
|
pending = live_assignment()
|
|
pending['deadlines']['target_scan_timeout_seconds'] = 1
|
|
state = {'phase': 'assigned', 'assignment': pending}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'RUNNER_STOP_TIMEOUT_SECONDS', 0.01,
|
|
), mock.patch.object(
|
|
slot, '_timeout_bundle', side_effect=lambda value: value,
|
|
) as timeout_bundle:
|
|
with self.assertRaises(remote_worker_client.RunnerContainmentPending):
|
|
slot._launch_runner(state)
|
|
timeout_bundle.assert_not_called()
|
|
self.assertGreaterEqual(process.kills, 2)
|
|
self.assertEqual(state['runner']['status'], 'stopping')
|
|
self.assertLess((datetime.now(timezone.utc) - started).total_seconds(), 2.0)
|
|
self.assertTrue(os.path.isfile(slot._runner_paths(state['runner'])['terminal']))
|
|
|
|
def test_cleanup_stall_records_cleaning_as_the_timed_out_phase(self):
|
|
process = NonExitingProcess()
|
|
captured = []
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
pending = live_assignment()
|
|
pending['deadlines']['target_scan_timeout_seconds'] = 1
|
|
state = {'phase': 'assigned', 'assignment': pending}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
paths = slot._runner_paths(state['runner'])
|
|
|
|
def launch(*_args, **_kwargs):
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'],
|
|
state['runner']['generation'], state['runner']['input_sha256'],
|
|
)
|
|
for phase in (
|
|
'preparing', 'waiting_permit', 'scanning',
|
|
'filtering', 'cleaning',
|
|
):
|
|
journal.emit(phase)
|
|
return process
|
|
|
|
slot._process_factory = launch
|
|
slot._event_phase = remote_worker_client.WorkerPhase.ASSIGNED
|
|
|
|
def timed_out(value):
|
|
captured.append(value['runner']['last_event']['phase'])
|
|
return value
|
|
|
|
with mock.patch.object(
|
|
remote_worker_client, 'RUNNER_STOP_TIMEOUT_SECONDS', 0.01,
|
|
), mock.patch.object(
|
|
slot, '_timeout_bundle', side_effect=timed_out,
|
|
):
|
|
with self.assertRaises(remote_worker_client.RunnerContainmentPending):
|
|
slot._launch_runner(state)
|
|
self.assertEqual(state['runner']['last_event']['phase'], 'cleaning')
|
|
self.assertEqual(captured, [])
|
|
|
|
def test_slot_state_migrates_shipped_legacy_shape_and_rejects_unknown_fields(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
slot._save({'phase': 'assigned', 'assignment': live_assignment()})
|
|
migrated = slot._load()
|
|
self.assertEqual(migrated['schema'], remote_worker_client.SLOT_STATE_SCHEMA)
|
|
self.assertIsNone(migrated['runner'])
|
|
malformed = {**migrated, 'server_command': 'forbidden'}
|
|
atomic_write_private_json(slot.state_path, malformed)
|
|
with self.assertRaisesRegex(WorkerClientError, 'state shape'):
|
|
slot._load()
|
|
|
|
def test_outcomeless_runner_crash_retries_three_generations_without_resetting_deadline(self):
|
|
launched = []
|
|
|
|
def factory(*_args, **_kwargs):
|
|
process = CrashedProcess(len(launched) + 1)
|
|
launched.append(process)
|
|
return process
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary, process_factory=factory)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with self.assertRaisesRegex(
|
|
runner.RunnerProtocolError, 'bounded generation retries',
|
|
):
|
|
slot._execute(state)
|
|
self.assertEqual(len(launched), 3)
|
|
self.assertIsNone(state['runner'])
|
|
self.assertEqual(len(state['retained_work']), 3)
|
|
deadlines = []
|
|
for reference in state['retained_work']:
|
|
value, _ = runner.load_runner_input(os.path.join(
|
|
slot.work_root, *reference.split('/'), runner.RUNNER_INPUT_NAME,
|
|
))
|
|
deadlines.append(value['scan_deadline_at'])
|
|
self.assertEqual(len(set(deadlines)), 1)
|
|
|
|
def test_timed_out_execute_generation_starts_one_timeout_bundle_without_crash_retries(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
scan_deadline = (started + timedelta(seconds=0.5)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z')
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment',
|
|
return_value={},
|
|
):
|
|
execute_runner = dict(slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
scan_deadline_at=scan_deadline,
|
|
watchdog_deadline_at=scan_deadline,
|
|
operation='execute', attempt=1,
|
|
))
|
|
paths = slot._runner_paths(execute_runner)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'],
|
|
execute_runner['generation'], execute_runner['input_sha256'],
|
|
)
|
|
for phase in ('preparing', 'waiting_permit', 'scanning'):
|
|
journal.emit(phase)
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation=execute_runner['generation'],
|
|
input_sha256=execute_runner['input_sha256'],
|
|
decision='timed_out', reason='scan_stage_deadline',
|
|
)
|
|
launched = []
|
|
|
|
def launch(value, **_kwargs):
|
|
launched.append(dict(value['runner']))
|
|
return value
|
|
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment',
|
|
return_value={},
|
|
), mock.patch.object(
|
|
slot, '_launch_runner', side_effect=launch,
|
|
):
|
|
completed = slot._execute(state)
|
|
self.assertIs(completed, state)
|
|
self.assertEqual(len(launched), 1)
|
|
self.assertEqual(launched[0]['operation'], 'timeout_bundle')
|
|
self.assertEqual(launched[0]['attempt'], 1)
|
|
self.assertEqual(launched[0]['scan_deadline_at'], scan_deadline)
|
|
self.assertEqual(launched[0]['timeout_phase'], 'scanning')
|
|
self.assertEqual(len(state['retained_work']), 1)
|
|
|
|
def test_restart_malformed_outcome_is_fenced_and_never_adopted(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
paths = slot._runner_paths(state['runner'])
|
|
atomic_write_private_json(paths['terminal'], {'schema': 1, 'stale': True})
|
|
state['runner']['attempt'] = 3
|
|
slot._save(state)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
), self.assertRaises(runner.RunnerProtocolError):
|
|
slot._execute(state)
|
|
self.assertTrue(os.path.isfile(os.path.join(
|
|
slot.work_root, 'abandoned', os.path.basename(paths['root']),
|
|
runner.RUNNER_TERMINAL_NAME,
|
|
)))
|
|
self.assertFalse(os.path.exists(bundle_ready_path(slot.bundle_root, 'd' * 32)))
|
|
|
|
def test_controller_rejects_completion_for_generation_already_marked_stopping(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
paths = slot._runner_paths(state['runner'])
|
|
identity = runner.serialize_process_identity(runner.current_process_identity())
|
|
runner.publish_start_gate(
|
|
paths['start'], generation=state['runner']['generation'],
|
|
input_sha256=state['runner']['input_sha256'],
|
|
host=identity, payload=identity,
|
|
)
|
|
|
|
def execute(validated, bundle_root, *_args, phase_callback=None, **_kwargs):
|
|
return WorkerAssignmentRunnerProtocolTests._stage_fixture(
|
|
validated, bundle_root, phase_callback,
|
|
)
|
|
|
|
with mock.patch.object(
|
|
runner.scanner, 'client_scan_launch_authority', return_value=nullcontext(),
|
|
), mock.patch.object(
|
|
runner.scanner, 'initialize_scanner_runtime',
|
|
), mock.patch.object(
|
|
runner, 'execute_protocol2_remote_claim', side_effect=execute,
|
|
):
|
|
self.assertEqual(runner.run_assignment(paths['root'], package_runtime()), 0)
|
|
state['runner']['status'] = 'stopping'
|
|
slot._save(state)
|
|
with self.assertRaisesRegex(
|
|
runner.RunnerProtocolError, 'closed runner generation',
|
|
):
|
|
slot._load_runner_terminal(state)
|
|
|
|
def test_late_ready_bundle_from_timed_out_generation_is_never_adopted(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
pending = live_assignment()
|
|
state = {'phase': 'assigned', 'assignment': pending}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
reservation = BundleReservation.from_mapping(pending['reservation'])
|
|
with ResultBundleWriter.open(slot.bundle_root, reservation) as writer:
|
|
writer.finish({'queue_status': 'done'})
|
|
paths = slot._runner_paths(state['runner'])
|
|
runner.publish_generation_terminal(
|
|
paths['terminal'], generation=state['runner']['generation'],
|
|
input_sha256=state['runner']['input_sha256'],
|
|
decision='timed_out', reason='scan_stage_deadline',
|
|
)
|
|
with self.assertRaisesRegex(
|
|
runner.RunnerProtocolError, 'lacks a valid completed generation',
|
|
):
|
|
slot._adopt_ready_bundle(state)
|
|
self.assertNotEqual(state['phase'], 'bundle_ready')
|
|
|
|
def test_restart_rerun_preserves_immutable_scan_clock_and_fences_generation(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
old_runner = dict(slot._new_runner(state))
|
|
old_paths = slot._runner_paths(old_runner)
|
|
with mock.patch.object(
|
|
slot, '_launch_runner', side_effect=lambda value: value,
|
|
):
|
|
recovered = slot._execute(state)
|
|
self.assertNotEqual(
|
|
recovered['runner']['generation'], old_runner['generation'],
|
|
)
|
|
self.assertEqual(
|
|
recovered['runner']['scan_started_at'], old_runner['scan_started_at'],
|
|
)
|
|
self.assertEqual(
|
|
recovered['runner']['scan_deadline_at'], old_runner['scan_deadline_at'],
|
|
)
|
|
self.assertTrue(os.path.isfile(os.path.join(
|
|
slot.work_root, 'abandoned', os.path.basename(old_paths['root']),
|
|
runner.RUNNER_TERMINAL_NAME,
|
|
)))
|
|
|
|
def test_controller_recovery_terminates_real_live_owned_containment_before_rerun(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
process = remote_worker_client.OwnedProcess(
|
|
[sys.executable, '-I', '-S', '-B', '-c', 'import time; time.sleep(60)'],
|
|
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
self.addCleanup(
|
|
lambda: process.kill() if process.poll() is None else None,
|
|
)
|
|
state['runner']['identity'] = {
|
|
'host': slot._identity_value(process.host_identity),
|
|
'payload': slot._identity_value(process.payload_identity),
|
|
'job_membership_verified': True,
|
|
}
|
|
state['runner']['status'] = 'running'
|
|
slot._save(state)
|
|
runner.bind_runner_owner(
|
|
slot.work_root, state['runner']['root_name'],
|
|
state['runner']['identity']['payload'],
|
|
)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
), mock.patch.object(
|
|
slot, '_launch_runner', side_effect=lambda value: value,
|
|
):
|
|
recovered = slot._execute(state)
|
|
self.assertIsNotNone(process.poll())
|
|
self.assertNotEqual(
|
|
recovered['runner']['generation'],
|
|
state['retained_work'][0].rsplit('-', 1)[-1],
|
|
)
|
|
|
|
def test_recovery_replays_valid_tail_with_original_runner_timestamps(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
events = []
|
|
slot = self._slot(temporary)
|
|
slot.event_callback = events.append
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
old = dict(slot._new_runner(state))
|
|
paths = slot._runner_paths(old)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'],
|
|
old['generation'], old['input_sha256'],
|
|
)
|
|
first = journal.emit('preparing')
|
|
second = journal.emit('waiting_permit')
|
|
with mock.patch.object(
|
|
slot, '_launch_runner', side_effect=lambda value: value,
|
|
):
|
|
slot._execute(state)
|
|
replayed = [
|
|
item for item in events
|
|
if (item.get('progress') or {}).get('runner_sequence')
|
|
]
|
|
self.assertEqual(
|
|
[item['timestamp'] for item in replayed],
|
|
[first['timestamp'], second['timestamp']],
|
|
)
|
|
self.assertEqual(
|
|
[item['phase_started_at'] for item in replayed],
|
|
[first['phase_started_at'], second['phase_started_at']],
|
|
)
|
|
|
|
def test_malformed_complete_event_is_fenced_before_generation_retry(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
old = dict(slot._new_runner(state))
|
|
paths = slot._runner_paths(old)
|
|
journal = runner.RunnerEventJournal(
|
|
paths['events'], paths['terminal'],
|
|
old['generation'], old['input_sha256'],
|
|
)
|
|
journal.emit('preparing')
|
|
with open(paths['events'], 'ab') as handle:
|
|
handle.write(b'{"bad":true}\n')
|
|
with mock.patch.object(
|
|
slot, '_launch_runner', side_effect=lambda value: value,
|
|
):
|
|
recovered = slot._execute(state)
|
|
self.assertNotEqual(recovered['runner']['generation'], old['generation'])
|
|
terminal, _ = runner.load_generation_terminal(
|
|
os.path.join(
|
|
slot.work_root, 'abandoned', old['root_name'],
|
|
runner.RUNNER_TERMINAL_NAME,
|
|
),
|
|
generation=old['generation'], input_sha256=old['input_sha256'],
|
|
)
|
|
self.assertEqual(terminal['decision'], 'fenced')
|
|
|
|
def test_deadline_produces_normal_phase_specific_timeout_bundle(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(state)
|
|
state['runner']['last_event'] = {
|
|
'schema': runner.RUNNER_PROTOCOL_SCHEMA,
|
|
'generation': state['runner']['generation'],
|
|
'input_sha256': state['runner']['input_sha256'],
|
|
'sequence': 1,
|
|
'timestamp': state['runner']['scan_started_at'],
|
|
'phase': 'waiting_permit',
|
|
'phase_started_at': state['runner']['scan_started_at'],
|
|
'progress': {},
|
|
}
|
|
slot._event_phase = remote_worker_client.WorkerPhase.WAITING_PERMIT
|
|
def publish_timeout(value, **_kwargs):
|
|
paths = slot._runner_paths(value['runner'])
|
|
identity = runner.serialize_process_identity(
|
|
runner.current_process_identity(),
|
|
)
|
|
runner.publish_start_gate(
|
|
paths['start'], generation=value['runner']['generation'],
|
|
input_sha256=value['runner']['input_sha256'],
|
|
host=identity, payload=identity,
|
|
)
|
|
self.assertEqual(
|
|
runner.run_assignment(paths['root'], package_runtime()), 0,
|
|
)
|
|
terminal = slot._load_runner_terminal(value)
|
|
return slot._adopt_runner_outcome(value, terminal['outcome'])
|
|
|
|
with mock.patch.object(
|
|
slot, '_launch_runner', side_effect=publish_timeout,
|
|
):
|
|
completed = slot._timeout_bundle(state)
|
|
self.assertEqual(completed['phase'], 'bundle_ready')
|
|
self.assertEqual(len(completed['retained_work']), 2)
|
|
self.assertTrue(all(
|
|
os.path.isdir(os.path.join(slot.work_root, *item.split('/')))
|
|
for item in completed['retained_work']
|
|
))
|
|
reader = ResultBundleReader(
|
|
bundle_ready_path(slot.bundle_root, 'd' * 32),
|
|
max_event_bytes=1024 * 1024,
|
|
)
|
|
metadata = reader.metadata()
|
|
self.assertEqual(metadata['error_class'], 'timeout')
|
|
self.assertEqual(metadata['scan_meta']['timed_out_phase'], 'waiting_permit')
|
|
self.assertTrue(metadata['scan_meta']['full_stage_timeout'])
|
|
self.assertEqual(reader.validate().diagnostic_count, 1)
|
|
self.assertEqual(
|
|
list(reader.iter_diagnostics())[0]['code'], 'scan.stage_timeout',
|
|
)
|
|
self.assertIn('deadline exceeded', list(reader.iter_errors())[0].lower())
|
|
|
|
def test_timeout_bundle_publish_stall_has_its_own_bounded_watchdog(self):
|
|
process = NonExitingProcess()
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary, process_factory=lambda *_args, **_kwargs: process)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=1)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
watchdog_deadline_at=(started + timedelta(seconds=0.2)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
operation='timeout_bundle', timeout_phase='cleaning',
|
|
)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'RUNNER_STOP_TIMEOUT_SECONDS', 0.01,
|
|
), self.assertRaisesRegex(
|
|
remote_worker_client.RunnerContainmentPending, 'teardown remains pending',
|
|
):
|
|
slot._launch_runner(state, allow_timeout_fallback=False)
|
|
self.assertGreaterEqual(process.kills, 2)
|
|
self.assertEqual(state['runner']['status'], 'stopping')
|
|
|
|
def test_blocked_owned_process_launcher_cannot_cross_absolute_deadline(self):
|
|
release = threading.Event()
|
|
process = KillableProcess()
|
|
|
|
def blocked_factory(*_args, **_kwargs):
|
|
release.wait(5)
|
|
return process
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary, process_factory=blocked_factory)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=1)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
watchdog_deadline_at=(started + timedelta(seconds=0.2)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
operation='timeout_bundle', timeout_phase='preparing',
|
|
)
|
|
before = time.monotonic()
|
|
with self.assertRaises(runner.RunnerProtocolError):
|
|
slot._launch_runner(state, allow_timeout_fallback=False)
|
|
self.assertLess(time.monotonic() - before, 1.0)
|
|
release.set()
|
|
deadline = time.monotonic() + 2
|
|
while process.kills == 0 and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
self.assertGreaterEqual(process.kills, 1)
|
|
self.assertTrue(slot._runner_launch_cleanup.wait(2))
|
|
|
|
def test_blocked_prelaunch_filesystem_becomes_normal_preparing_timeout_bundle(self):
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
finished = threading.Event()
|
|
original_create = remote_worker_client.create_runner_root
|
|
calls = [0]
|
|
|
|
def blocked_create(*args, **kwargs):
|
|
calls[0] += 1
|
|
if calls[0] > 1:
|
|
return original_create(*args, **kwargs)
|
|
entered.set()
|
|
release.wait(5)
|
|
try:
|
|
return original_create(*args, **kwargs)
|
|
finally:
|
|
finished.set()
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(temporary)
|
|
pending = live_assignment()
|
|
pending['deadlines']['target_scan_timeout_seconds'] = 1
|
|
state = {'phase': 'assigned', 'assignment': pending}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
|
|
def publish_timeout(value, **_kwargs):
|
|
paths = slot._runner_paths(value['runner'])
|
|
identity = runner.serialize_process_identity(
|
|
runner.current_process_identity(),
|
|
)
|
|
runner.publish_start_gate(
|
|
paths['start'], generation=value['runner']['generation'],
|
|
input_sha256=value['runner']['input_sha256'],
|
|
host=identity, payload=identity,
|
|
)
|
|
self.assertEqual(
|
|
runner.run_assignment(paths['root'], package_runtime()), 0,
|
|
)
|
|
terminal = slot._load_runner_terminal(value)
|
|
return slot._adopt_runner_outcome(value, terminal['outcome'])
|
|
|
|
before = time.monotonic()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
), mock.patch.object(
|
|
remote_worker_client, 'create_runner_root', side_effect=blocked_create,
|
|
), mock.patch.object(
|
|
runner, 'validate_protocol2_remote_assignment',
|
|
return_value={'reservation': BundleReservation.from_mapping(
|
|
pending['reservation'],
|
|
)},
|
|
), mock.patch.object(
|
|
slot, '_launch_runner', side_effect=publish_timeout,
|
|
):
|
|
completed = slot._execute(state)
|
|
self.assertTrue(entered.is_set())
|
|
self.assertLess(time.monotonic() - before, 2.0)
|
|
self.assertEqual(completed['phase'], 'bundle_ready')
|
|
metadata = ResultBundleReader(
|
|
bundle_ready_path(slot.bundle_root, 'd' * 32),
|
|
max_event_bytes=1024 * 1024,
|
|
).metadata()
|
|
self.assertEqual(metadata['error_class'], 'timeout')
|
|
self.assertEqual(metadata['scan_meta']['timed_out_phase'], 'preparing')
|
|
release.set()
|
|
self.assertTrue(finished.wait(2))
|
|
|
|
def test_watchdog_kills_while_state_persistence_is_blocked(self):
|
|
process = KillableProcess()
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(
|
|
temporary, process_factory=lambda *_args, **_kwargs: process,
|
|
)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=2)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
watchdog_deadline_at=(started + timedelta(seconds=1)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
operation='timeout_bundle', timeout_phase='preparing',
|
|
)
|
|
original_save = slot._save
|
|
|
|
def slow_save(value):
|
|
if (value.get('runner') or {}).get('status') == 'running':
|
|
time.sleep(1.4)
|
|
return original_save(value)
|
|
|
|
before = time.monotonic()
|
|
with mock.patch.object(slot, '_save', side_effect=slow_save), \
|
|
self.assertRaises(remote_worker_client.RunnerContainmentPending):
|
|
slot._launch_runner(state, allow_timeout_fallback=False)
|
|
self.assertIsNotNone(process.killed_at)
|
|
self.assertLess(process.killed_at - before, 1.3)
|
|
persisted = slot._load()
|
|
self.assertEqual(persisted['phase'], 'assigned')
|
|
self.assertEqual(
|
|
persisted['runner']['generation'], state['runner']['generation'],
|
|
)
|
|
self.assertNotEqual(persisted['phase'], 'bundle_ready')
|
|
self.assertIn(0, remote_worker_client.persisted_slot_ids(slot.state_dir))
|
|
|
|
def test_blocked_startup_gate_write_enters_preparing_timeout_result_path(self):
|
|
process = KillableProcess()
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(
|
|
temporary, process_factory=lambda *_args, **_kwargs: process,
|
|
)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=2)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
watchdog_deadline_at=(started + timedelta(seconds=1)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
operation='execute',
|
|
)
|
|
original_gate = remote_worker_client.publish_start_gate
|
|
|
|
def slow_gate(*args, **kwargs):
|
|
time.sleep(1.4)
|
|
return original_gate(*args, **kwargs)
|
|
|
|
captured = []
|
|
|
|
def timeout_result(value):
|
|
captured.append(
|
|
(value['runner'].get('last_event') or {}).get('phase')
|
|
or 'preparing'
|
|
)
|
|
return value
|
|
|
|
before = time.monotonic()
|
|
with mock.patch.object(
|
|
remote_worker_client, 'publish_start_gate', side_effect=slow_gate,
|
|
), mock.patch.object(
|
|
slot, '_timeout_bundle', side_effect=timeout_result,
|
|
):
|
|
slot._launch_runner(state)
|
|
terminal, _ = runner.load_generation_terminal(
|
|
slot._runner_paths(state['runner'])['terminal'],
|
|
generation=state['runner']['generation'],
|
|
input_sha256=state['runner']['input_sha256'],
|
|
)
|
|
self.assertEqual(terminal['decision'], 'timed_out')
|
|
self.assertEqual(captured, ['preparing'])
|
|
self.assertLess(process.killed_at - before, 1.3)
|
|
|
|
def test_watchdog_kills_while_event_drain_is_blocked(self):
|
|
process = KillableProcess()
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
slot = self._slot(
|
|
temporary, process_factory=lambda *_args, **_kwargs: process,
|
|
)
|
|
state = {'phase': 'assigned', 'assignment': live_assignment()}
|
|
slot._save(state)
|
|
state = slot._load()
|
|
started = datetime.now(timezone.utc)
|
|
with mock.patch.object(
|
|
remote_worker_client, 'validate_protocol2_remote_assignment', return_value={},
|
|
):
|
|
slot._new_runner(
|
|
state,
|
|
scan_started_at=started.isoformat(timespec='milliseconds').replace('+00:00', 'Z'),
|
|
scan_deadline_at=(started + timedelta(seconds=2)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
watchdog_deadline_at=(started + timedelta(seconds=1)).isoformat(
|
|
timespec='milliseconds',
|
|
).replace('+00:00', 'Z'),
|
|
operation='timeout_bundle', timeout_phase='cleaning',
|
|
)
|
|
before = time.monotonic()
|
|
with mock.patch.object(
|
|
slot, '_drain_runner_events', side_effect=lambda *_args, **_kwargs: time.sleep(1.4),
|
|
), self.assertRaises(runner.RunnerProtocolError):
|
|
slot._launch_runner(state, allow_timeout_fallback=False)
|
|
self.assertIsNotNone(process.killed_at)
|
|
self.assertLess(process.killed_at - before, 1.3)
|
|
|
|
def test_upload_retry_preserves_bundle_and_uses_actual_receipt_wait_phase(self):
|
|
class API:
|
|
def __init__(self):
|
|
self.calls = 0
|
|
|
|
def status(self, reservation_id):
|
|
return {
|
|
'reservation_id': reservation_id, 'state': 'scanning',
|
|
'resolution': None, 'expires_at': '2999-01-01T00:00:00Z',
|
|
}
|
|
|
|
def upload(self, reservation_id, path):
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
raise WorkerClientError('injected upload retry')
|
|
digest = runner.hashlib.sha256(open(path, 'rb').read()).hexdigest()
|
|
return {
|
|
'reservation_id': reservation_id, 'receipt_id': '1' * 64,
|
|
'bundle_id': 'd' * 32, 'scan_event_id': 'e' * 32,
|
|
'resolution': 'bundle_accepted', 'payload_sha256': digest,
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
api = API()
|
|
events = []
|
|
slot = self._slot(temporary, api=api)
|
|
slot.event_callback = events.append
|
|
pending = live_assignment()
|
|
reservation = BundleReservation.from_mapping(pending['reservation'])
|
|
with ResultBundleWriter.open(slot.bundle_root, reservation) as writer:
|
|
commit = writer.finish({'queue_status': 'done'})
|
|
slot._save({
|
|
'schema': remote_worker_client.SLOT_STATE_SCHEMA,
|
|
'phase': 'bundle_ready', 'assignment': pending,
|
|
'runner': None, 'retained_work': [], 'bundle': commit.as_dict(),
|
|
})
|
|
with self.assertRaises(WorkerClientError):
|
|
slot.step()
|
|
self.assertTrue(os.path.isfile(bundle_ready_path(slot.bundle_root, 'd' * 32)))
|
|
self.assertTrue(slot.step())
|
|
self.assertEqual(api.calls, 2)
|
|
phases = [item['phase'] for item in events]
|
|
self.assertIn('backoff', phases)
|
|
self.assertEqual(phases[-2:], ['awaiting_receipt', 'idle'])
|
|
|
|
def test_upload_request_send_and_response_wait_have_distinct_blocked_phases(self):
|
|
send_entered = threading.Event()
|
|
release_send = threading.Event()
|
|
response_entered = threading.Event()
|
|
release_response = threading.Event()
|
|
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
events = []
|
|
api = object.__new__(remote_worker_client.WorkerHTTPClient)
|
|
api.token = 'x' * 32
|
|
api.status = lambda reservation_id: {
|
|
'reservation_id': reservation_id, 'state': 'scanning',
|
|
'resolution': None, 'expires_at': '2999-01-01T00:00:00Z',
|
|
}
|
|
slot = self._slot(temporary, api=api)
|
|
slot.event_callback = events.append
|
|
pending = live_assignment()
|
|
reservation = BundleReservation.from_mapping(pending['reservation'])
|
|
with ResultBundleWriter.open(slot.bundle_root, reservation) as writer:
|
|
commit = writer.finish({'queue_status': 'done'})
|
|
path = bundle_ready_path(slot.bundle_root, reservation.bundle_id)
|
|
payload_sha256 = runner.hashlib.sha256(open(path, 'rb').read()).hexdigest()
|
|
receipt = {
|
|
'reservation_id': reservation.reservation_id,
|
|
'receipt_id': '1' * 64,
|
|
'bundle_id': reservation.bundle_id,
|
|
'scan_event_id': reservation.scan_event_id,
|
|
'resolution': 'bundle_accepted',
|
|
'payload_sha256': payload_sha256,
|
|
}
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
@staticmethod
|
|
def read(_maximum):
|
|
return runner.json.dumps(receipt).encode('utf-8')
|
|
|
|
@staticmethod
|
|
def getheader(_name):
|
|
return None
|
|
|
|
class Connection:
|
|
def putrequest(self, *_args):
|
|
pass
|
|
|
|
def putheader(self, *_args):
|
|
pass
|
|
|
|
def endheaders(self):
|
|
pass
|
|
|
|
def send(self, _chunk):
|
|
send_entered.set()
|
|
self_test.assertTrue(release_send.wait(5))
|
|
|
|
def getresponse(self):
|
|
response_entered.set()
|
|
self_test.assertTrue(release_response.wait(5))
|
|
return Response()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
self_test = self
|
|
api._connection = Connection
|
|
slot._save({
|
|
'schema': remote_worker_client.SLOT_STATE_SCHEMA,
|
|
'phase': 'bundle_ready', 'assignment': pending,
|
|
'runner': None, 'retained_work': [], 'bundle': commit.as_dict(),
|
|
})
|
|
result = []
|
|
thread = threading.Thread(target=lambda: result.append(slot.step()))
|
|
thread.start()
|
|
self.assertTrue(send_entered.wait(5))
|
|
self.assertEqual(events[-1]['phase'], 'uploading')
|
|
self.assertFalse(response_entered.is_set())
|
|
release_send.set()
|
|
self.assertTrue(response_entered.wait(5))
|
|
self.assertEqual(events[-1]['phase'], 'awaiting_receipt')
|
|
release_response.set()
|
|
thread.join(5)
|
|
self.assertEqual(result, [True])
|
|
|
|
def test_upload_send_failure_enters_backoff_before_transport_retry(self):
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
events = []
|
|
api = object.__new__(remote_worker_client.WorkerHTTPClient)
|
|
api.token = 'x' * 32
|
|
api.status = lambda reservation_id: {
|
|
'reservation_id': reservation_id, 'state': 'scanning',
|
|
'resolution': None, 'expires_at': '2999-01-01T00:00:00Z',
|
|
}
|
|
slot = self._slot(temporary, api=api)
|
|
slot.event_callback = events.append
|
|
pending = live_assignment()
|
|
reservation = BundleReservation.from_mapping(pending['reservation'])
|
|
with ResultBundleWriter.open(slot.bundle_root, reservation) as writer:
|
|
commit = writer.finish({'queue_status': 'done'})
|
|
path = bundle_ready_path(slot.bundle_root, reservation.bundle_id)
|
|
payload_sha256 = runner.hashlib.sha256(open(path, 'rb').read()).hexdigest()
|
|
attempts = [0]
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
@staticmethod
|
|
def read(_maximum):
|
|
return runner.json.dumps({
|
|
'reservation_id': reservation.reservation_id,
|
|
'receipt_id': '1' * 64,
|
|
'bundle_id': reservation.bundle_id,
|
|
'scan_event_id': reservation.scan_event_id,
|
|
'resolution': 'bundle_accepted',
|
|
'payload_sha256': payload_sha256,
|
|
}).encode('utf-8')
|
|
|
|
@staticmethod
|
|
def getheader(_name):
|
|
return None
|
|
|
|
class Connection:
|
|
def putrequest(self, *_args):
|
|
pass
|
|
|
|
def putheader(self, *_args):
|
|
pass
|
|
|
|
def endheaders(self):
|
|
pass
|
|
|
|
def send(self, _chunk):
|
|
if attempts[0] == 0:
|
|
attempts[0] += 1
|
|
raise OSError('injected send failure')
|
|
|
|
def getresponse(self):
|
|
return Response()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
api._connection = Connection
|
|
slot._save({
|
|
'schema': remote_worker_client.SLOT_STATE_SCHEMA,
|
|
'phase': 'bundle_ready', 'assignment': pending,
|
|
'runner': None, 'retained_work': [], 'bundle': commit.as_dict(),
|
|
})
|
|
with self.assertRaises(OSError):
|
|
slot.step()
|
|
self.assertEqual(events[-1]['phase'], 'backoff')
|
|
self.assertTrue(slot.step())
|
|
self.assertIn('awaiting_receipt', [item['phase'] for item in events])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|