#!/usr/bin/env python3 """Exercise the real packaged Windows and Linux remote workers end to end. Docker is intentionally reached only through the configured WSL distribution. The verifier creates individually named and labelled resources, never invokes Compose or a broad cleanup command, and retains all owned artifacts on failure. """ import argparse import hashlib import json import os from pathlib import Path import re import secrets import shutil import socket import ssl import stat import subprocess import sys import threading import time LINUX_IMAGE = 'truf-remote-worker:linux-x86_64' TEST_IMAGE = 'truf-worker-test:test' WINDOWS_ARTIFACT = 'dist/truf-worker-windows-x86_64' TARGETS = ( 'https://gitlab.com/truf-e2e/a.git', 'https://gitlab.com/truf-e2e/b.git', ) DOCKER_TARGET = 'docker.io/truf-e2e/synthetic@sha256:' + ('7' * 64) HUGGINGFACE_TARGET = 'truf-e2e/synthetic' DIRECT_TARGETS = { 'dockerhub': DOCKER_TARGET, 'huggingface': HUGGINGFACE_TARGET, } ALL_TARGETS = (*TARGETS, DOCKER_TARGET, HUGGINGFACE_TARGET) PARALLELISM = 2 MAX_OUTPUT = 2 * 1024 * 1024 MAX_DOCKER_RESOURCES = 1024 MAX_SNAPSHOT_BYTES = 8 * 1024 * 1024 MAX_SAFE_EVIDENCE = 64 * 1024 RUN_LABEL = 'com.truf.packaged-worker-e2e.run' PHASE_LABEL = 'com.truf.packaged-worker-e2e.phase' KIND_LABEL = 'com.truf.packaged-worker-e2e.kind' RESOURCE_NAME = re.compile(r'truf-packaged-worker-e2e-[a-f0-9]{16}-(windows|linux)-[a-z-]+') HEX_40 = re.compile(r'[a-f0-9]{40}') HEX_64 = re.compile(r'[a-f0-9]{64}') CONTAINER_METADATA_FORMAT = ( '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Image}},' '"status":{{json .State.Status}},"running":{{json .State.Running}},' '"paused":{{json .State.Paused}},"restarting":{{json .State.Restarting}},' '"dead":{{json .State.Dead}},"mounts":{{json .Mounts}}}' ) VOLUME_METADATA_FORMAT = ( '{"name":{{json .Name}},"driver":{{json .Driver}},"scope":{{json .Scope}},' '"created":{{json .CreatedAt}},"mountpoint":{{json .Mountpoint}},' '"labels":{{json .Labels}},"options":{{json .Options}}}' ) def isolated_test_subnet(run_id, scope): value = int.from_bytes( hashlib.sha256(f'{run_id}\0{scope}'.encode('ascii')).digest()[:2], 'big', ) & 0x1fff return f'198.{18 + (value >> 12)}.{(value >> 4) & 0xff}.{(value & 0xf) * 16}/28' LOOPBACK_PROXY_SCRIPT = r'''import socket import sys import threading listen_port = int(sys.argv[1]) upstream_host = sys.argv[2] upstream_port = int(sys.argv[3]) def pump(source, destination): try: while True: block = source.recv(64 * 1024) if not block: break destination.sendall(block) except OSError: pass try: destination.shutdown(socket.SHUT_WR) except OSError: pass def bridge(client): upstream = None try: upstream = socket.create_connection((upstream_host, upstream_port), timeout=5) print('proxy upstream connected', flush=True) client.settimeout(None) upstream.settimeout(None) outbound = threading.Thread(target=pump, args=(client, upstream), daemon=True) outbound.start() pump(upstream, client) outbound.join(5) except OSError as exc: print('proxy upstream failed: ' + type(exc).__name__, flush=True) finally: client.close() if upstream is not None: upstream.close() listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(('127.0.0.1', listen_port)) listener.listen(32) print('proxy ready', flush=True) while True: connection, _ = listener.accept() threading.Thread(target=bridge, args=(connection,), daemon=True).start() ''' SEED_SCRIPT = r'''import os import stat root = '/seed' for current, directories, files in os.walk(root, topdown=False, followlinks=False): for name in directories + files: path = os.path.join(current, name) details = os.lstat(path) if stat.S_ISLNK(details.st_mode) or not ( stat.S_ISDIR(details.st_mode) or stat.S_ISREG(details.st_mode) ): raise SystemExit('unsupported seed entry') os.chown(path, 10001, 10001, follow_symlinks=False) os.chmod(path, 0o700 if stat.S_ISDIR(details.st_mode) else 0o600) os.chown(root, 10001, 10001, follow_symlinks=False) os.chmod(root, 0o700) os.unlink('/seed/_seed.py') ''' INSPECT_WORKER_SCRIPT = r'''import hashlib import json import os from pathlib import Path import stat def digest(path): value = hashlib.sha256() with open(path, 'rb', buffering=0) as handle: for block in iter(lambda: handle.read(1024 * 1024), b''): value.update(block) return value.hexdigest() def regular(path): details = os.stat(path, follow_symlinks=False) if not stat.S_ISREG(details.st_mode) or os.path.islink(path): raise RuntimeError('non-regular worker artifact') return details def tree_identity(root): identity = hashlib.sha256() count = 0 active_entries = 0 root = Path(root) if not root.exists(): return {'count': 0, 'active_entries': 0, 'sha256': identity.hexdigest()} for current, directories, files in os.walk(root, followlinks=False): directories.sort() files.sort() for name in directories: path = Path(current) / name if path.is_symlink() or not path.is_dir(): raise RuntimeError('unsupported worker directory') if path.relative_to(root).parts[0] != 'abandoned': active_entries += 1 for name in files: path = Path(current) / name details = regular(path) if path.relative_to(root).parts[0] != 'abandoned': active_entries += 1 relative = path.relative_to(root).as_posix().encode('utf-8') identity.update(relative + b'\0') identity.update(str(details.st_size).encode('ascii') + b'\0') identity.update(str(details.st_mtime_ns).encode('ascii') + b'\0') identity.update(digest(path).encode('ascii') + b'\n') count += 1 return { 'count': count, 'active_entries': active_entries, 'sha256': identity.hexdigest(), } states = {} state_root = Path('/data/state-base/truf/remote-worker') if state_root.exists(): for path in sorted(state_root.glob('slot-*.json')): details = regular(path) with open(path, 'rb') as handle: value = json.load(handle) reservation = dict((value.get('assignment') or {}).get('reservation') or {}) assignment = dict(value.get('assignment') or {}) snapshot = dict(assignment.get('execution_snapshot') or {}) planning = dict(snapshot.get('planning') or {}) states[path.name] = { 'phase': value.get('phase'), 'reservation_id': reservation.get('reservation_id'), 'bundle_id': reservation.get('bundle_id'), 'scan_event_id': reservation.get('scan_event_id'), 'source': reservation.get('source'), 'platform': reservation.get('platform'), 'target': reservation.get('target'), 'planning_kind': planning.get('kind'), 'sha256': digest(path), 'size': details.st_size, 'mtime_ns': details.st_mtime_ns, } bundles = {} bundle_root = Path('/data/client/truf/remote-worker/bundles') if bundle_root.exists(): for path in sorted(bundle_root.rglob('*.trb')): details = regular(path) bundles[path.relative_to(bundle_root).as_posix()] = { 'sha256': digest(path), 'size': details.st_size, 'mtime_ns': details.st_mtime_ns, } shutdown_receipt = None receipt_path = state_root / 'control' / 'worker.exit.json' if receipt_path.exists(): regular(receipt_path) with open(receipt_path, 'rb') as handle: shutdown_receipt = json.load(handle) print(json.dumps({ 'states': states, 'bundles': bundles, 'work': tree_identity('/data/client/truf/remote-worker/work'), 'shutdown_receipt': shutdown_receipt, }, ensure_ascii=True, sort_keys=True, separators=(',', ':'))) ''' PUBLISH_DIRECT_SCRIPT = r'''import json import os from datetime import datetime, timezone from pathlib import Path import stat import sys def fail(message): raise RuntimeError('direct bundle publisher: ' + message) def require(condition, message): if not condition: fail(message) require(len(sys.argv) == 4, 'arguments') package_root = Path(sys.argv[1]).resolve(strict=True) state_root = Path(sys.argv[2]).resolve(strict=True) bundle_root = Path(sys.argv[3]).resolve(strict=True) manifest_path = package_root / 'worker-package.json' with open(manifest_path, 'rb') as handle: bootstrap_manifest = json.load(handle) app_root = package_root / str(bootstrap_manifest.get('app_root') or '') require(app_root.is_dir(), 'application root') sys.path.insert(0, os.fspath(app_root / 'dependencies')) sys.path.insert(0, os.fspath(app_root)) from scan_execution import ( # noqa: E402 QueueDispositionPolicy, stage_scan_result_in_scope, validate_protocol2_remote_assignment, ) from worker_package import ( # noqa: E402 load_worker_package_manifest, worker_package_build_compatibility, ) manifest = load_worker_package_manifest(manifest_path) local_build = worker_package_build_compatibility(manifest) capabilities = tuple( (item['source'], item['platform'], item['planning_kind']) for item in manifest['capabilities'] ) expected = { ('dockerhub', 'docker', 'docker_direct_v1'), ('huggingface', 'huggingface', 'huggingface_space_v1'), } state_paths = sorted(state_root.glob('slot-*.json')) require({path.name for path in state_paths} == {'slot-0.json', 'slot-1.json'}, 'slot states') require(not any(bundle_root.rglob('*.trb')), 'bundle storage is not empty') seen = set() for path in state_paths: details = path.stat(follow_symlinks=False) require(stat.S_ISREG(details.st_mode) and not path.is_symlink(), 'regular slot state') require(0 < details.st_size <= 64 * 1024 * 1024, 'slot state bound') with open(path, 'rb') as handle: state = json.load(handle) require(state.get('phase') == 'assigned', 'slot phase') assignment = dict(state.get('assignment') or {}) validated = validate_protocol2_remote_assignment( assignment, local_build, capabilities, ) reservation = validated['reservation'] identity = ( reservation.source, reservation.platform, validated['planning_kind'], ) require(identity in expected and identity not in seen, 'direct capability') require( validated['snapshot']['credential_ref'] == { 'source': reservation.source, 'auth_entry': '', }, 'direct credential reference', ) now = datetime.now(timezone.utc).isoformat() result = { 'target': reservation.target, 'scan_type': reservation.platform, 'scan_event_id': reservation.scan_event_id, 'findings': [], 'errors': [], 'warnings': [], 'scan_started_at': now, 'timestamp': now, 'duration_sec': 0.0, 'scan_meta': {'synthetic_claim_to_ingestion': True}, } limits = dict(assignment['limits']) staged = stage_scan_result_in_scope( result, reservation, os.fspath(bundle_root), assignment['event_scan_options'], QueueDispositionPolicy(**dict(assignment['queue_policy'])), attempts=int(assignment['reservation'].get('attempts') or 1), candidate_max_items=int(limits.get('candidate_max_items') or 2000), candidate_max_bytes=int(limits.get('candidate_max_bytes') or 2 * 1024 * 1024), require_s_drive=False, ) require( staged.queue_status == 'done' and staged.finding_count == staged.error_count == staged.candidate_count == 0, 'clean staged bundle', ) seen.add(identity) require(seen == expected, 'direct source coverage') require(len(tuple(bundle_root.rglob('*.trb'))) == 2, 'published bundle count') print('published direct bundles: 2') ''' class Failure(RuntimeError): """A fixed diagnostic label that cannot contain credentials or findings.""" def require(condition, label): if not condition: raise Failure(label) def sha256_bytes(value): return hashlib.sha256(value).hexdigest() def sha256_file(path): digest = hashlib.sha256() with open(path, 'rb', buffering=0) as handle: for block in iter(lambda: handle.read(1024 * 1024), b''): digest.update(block) return digest.hexdigest() def write_bytes(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) with open(path, 'xb') as handle: handle.write(value) def write_json(path, value): write_bytes( path, json.dumps( value, ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii') + b'\n', ) def load_json_bytes(value, label): require(len(value) <= MAX_OUTPUT, label + '_size') try: result = json.loads(value.decode('utf-8', errors='strict')) except (UnicodeError, ValueError): raise Failure(label + '_json') from None return result def regular_file(path, label, maximum=MAX_OUTPUT): path = Path(path) try: details = path.stat(follow_symlinks=False) except OSError: raise Failure(label + '_missing') from None require( stat.S_ISREG(details.st_mode) and not path.is_symlink() and 0 < details.st_size <= maximum, label + '_regular', ) return path def synthetic_openai_token(label): """Return a fixed, deliberately fake, detector-shaped high-entropy token.""" alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' def material(part): seed = hashlib.sha512( ('truf-packaged-worker-e2e:' + label + ':' + part).encode('ascii') ).hexdigest() return ''.join( alphabet[int(seed[index:index + 2], 16) % len(alphabet)] for index in range(0, len(seed), 2) )[:24] token = 'sk-proj-' + material('prefix') + 'T3BlbkFJ' + material('suffix') require( re.fullmatch(r'sk-proj-[A-Za-z0-9]{24}T3BlbkFJ[A-Za-z0-9]{24}', token), 'synthetic_token_shape', ) return token class CommandRunner: def __init__(self, root, run_root, distro, deadline): self.root = Path(root) self.run_root = Path(run_root) self.deadline = deadline self.wsl = shutil.which('wsl.exe') or shutil.which('wsl') require(self.wsl, 'wsl_unavailable') self.docker_prefix = [ self.wsl, '-d', distro, '--', 'sudo', '-n', 'docker', ] allowed = ( 'PATH', 'PATHEXT', 'SystemRoot', 'SYSTEMROOT', 'WINDIR', 'COMSPEC', 'TEMP', 'TMP', 'USERPROFILE', 'WSLENV', ) self.host_env = { name: os.environ[name] for name in allowed if name in os.environ } def execute(self, label, command, *, timeout=60, check=True, env=None, cwd=None): remaining = self.deadline - time.monotonic() require(remaining > 0, 'aggregate_timeout') timeout = max(0.1, min(float(timeout), remaining)) process = None try: process = subprocess.Popen( command, cwd=os.fspath(cwd or self.root), env=env or self.host_env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0), ) try: stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() process.communicate() raise Failure(label + '_timeout') from None except Failure: raise except OSError: raise Failure(label + '_unavailable') from None require( len(stdout) <= MAX_OUTPUT and len(stderr) <= MAX_OUTPUT, label + '_output_bound', ) if check: if process.returncode != 0: error = Failure(label + '_failed') error.exit_code = process.returncode raise error return process.returncode, stdout, stderr def docker(self, label, arguments, *, timeout=60, check=True): return self.execute( label, [*self.docker_prefix, *arguments], timeout=timeout, check=check, ) def docker_json(self, label, arguments, *, timeout=60): _, stdout, _ = self.docker(label, arguments, timeout=timeout) return load_json_bytes(stdout, label) def wsl_path(self, path): _, stdout, _ = self.execute( 'wsl_path', [self.wsl, '-d', self.docker_prefix[2], '--', 'wslpath', '-a', '-u', os.fspath(path).replace('\\', '/')], timeout=30, ) try: value = stdout.decode('utf-8', errors='strict').strip() except UnicodeError: raise Failure('wsl_path_encoding') from None require(value.startswith('/') and '\x00' not in value, 'wsl_path_invalid') return value class ResourceTracker: def __init__(self, runner, run_id): self.runner = runner self.run_id = run_id self.resources = {'containers': {}, 'volumes': {}, 'networks': {}} self.foreign_baseline = None def labels(self, phase, kind): return [ '--label', f'{RUN_LABEL}={self.run_id}', '--label', f'{PHASE_LABEL}={phase}', '--label', f'{KIND_LABEL}={kind}', ] @staticmethod def singular(kind): return {'containers': 'container', 'volumes': 'volume', 'networks': 'network'}[kind] def list_names(self, kind, *, name=None, owned=False): singular = self.singular(kind) arguments = [singular, 'ls'] if kind == 'containers': arguments.append('--all') if name is not None: pattern = '^/' + name + '$' if kind == 'containers' else '^' + name + '$' arguments.extend(['--filter', 'name=' + pattern]) if owned: arguments.extend(['--filter', f'label={RUN_LABEL}={self.run_id}']) field = 'Names' if kind == 'containers' else 'Name' arguments.append('--format={{.' + field + '}}') _, stdout, _ = self.runner.docker('list_' + kind, arguments) try: names = {line for line in stdout.decode('ascii').splitlines() if line} except UnicodeError: raise Failure('resource_name_encoding') from None require(len(names) <= 32, 'resource_inventory_bound') return names def guard_new(self, kind, name): require(RESOURCE_NAME.fullmatch(name), 'resource_name_guard') require(not self.list_names(kind, name=name), 'resource_name_collision') def inspect_owned(self, kind, name, expected=None): singular = self.singular(kind) if kind == 'containers': template = ( '{"id":{{json .Id}},"name":{{json .Name}},' '"run":{{json (index .Config.Labels "' + RUN_LABEL + '")}},' '"phase":{{json (index .Config.Labels "' + PHASE_LABEL + '")}},' '"kind":{{json (index .Config.Labels "' + KIND_LABEL + '")}},' '"running":{{json .State.Running}},"paused":{{json .State.Paused}},' '"restarting":{{json .State.Restarting}}}' ) else: template = ( '{"id":{{json ' + ('.Id' if kind == 'networks' else '""') + '}},' '"name":{{json .Name}},"run":{{json (index .Labels "' + RUN_LABEL + '")}},' '"phase":{{json (index .Labels "' + PHASE_LABEL + '")}},' '"kind":{{json (index .Labels "' + KIND_LABEL + '")}}}' ) value = self.runner.docker_json( 'inspect_owned_' + singular, [singular, 'inspect', '--format', template, name], ) require( value.get('name') == ('/' + name if kind == 'containers' else name) and value.get('run') == self.run_id and isinstance(value.get('phase'), str) and isinstance(value.get('kind'), str) and (kind == 'volumes' or HEX_64.fullmatch(str(value.get('id') or ''))), 'owned_' + singular + '_identity_guard', ) if expected is not None: require( (expected.get('id') is None or value.get('id') == expected['id']) and value['phase'] == expected['phase'] and value['kind'] == expected['kind'], 'owned_' + singular + '_identity_changed', ) return value def metadata_names(self, kind): arguments = ([kind, 'ls', '--all', '--no-trunc', '--format={{.ID}}'] if kind == 'container' else [kind, 'ls', '--format={{.Name}}']) _, stdout, _ = self.runner.docker('foreign_' + kind + '_list', arguments) try: values = {line for line in stdout.decode('ascii').splitlines() if line} except UnicodeError: raise Failure('foreign_' + kind + '_inventory_encoding') from None require(len(values) <= MAX_DOCKER_RESOURCES, 'foreign_' + kind + '_inventory_bound') return values def container_metadata(self, identifier): require(HEX_64.fullmatch(identifier), 'foreign_container_id_guard') value = self.runner.docker_json( 'foreign_container_inspect', ['container', 'inspect', '--format', CONTAINER_METADATA_FORMAT, identifier], ) require( value.get('id') == identifier and isinstance(value.get('name'), str) and HEX_64.fullmatch(str(value.get('image') or '').removeprefix('sha256:')) and value.get('status') in ( 'created', 'running', 'paused', 'restarting', 'removing', 'exited', 'dead', ) and all(type(value.get(key)) is bool for key in ( 'running', 'paused', 'restarting', 'dead', )) and isinstance(value.get('mounts'), list) and len(value['mounts']) <= 128, 'foreign_container_metadata_guard', ) mounts = [{ key: mount.get(key) for key in ( 'Type', 'Name', 'Source', 'Destination', 'Driver', 'Mode', 'RW', 'Propagation', ) } for mount in value['mounts']] return { 'id': value['id'], 'name': value['name'], 'image': value['image'], 'status': value['status'], 'running': value['running'], 'paused': value['paused'], 'restarting': value['restarting'], 'dead': value['dead'], 'mounts_sha256': sha256_bytes(json.dumps( mounts, ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii')), } def volume_metadata(self, name): value = self.runner.docker_json( 'foreign_volume_inspect', ['volume', 'inspect', '--format', VOLUME_METADATA_FORMAT, name], ) require( value.get('name') == name and isinstance(value.get('driver'), str) and isinstance(value.get('scope'), str) and isinstance(value.get('mountpoint'), str) and (value.get('created') is None or isinstance(value['created'], str)) and (value.get('labels') is None or isinstance(value['labels'], dict)) and (value.get('options') is None or isinstance(value['options'], dict)), 'foreign_volume_metadata_guard', ) return { 'name': name, 'driver': value['driver'], 'scope': value['scope'], 'created': value['created'], 'mountpoint_sha256': sha256_bytes(value['mountpoint'].encode('utf-8')), 'labels_sha256': sha256_bytes(json.dumps( value['labels'], ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii')), 'options_sha256': sha256_bytes(json.dumps( value['options'], ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii')), } def metadata_snapshot(self, *, exclude_owned=False): containers = {} owned_ids = { value['id']: (name, value) for name, value in self.resources['containers'].items() if value.get('id') } for identifier in sorted(self.metadata_names('container')): if exclude_owned and identifier in owned_ids: name, expected = owned_ids[identifier] self.inspect_owned('containers', name, expected) continue containers[identifier] = self.container_metadata(identifier) volumes = {} for name in sorted(self.metadata_names('volume')): value = self.volume_metadata(name) expected = self.resources['volumes'].get(name) if exclude_owned and expected is not None and expected.get('metadata') == value: continue volumes[name] = value snapshot = {'containers': containers, 'volumes': volumes} require(len(json.dumps(snapshot, ensure_ascii=True, sort_keys=True)) <= MAX_SNAPSHOT_BYTES, 'foreign_metadata_snapshot_bound') return snapshot def snapshot_foreign(self): require(self.foreign_baseline is None, 'foreign_snapshot_already_taken') self.foreign_baseline = self.metadata_snapshot() def assert_foreign_unchanged(self): require(self.foreign_baseline is not None, 'foreign_snapshot_missing') require(self.metadata_snapshot(exclude_owned=True) == self.foreign_baseline, 'foreign_docker_state_changed') def create_volume(self, name, phase, role): self.guard_new('volumes', name) self.resources['volumes'][name] = {'id': None, 'phase': phase, 'kind': role} _, stdout, _ = self.runner.docker( 'create_volume', ['volume', 'create', *self.labels(phase, role), name], ) require(stdout.decode('ascii', errors='ignore').strip() == name, 'volume_create_result') self.inspect_owned('volumes', name, self.resources['volumes'][name]) self.resources['volumes'][name]['metadata'] = self.volume_metadata(name) def create_network(self, name, phase): self.guard_new('networks', name) self.resources['networks'][name] = { 'id': None, 'phase': phase, 'kind': 'network', } _, stdout, _ = self.runner.docker( 'create_network', ['network', 'create', '--driver', 'bridge', '--internal', '--subnet', isolated_test_subnet(self.run_id, phase), *self.labels(phase, 'network'), name], ) identifier = stdout.decode('ascii', errors='strict').strip() require(HEX_64.fullmatch(identifier), 'network_create_result') self.resources['networks'][name]['id'] = identifier self.inspect_owned('networks', name, self.resources['networks'][name]) def planned_container(self, name, phase, role): self.resources['containers'][name] = {'id': None, 'phase': phase, 'kind': role} def created_container(self, name, identifier): self.resources['containers'][name]['id'] = identifier self.inspect_owned('containers', name, self.resources['containers'][name]) def inventory(self): inventory = {} for kind in self.resources: try: inventory[kind] = sorted(self.list_names(kind, owned=True)) except Exception: inventory[kind] = sorted(self.resources[kind]) return inventory def guarded_stop(self): stopped = 0 for name, expected in reversed(self.resources['containers'].items()): try: value = self.inspect_owned('containers', name, expected) if expected['id'] is None: expected['id'] = value['id'] if value['running'] or value['paused'] or value['restarting']: value = self.inspect_owned('containers', name, expected) code, _, _ = self.runner.docker( 'guarded_stop_owned_container', ['container', 'stop', '--time', '30', value['id']], timeout=45, check=False, ) stopped += int(code == 0) except (Exception, KeyboardInterrupt): continue return stopped def cleanup(self): inventory = { kind: self.list_names(kind, owned=True) for kind in self.resources } for kind, expected in self.resources.items(): require(inventory[kind] == set(expected), 'cleanup_ownership_guard') for name, expected in reversed(self.resources['containers'].items()): value = self.inspect_owned('containers', name, expected) if value['running'] or value['paused'] or value['restarting']: value = self.inspect_owned('containers', name, expected) self.runner.docker( 'stop_owned_container', ['container', 'stop', '--time', '30', value['id']], timeout=45, ) value = self.inspect_owned('containers', name, expected) self.runner.docker( 'remove_owned_container', ['container', 'rm', value['id']], timeout=45, ) for name, expected in reversed(self.resources['volumes'].items()): self.inspect_owned('volumes', name, expected) self.runner.docker('remove_owned_volume', ['volume', 'rm', name]) for name, expected in reversed(self.resources['networks'].items()): value = self.inspect_owned('networks', name, expected) self.runner.docker('remove_owned_network', ['network', 'rm', value['id']]) require( all(not self.list_names(kind, owned=True) for kind in self.resources), 'cleanup_incomplete', ) self.assert_foreign_unchanged() class WindowsClient: def __init__(self, command, environment, log_path): self.command = command self.environment = environment self.log_path = Path(log_path) self.process = None self.reader = None self.output = bytearray() self.output_lock = threading.Lock() self.output_exceeded = False def _read_output(self, process): for block in iter(lambda: process.stdout.read(64 * 1024), b''): with self.output_lock: if len(self.output) + len(block) <= MAX_OUTPUT: self.output.extend(block) else: self.output_exceeded = True def read_log(self): with self.output_lock: require(not self.output_exceeded, 'windows_log_bound') return bytes(self.output) def start(self): require(self.process is None, 'windows_client_already_started') try: self.process = subprocess.Popen( self.command, cwd=os.fspath(self.log_path.parent), env=self.environment, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags=( getattr(subprocess, 'CREATE_NEW_PROCESS_GROUP', 0) | getattr(subprocess, 'CREATE_NO_WINDOW', 0) ), ) except OSError: raise Failure('windows_client_unavailable') from None self.reader = threading.Thread(target=self._read_output, args=(self.process,), daemon=True) self.reader.start() def alive(self): return self.process is not None and self.process.poll() is None def stop(self): if self.process is None: return if self.process.poll() is None: taskkill = shutil.which('taskkill.exe') or 'taskkill.exe' subprocess.run( [taskkill, '/PID', str(self.process.pid), '/T', '/F'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0), ) try: self.process.wait(timeout=15) except subprocess.TimeoutExpired: self.process.kill() self.process.wait(timeout=5) if self.reader is not None: self.reader.join(timeout=5) if self.process.stdout is not None: self.process.stdout.close() if self.reader is not None: self.reader.join(timeout=1) self.process = None self.reader = None class LoopbackProxy: def __init__(self, runner, upstream, log_path): self.runner = runner self.upstream = upstream self.log_path = Path(log_path) self.process = None self.port = None self.output = None def start(self): require(self.process is None, 'loopback_proxy_already_started') with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation: reservation.bind(('127.0.0.1', 0)) self.port = int(reservation.getsockname()[1]) try: self.output = self.log_path.open('wb') self.process = subprocess.Popen( [self.runner.wsl, '-d', self.runner.docker_prefix[2], '--', 'python3', '-u', '-c', LOOPBACK_PROXY_SCRIPT, str(self.port), self.upstream, '8443'], cwd=os.fspath(self.runner.run_root), env=self.runner.host_env, stdin=subprocess.DEVNULL, stdout=self.output, stderr=subprocess.STDOUT, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0), ) except OSError: if self.output is not None: self.output.close() self.output = None raise Failure('loopback_proxy_unavailable') from None end = min(self.runner.deadline, time.monotonic() + 60) context = ssl.create_default_context( cafile=os.fspath(self.runner.root / 'tests' / 'fixtures' / 'worker_tls_cert.pem'), ) while time.monotonic() < end: require(self.process.poll() is None, 'loopback_proxy_exited') try: with socket.create_connection( ('127.0.0.1', self.port), timeout=0.5, ) as connection, context.wrap_socket( connection, server_hostname='localhost', ): return self.port except OSError: time.sleep(0.1) try: detail = self.log_path.read_text(encoding='utf-8', errors='replace') except OSError: detail = '' if 'ConnectionRefusedError' in detail: raise Failure('loopback_proxy_upstream_refused') if 'TimeoutError' in detail: raise Failure('loopback_proxy_upstream_timeout') if 'proxy upstream connected' in detail: raise Failure('loopback_proxy_tls_timeout') raise Failure('loopback_proxy_timeout') def stop(self): if self.process is not None and self.process.poll() is None: self.process.terminate() try: self.process.wait(timeout=10) except subprocess.TimeoutExpired: self.process.kill() self.process.wait(timeout=5) self.process = None if self.output is not None: self.output.close() self.output = None class Verifier: def __init__(self, args): self.args = args self.script = Path(__file__).resolve(strict=True) self.root = self.script.parent.parent.resolve(strict=True) require(self.script == self.root / 'docker' / 'verify_packaged_workers.py', 'script_path') require('build/' in (self.root / '.gitignore').read_text(encoding='utf-8').splitlines(), 'build_not_gitignored') self.run_id = secrets.token_hex(8) self.prefix = 'truf-packaged-worker-e2e-' + self.run_id build = self.root / 'build' build.mkdir(exist_ok=True) self.run_root = build / ('pwe-' + self.run_id) self.run_root.mkdir() self.deadline = time.monotonic() + args.timeout_seconds self.runner = CommandRunner(self.root, self.run_root, args.wsl_distro, self.deadline) self.tracker = ResourceTracker(self.runner, self.run_id) self.windows_artifact = Path(args.windows_artifact) if not self.windows_artifact.is_absolute(): self.windows_artifact = self.root / self.windows_artifact self.windows_artifact = self.windows_artifact.resolve(strict=True) require(self.windows_artifact.is_relative_to(self.root), 'windows_artifact_outside_root') self.tokens = { TARGETS[0]: synthetic_openai_token('a'), TARGETS[1]: synthetic_openai_token('b'), } require(len(set(self.tokens.values())) == 2, 'synthetic_tokens_distinct') self.device_tokens = {} self.windows_client = None self.loopback_proxy = None self.image_details = {} self.repository_heads = {} self.phase_results = {} def image(self, reference, label, *, worker=False): value = self.runner.docker_json(label, ['image', 'inspect', reference]) require(isinstance(value, list) and len(value) == 1, label + '_shape') details = value[0] require( details.get('Os') == 'linux' and details.get('Architecture') == 'amd64' and HEX_64.fullmatch(str(details.get('Id') or '').removeprefix('sha256:')), label + '_platform', ) if worker: expected = [ '/usr/bin/tini', '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B', '/opt/truf-worker/app/remote_worker_bootstrap.py', '--', ] require(details.get('Config', {}).get('Entrypoint') == expected, 'linux_worker_entrypoint') require(details.get('Config', {}).get('Cmd') == ['run'], 'linux_worker_default_command') require(details.get('Config', {}).get('User') == '10001:10001', 'linux_worker_user') self.image_details[label] = details['Id'] def prepare_repositories(self): git = shutil.which('git.exe') or shutil.which('git') require(git, 'git_unavailable') repositories = self.run_root / 'repositories' repositories.mkdir() base_env = dict(self.runner.host_env) base_env.update({ 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': 'NUL', 'GIT_TERMINAL_PROMPT': '0', 'TZ': 'UTC', 'GIT_AUTHOR_NAME': 'TRUF Packaged Worker Fixture', 'GIT_AUTHOR_EMAIL': 'packaged-worker-fixture@invalid.example', 'GIT_COMMITTER_NAME': 'TRUF Packaged Worker Fixture', 'GIT_COMMITTER_EMAIL': 'packaged-worker-fixture@invalid.example', 'GIT_AUTHOR_DATE': '2001-01-01T00:00:00+00:00', 'GIT_COMMITTER_DATE': '2001-01-01T00:00:00+00:00', }) for index, target in enumerate(TARGETS): name = chr(ord('a') + index) path = repositories / name path.mkdir() write_bytes(path / 'synthetic.env', ('OPENAI_API_KEY=' + self.tokens[target] + '\n').encode('ascii')) self.runner.execute( 'git_init_' + name, [git, 'init', '--quiet', '--initial-branch=main', '--template=', os.fspath(path)], env=base_env, ) self.runner.execute( 'git_config_' + name, [git, '-C', os.fspath(path), 'config', 'core.autocrlf', 'false'], env=base_env, ) self.runner.execute( 'git_add_' + name, [git, '-C', os.fspath(path), 'add', '--', 'synthetic.env'], env=base_env, ) self.runner.execute( 'git_commit_' + name, [git, '-C', os.fspath(path), 'commit', '--quiet', '--no-gpg-sign', '--message', 'Add deterministic synthetic detector fixture'], env=base_env, ) _, stdout, _ = self.runner.execute( 'git_head_' + name, [git, '-C', os.fspath(path), 'rev-parse', 'HEAD'], env=base_env, ) head = stdout.decode('ascii', errors='strict').strip() require(HEX_40.fullmatch(head), 'fixture_commit_hash') _, status_output, _ = self.runner.execute( 'git_status_' + name, [git, '-C', os.fspath(path), 'status', '--porcelain=v1'], env=base_env, ) require(not status_output, 'fixture_repository_dirty') self.repository_heads[target] = head require(len(set(self.repository_heads.values())) == 2, 'fixture_commits_distinct') def prepare_fixtures(self): manifest_path = regular_file( self.windows_artifact / 'worker-package.json', 'windows_manifest', ) for relative, label in ( ('truf-worker.cmd', 'windows_cli_entrypoint'), ('run-worker.cmd', 'windows_entrypoint'), ('runtime/python/python.exe', 'windows_python'), ('bin/trufflehog.exe', 'windows_scanner'), ): regular_file(self.windows_artifact / Path(relative), label, maximum=256 * 1024 * 1024) require( b'remote_worker_bootstrap.py" -- %*' in (self.windows_artifact / 'truf-worker.cmd').read_bytes(), 'windows_cli_entrypoint_contents', ) require( b'remote_worker_bootstrap.py" -- run %*' in (self.windows_artifact / 'run-worker.cmd').read_bytes(), 'windows_compat_entrypoint_contents', ) windows_manifest = manifest_path.read_bytes() windows_value = load_json_bytes(windows_manifest, 'windows_manifest') expected_capabilities = { ('gitlab', 'gitlab', 'exact_git_v1'), ('dockerhub', 'docker', 'docker_direct_v1'), ('huggingface', 'huggingface', 'huggingface_space_v1'), } windows_capabilities = { ( item.get('source'), item.get('platform'), item.get('planning_kind'), ) for item in windows_value.get('capabilities', []) if isinstance(item, dict) } require( windows_value.get('schema') == 3 and windows_value.get('protocol_version') == 2 and windows_value.get('platform_tag') == 'windows-x86_64', 'windows_manifest_identity', ) require( windows_capabilities == expected_capabilities, 'windows_manifest_capabilities', ) helper_name = self.prefix + '-linux-manifest' self.create_container( helper_name, 'linux', 'manifest-reader', ['--network', 'none', '--entrypoint', '/bin/cat', self.args.linux_image, '/opt/truf-worker/worker-package.json'], ) _, linux_manifest, _ = self.runner.docker( 'linux_manifest', ['container', 'start', '--attach', helper_name], ) _, exit_code, _ = self.runner.docker( 'linux_manifest_exit', ['container', 'inspect', '--format={{.State.ExitCode}}', helper_name], ) require(exit_code.strip() == b'0', 'linux_manifest_reader_exit') linux_value = load_json_bytes(linux_manifest, 'linux_manifest') linux_capabilities = { ( item.get('source'), item.get('platform'), item.get('planning_kind'), ) for item in linux_value.get('capabilities', []) if isinstance(item, dict) } require( linux_value.get('schema') == 3 and linux_value.get('protocol_version') == 2 and linux_value.get('platform_tag') == 'linux-x86_64', 'linux_manifest_identity', ) require( linux_capabilities == expected_capabilities and linux_capabilities == windows_capabilities, 'linux_manifest_capabilities', ) fixture = self.run_root / 'fixture' fixture.mkdir() write_json(fixture / 'windows-manifest.json', windows_value) write_json(fixture / 'linux-manifest.json', linux_value) write_json(fixture / 'fixture.json', { 'schema': 1, 'targets': list(TARGETS), 'direct_targets': DIRECT_TARGETS, 'repositories': { target: {'head_sha': self.repository_heads[target]} for target in TARGETS }, }) certificate = regular_file( self.root / 'tests' / 'fixtures' / 'worker_tls_cert.pem', 'tls_certificate', ) private_key = regular_file( self.root / 'tests' / 'fixtures' / 'worker_tls_key.pem', 'tls_private_key', ) decoded = ssl._ssl._test_decode_cert(os.fspath(certificate)) require(('DNS', 'localhost') in decoded.get('subjectAltName', ()), 'tls_localhost_san') require(ssl.cert_time_to_seconds(decoded['notAfter']) > time.time(), 'tls_certificate_expired') write_bytes(fixture / 'worker_tls_cert.pem', certificate.read_bytes()) write_bytes(fixture / 'worker_tls_key.pem', private_key.read_bytes()) self.fixture = fixture self.manifest_hashes = { 'windows': sha256_bytes(windows_manifest), 'linux': sha256_bytes(linux_manifest), } def preflight(self): require(os.name == 'nt', 'windows_host_required') self.tracker.snapshot_foreign() self.image(self.args.linux_image, 'linux_image', worker=True) self.image(self.args.test_image, 'test_image') self.prepare_repositories() self.prepare_fixtures() def phase_names(self, phase): base = self.prefix + '-' + phase return { 'network': base + '-network', 'server_data': base + '-server-data', 'server_seed': base + '-server-seed', 'fixture': base + '-fixture-data', 'worker_data': base + '-worker-data', 'fixture_seed': base + '-fixture-seed', 'worker_seed': base + '-worker-seed', 'server': base + '-server', 'worker': base + '-worker', } def create_container(self, name, phase, role, arguments): self.tracker.guard_new('containers', name) self.tracker.planned_container(name, phase, role) _, stdout, _ = self.runner.docker( 'create_' + role, ['container', 'create', '--pull=never', '--name', name, *self.tracker.labels(phase, role), *arguments], ) identifier = stdout.decode('ascii', errors='ignore').strip() require(HEX_64.fullmatch(identifier), role + '_container_id') self.tracker.created_container(name, identifier) def seed_volume(self, phase, role, name, volume, source): self.create_container( name, phase, role, ['--network', 'none', '--user', '0:0', '--mount', f'type=volume,source={volume},target=/seed', '--entrypoint', '/usr/local/bin/python3', self.args.test_image, '-u', '-I', '-S', '-B', '/seed/_seed.py'], ) source_wsl = self.runner.wsl_path(Path(source).resolve(strict=True)) self.runner.docker( 'copy_' + role, ['container', 'cp', source_wsl + '/.', name + ':/seed/'], timeout=120, ) self.runner.docker('start_' + role, ['container', 'start', '--attach', name], timeout=120) _, stdout, _ = self.runner.docker( 'wait_' + role, ['container', 'inspect', '--format={{.State.ExitCode}}', name], ) require(stdout.strip() == b'0', role + '_exit') def prepare_phase_resources(self, phase): names = self.phase_names(phase) phase_root = self.run_root / phase phase_root.mkdir() if phase == 'windows': write_bytes( phase_root / 'publish-direct.py', PUBLISH_DIRECT_SCRIPT.encode('ascii'), ) fixture_seed = phase_root / 'fixture-seed' shutil.copytree(self.fixture, fixture_seed) write_bytes(fixture_seed / '_seed.py', SEED_SCRIPT.encode('ascii')) server_seed = phase_root / 'server-seed' server_seed.mkdir() write_bytes(server_seed / '_seed.py', SEED_SCRIPT.encode('ascii')) self.tracker.create_network(names['network'], phase) self.tracker.create_volume(names['server_data'], phase, 'server-data') self.seed_volume( phase, 'server-seed', names['server_seed'], names['server_data'], server_seed, ) self.tracker.create_volume(names['fixture'], phase, 'fixture-data') self.seed_volume( phase, 'fixture-seed', names['fixture_seed'], names['fixture'], fixture_seed, ) if phase == 'linux': worker_seed = phase_root / 'worker-seed' worker_seed.mkdir() shutil.copytree(self.run_root / 'repositories', worker_seed / 'repos') shutil.copyfile( self.fixture / 'worker_tls_cert.pem', worker_seed / 'worker_tls_cert.pem', ) write_bytes(worker_seed / '_seed.py', SEED_SCRIPT.encode('ascii')) write_bytes(worker_seed / '_inspect.py', INSPECT_WORKER_SCRIPT.encode('ascii')) write_bytes( worker_seed / '_publish_direct.py', PUBLISH_DIRECT_SCRIPT.encode('ascii'), ) self.tracker.create_volume(names['worker_data'], phase, 'worker-data') self.seed_volume( phase, 'worker-seed', names['worker_seed'], names['worker_data'], worker_seed, ) self.device_tokens[phase] = secrets.token_hex(32) self.start_server(phase, names) self.wait_control(phase, names['server'], 'prepared.json', { 'schema': 1, 'phase': phase, 'target_count': 2, }, 180) return names, phase_root def start_server(self, phase, names): self.create_container( names['server'], phase, 'server', ['--network', names['network'], '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--pids-limit', '512', '--stop-timeout', '30', '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=128m,mode=1777', '--mount', f'type=volume,source={names["server_data"]},target=/data', '--mount', f'type=volume,source={names["fixture"]},target=/fixture,readonly', '--env', 'TRUF_WORKER_E2E_TOKEN=' + self.device_tokens[phase], '--env', 'TRUF_WORKER_E2E_PHASE=' + phase, '--entrypoint', '/usr/bin/tini', self.args.test_image, '--', '/usr/local/bin/python3', '-u', '-I', '-S', '-B', '/opt/truf/tests/packaged_worker_e2e_server.py'], ) self.runner.docker('start_server', ['container', 'start', names['server']]) require(self.container_running(names['server']), 'server_not_running') def container_running(self, name): code, stdout, _ = self.runner.docker( 'container_running', ['container', 'inspect', '--format={{.State.Running}}', name], check=False, ) return code == 0 and stdout.strip() == b'true' def read_control(self, server, name): code, stdout, _ = self.runner.docker( 'read_control', ['container', 'exec', server, '/bin/cat', '/data/control/' + name], check=False, ) if code: return None value = load_json_bytes(stdout, 'control_' + name.replace('.', '_')) require(isinstance(value, dict), 'control_object') return value def wait_until(self, label, function, seconds, *, alive=None): end = min(self.deadline, time.monotonic() + seconds) while time.monotonic() < end: value = function() if value is not None and value is not False: return value if alive is not None: require(alive(), label + '_process_exited') time.sleep(0.25) raise Failure(label + '_timeout') def wait_control(self, phase, server, name, expected, seconds, *, alive=None): def marker(): value = self.read_control(server, name) if value is None: return None require(value == expected, phase + '_' + name.replace('.', '_') + '_content') return value return self.wait_until( phase + '_' + name.replace('.', '_'), marker, seconds, alive=lambda: self.container_running(server) and (alive is None or alive()), ) @staticmethod def git_environment(local_repositories, certificate, *, linux=False): environment = { 'SSL_CERT_FILE': os.fspath(certificate), 'GIT_ALLOW_PROTOCOL': 'file', 'GIT_TERMINAL_PROMPT': '0', 'GIT_LFS_SKIP_SMUDGE': '1', 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_GLOBAL': '/dev/null' if linux else 'NUL', 'GIT_CONFIG_COUNT': '2', } for index, target in enumerate(TARGETS): local = local_repositories[index] uri = local if linux else Path(local).resolve(strict=True).as_uri() environment[f'GIT_CONFIG_KEY_{index}'] = f'url.{uri}.insteadOf' environment[f'GIT_CONFIG_VALUE_{index}'] = target return environment def windows_snapshot(self, phase_root): state_root = self.run_root / 'w' / 'TRUF' / 'RemoteWorker' bundle_root = state_root / 'bundles' work_root = state_root / 'work' states = {} if state_root.exists(): for path in sorted(state_root.glob('slot-*.json')): regular_file(path, 'windows_state', maximum=64 * 1024 * 1024) value = load_json_bytes(path.read_bytes(), 'windows_state') assignment = dict(value.get('assignment') or {}) reservation = dict(assignment.get('reservation') or {}) snapshot = dict(assignment.get('execution_snapshot') or {}) planning = dict(snapshot.get('planning') or {}) details = path.stat(follow_symlinks=False) states[path.name] = { 'phase': value.get('phase'), 'reservation_id': reservation.get('reservation_id'), 'bundle_id': reservation.get('bundle_id'), 'scan_event_id': reservation.get('scan_event_id'), 'source': reservation.get('source'), 'platform': reservation.get('platform'), 'target': reservation.get('target'), 'planning_kind': planning.get('kind'), 'sha256': sha256_file(path), 'size': details.st_size, 'mtime_ns': details.st_mtime_ns, } bundles = {} if bundle_root.exists(): for path in sorted(bundle_root.rglob('*.trb')): regular_file(path, 'windows_bundle', maximum=64 * 1024 * 1024) details = path.stat(follow_symlinks=False) bundles[path.relative_to(bundle_root).as_posix()] = { 'sha256': sha256_file(path), 'size': details.st_size, 'mtime_ns': details.st_mtime_ns, } snapshot = {'states': states, 'bundles': bundles} if not self.durable(snapshot): return snapshot snapshot['work'] = self.host_tree_identity(work_root) return snapshot @staticmethod def host_tree_identity(root): digest = hashlib.sha256() count = 0 active_entries = 0 root = Path(root) if not root.exists(): return {'count': 0, 'active_entries': 0, 'sha256': digest.hexdigest()} for current, directories, files in os.walk(root, followlinks=False): directories.sort() files.sort() for name in directories: path = Path(current) / name require(not path.is_symlink() and path.is_dir(), 'windows_work_directory') if path.relative_to(root).parts[0] != 'abandoned': active_entries += 1 for name in files: path = regular_file( Path(current) / name, 'windows_work_file', maximum=256 * 1024 * 1024, ) if path.relative_to(root).parts[0] != 'abandoned': active_entries += 1 details = path.stat(follow_symlinks=False) digest.update(path.relative_to(root).as_posix().encode('utf-8') + b'\0') digest.update(str(details.st_size).encode('ascii') + b'\0') digest.update(str(details.st_mtime_ns).encode('ascii') + b'\0') digest.update(sha256_file(path).encode('ascii') + b'\n') count += 1 return { 'count': count, 'active_entries': active_entries, 'sha256': digest.hexdigest(), } def linux_snapshot(self, worker): code, stdout, _ = self.runner.docker( 'linux_worker_snapshot', ['container', 'exec', worker, '/usr/local/bin/python3', '-u', '-I', '-S', '-B', '/data/_inspect.py'], check=False, ) if code: return None value = load_json_bytes(stdout, 'linux_worker_snapshot') require(isinstance(value, dict), 'linux_snapshot_object') return value @staticmethod def durable(snapshot): if not isinstance(snapshot, dict): return False states = snapshot.get('states') bundles = snapshot.get('bundles') if set(states or {}) != {'slot-0.json', 'slot-1.json'} or len(bundles or {}) != 2: return False bundle_ids = set() for state in states.values(): if ( state.get('phase') != 'bundle_ready' or not HEX_64.fullmatch(str(state.get('sha256') or '')) or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('bundle_id') or '')) or int(state.get('reservation_id') or 0) <= 0 ): return False bundle_ids.add(state['bundle_id']) if len(bundle_ids) != 2: return False for path, entry in bundles.items(): if ( Path(path).name.removesuffix('.trb') not in bundle_ids or not HEX_64.fullmatch(str(entry.get('sha256') or '')) or int(entry.get('size') or 0) <= 0 ): return False return True @staticmethod def direct_assigned(snapshot, bundle_count): if not isinstance(snapshot, dict): return False states = snapshot.get('states') bundles = snapshot.get('bundles') if ( set(states or {}) != {'slot-0.json', 'slot-1.json'} or len(bundles or {}) != bundle_count ): return False expected = { ('dockerhub', 'docker', 'docker_direct_v1', DOCKER_TARGET), ('huggingface', 'huggingface', 'huggingface_space_v1', HUGGINGFACE_TARGET), } identities = set() bundle_ids = set() for state in states.values(): if ( state.get('phase') != 'assigned' or not HEX_64.fullmatch(str(state.get('sha256') or '')) or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('bundle_id') or '')) or int(state.get('reservation_id') or 0) <= 0 or not re.fullmatch(r'[a-f0-9]{32,64}', str(state.get('scan_event_id') or '')) ): return False identities.add(( state.get('source'), state.get('platform'), state.get('planning_kind'), state.get('target'), )) bundle_ids.add(state['bundle_id']) if identities != expected or len(bundle_ids) != 2: return False for path, entry in bundles.items(): if ( Path(path).name.removesuffix('.trb') not in bundle_ids or not HEX_64.fullmatch(str(entry.get('sha256') or '')) or int(entry.get('size') or 0) <= 0 ): return False work = snapshot.get('work') if work is not None and int(work.get('active_entries') or 0) != 0: return False return True @staticmethod def resolved_client_storage(snapshot): if not isinstance(snapshot, dict) or snapshot.get('bundles'): return False work = snapshot.get('work') or {} if int(work.get('active_entries') or 0) != 0: return False for state in (snapshot.get('states') or {}).values(): if ( state.get('phase') != 'claiming' or state.get('reservation_id') is not None or state.get('bundle_id') is not None or state.get('scan_event_id') is not None ): return False return True def assert_logs_safe(self, logs): forbidden = [*self.tokens.values(), *self.device_tokens.values()] for value in logs: require(len(value) <= MAX_OUTPUT, 'log_output_bound') for secret in forbidden: require(secret.encode('ascii') not in value, 'secret_present_in_log') def docker_logs(self, container): _, stdout, stderr = self.runner.docker( 'docker_logs', ['container', 'logs', container], check=False, ) return stdout + (b'\n' if stdout and stderr else b'') + stderr def assert_restart_stable(self, phase, snapshot, expected, alive): started = time.monotonic() end = min(self.deadline, time.monotonic() + 3) # The expected snapshot was captured before the real process restart. checks = 1 current = None while time.monotonic() < end: require(alive(), phase + '_restart_process_exited') current = snapshot() require( isinstance(current, dict) and current.get('states') == expected.get('states') and current.get('bundles') == expected.get('bundles') and int((current.get('work') or {}).get('count') or 0) <= int((expected.get('work') or {}).get('count') or 0), phase + '_restart_rescanned_or_rewrote_bundle', ) require(alive(), phase + '_restart_process_exited') checks += 1 time.sleep(0.25) require( checks >= 2 and time.monotonic() - started >= 2.5, phase + '_restart_stability_window', ) return current def restore_and_wait_direct(self, phase, names, snapshot, alive): self.runner.docker( phase + '_restore', ['container', 'exec', names['server'], '/usr/bin/touch', '/data/control/restore'], ) return self.wait_until( phase + '_direct_claims', lambda: ( value if self.direct_assigned(value := snapshot(), 0) else None ), 240, alive=lambda: self.container_running(names['server']) and alive(), ) def publish_windows_direct(self, phase_root): state_root = self.run_root / 'w' / 'TRUF' / 'RemoteWorker' environment = dict(self.runner.host_env) environment.update({ 'LOCALAPPDATA': os.fspath(phase_root / 'localappdata'), 'TEMP': os.fspath(phase_root / 'temp'), 'TMP': os.fspath(phase_root / 'temp'), }) _, stdout, stderr = self.runner.execute( 'windows_publish_direct', [ os.fspath(self.windows_artifact / 'runtime' / 'python' / 'python.exe'), '-I', '-S', '-B', os.fspath(phase_root / 'publish-direct.py'), os.fspath(self.windows_artifact), os.fspath(state_root), os.fspath(state_root / 'bundles'), ], timeout=120, env=environment, ) require( stdout.strip() == b'published direct bundles: 2' and not stderr, 'windows_direct_publish_output', ) def publish_linux_direct(self, worker): _, stdout, stderr = self.runner.docker( 'linux_publish_direct', [ 'container', 'exec', worker, '/usr/local/bin/python3', '-u', '-I', '-S', '-B', '/data/_publish_direct.py', '/opt/truf-worker', '/data/state-base/truf/remote-worker', '/data/client/truf/remote-worker/bundles', ], timeout=120, ) require( stdout.strip() == b'published direct bundles: 2' and not stderr, 'linux_direct_publish_output', ) def start_windows_client(self, phase_root, port): localappdata = self.run_root / 'w' temporary = phase_root / 'temp' localappdata.mkdir() temporary.mkdir() system_keys = ( 'PATH', 'PATHEXT', 'SystemRoot', 'SYSTEMROOT', 'WINDIR', 'COMSPEC', 'NUMBER_OF_PROCESSORS', 'PROCESSOR_ARCHITECTURE', ) environment = { key: os.environ[key] for key in system_keys if key in os.environ } environment.update({ 'LOCALAPPDATA': os.fspath(localappdata), 'TEMP': os.fspath(temporary), 'TMP': os.fspath(temporary), **self.git_environment( [self.run_root / 'repositories' / 'a', self.run_root / 'repositories' / 'b'], self.fixture / 'worker_tls_cert.pem', ), }) command = [ environment.get('COMSPEC') or 'cmd.exe', '/d', '/c', os.fspath(self.windows_artifact / 'run-worker.cmd'), '--server', f'https://localhost:{port}', '--token', self.device_tokens['windows'], '--parallelism', str(PARALLELISM), ] self.windows_client = WindowsClient(command, environment, phase_root / 'client.log') self.windows_client.start() def windows_phase(self): names, phase_root = self.prepare_phase_resources('windows') _, stdout, _ = self.runner.docker( 'windows_server_address', ['container', 'inspect', '--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}', names['server']], ) address = stdout.decode('ascii', errors='strict').strip() require(re.fullmatch(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', address), 'windows_server_address') self.loopback_proxy = LoopbackProxy( self.runner, address, phase_root / 'loopback-proxy.log', ) port = self.loopback_proxy.start() self.start_windows_client(phase_root, port) try: self.wait_control('windows', names['server'], 'outage.json', { 'schema': 1, 'claim_count': 2, 'phase': 'windows', }, 240, alive=self.windows_client.alive) before = self.wait_until( 'windows_durable_bundle', lambda: (snapshot if self.durable(snapshot := self.windows_snapshot(phase_root)) else None), 240, alive=self.windows_client.alive, ) self.windows_client.stop() self.windows_client.start() self.assert_restart_stable( 'windows', lambda: self.windows_snapshot(phase_root), before, self.windows_client.alive, ) snapshot = lambda: self.windows_snapshot(phase_root) self.restore_and_wait_direct( 'windows', names, snapshot, self.windows_client.alive, ) self.publish_windows_direct(phase_root) direct = self.wait_until( 'windows_direct_bundles', lambda: ( value if self.direct_assigned(value := snapshot(), 2) else None ), 120, alive=self.windows_client.alive, ) self.windows_client.stop() self.windows_client.start() self.assert_restart_stable( 'windows', snapshot, direct, self.windows_client.alive, ) self.complete_after_direct_ready('windows', names, snapshot) return self.finish_phase('windows', names, phase_root, (before, direct)) finally: if self.windows_client is not None: self.windows_client.stop() if self.loopback_proxy is not None: self.loopback_proxy.stop() def create_linux_worker(self, names): environment = self.git_environment( ['file:///data/repos/a', 'file:///data/repos/b'], '/data/worker_tls_cert.pem', linux=True, ) arguments = [ '--network', 'container:' + names['server'], '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--pids-limit', '256', '--stop-timeout', '45', '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=128m,mode=1777', '--mount', f'type=volume,source={names["worker_data"]},target=/data', ] for key, value in environment.items(): arguments.extend(['--env', key + '=' + value]) arguments.extend([ '--env', 'XDG_DATA_HOME=/data/client', '--env', 'XDG_STATE_HOME=/data/state-base', self.args.linux_image, '--server', 'https://localhost:8443', '--token', self.device_tokens['linux'], '--parallelism', str(PARALLELISM), ]) self.create_container(names['worker'], 'linux', 'worker', arguments) self.runner.docker('start_linux_worker', ['container', 'start', names['worker']]) require(self.container_running(names['worker']), 'linux_worker_not_running') def linux_phase(self): names, phase_root = self.prepare_phase_resources('linux') self.create_linux_worker(names) self.wait_control('linux', names['server'], 'outage.json', { 'schema': 1, 'claim_count': 2, 'phase': 'linux', }, 240, alive=lambda: self.container_running(names['worker'])) before = self.wait_until( 'linux_durable_bundle', lambda: (snapshot if self.durable(snapshot := self.linux_snapshot(names['worker'])) else None), 240, alive=lambda: self.container_running(names['worker']), ) worker = self.tracker.inspect_owned( 'containers', names['worker'], self.tracker.resources['containers'][names['worker']], ) self.runner.docker( 'stop_linux_worker', ['container', 'stop', '--time', '45', worker['id']], timeout=60, ) require(not self.container_running(names['worker']), 'linux_worker_stop') self.runner.docker('restart_linux_worker', ['container', 'start', names['worker']]) self.assert_restart_stable( 'linux', lambda: self.linux_snapshot(names['worker']), before, lambda: self.container_running(names['worker']), ) snapshot = lambda: self.linux_snapshot(names['worker']) alive = lambda: self.container_running(names['worker']) self.restore_and_wait_direct('linux', names, snapshot, alive) self.publish_linux_direct(names['worker']) direct = self.wait_until( 'linux_direct_bundles', lambda: ( value if self.direct_assigned(value := snapshot(), 2) else None ), 120, alive=alive, ) worker = self.tracker.inspect_owned( 'containers', names['worker'], self.tracker.resources['containers'][names['worker']], ) self.runner.docker( 'stop_linux_direct_worker', ['container', 'stop', '--time', '45', worker['id']], timeout=60, ) require(not self.container_running(names['worker']), 'linux_direct_worker_stop') self.runner.docker('restart_linux_direct_worker', ['container', 'start', names['worker']]) self.assert_restart_stable('linux_direct', snapshot, direct, alive) self.complete_after_direct_ready('linux', names, snapshot) return self.finish_phase('linux', names, phase_root, (before, direct)) def complete_after_direct_ready(self, phase, names, snapshot): self.runner.docker( phase + '_direct_ready', ['container', 'exec', names['server'], '/usr/bin/touch', '/data/control/direct-ready'], ) def completed(): return self.read_control(names['server'], 'completed.json') try: evidence = self.wait_until( phase + '_completed', completed, 240, alive=lambda: self.container_running(names['server']), ) except Failure: waiting = self.read_control(names['server'], 'completion-wait.json') if waiting is not None: require( set(waiting) == {'schema', 'phase', 'reason'} and waiting['schema'] == 1 and waiting['phase'] == phase and isinstance(waiting['reason'], str) and re.fullmatch(r'[A-Za-z0-9 _-]{1,80}', waiting['reason']), phase + '_completion_wait_shape', ) print(phase + ' completion wait: ' + waiting['reason'], flush=True) raise def clean(): value = snapshot() if self.resolved_client_storage(value): return value return None self.wait_until(phase + '_client_cleanup', clean, 120) self.phase_results[phase] = evidence def finish_phase(self, phase, names, phase_root, restart_snapshots): if phase == 'linux': worker = self.tracker.inspect_owned( 'containers', names['worker'], self.tracker.resources['containers'][names['worker']], ) self.runner.docker( 'final_stop_linux_worker', ['container', 'stop', '--time', '45', worker['id']], timeout=60, ) final, stopped_reader = self.linux_snapshot_stopped( names['worker'], names['worker_data'], phase, ) else: self.windows_client.stop() final = self.windows_snapshot(phase_root) require(self.resolved_client_storage(final), phase + '_final_cleanup') if phase == 'linux': receipt = final.get('shutdown_receipt') require( isinstance(receipt, dict) and set(receipt) == { 'schema', 'instance_id', 'completed_at', 'exit_code', 'drained', 'last_sequence', } and receipt.get('schema') == 1 and receipt.get('drained') is True and receipt.get('exit_code') == 0, 'linux_clean_shutdown_receipt', ) logs = [] for role in ('server_seed', 'fixture_seed', 'server'): logs.append(self.docker_logs(names[role])) if phase == 'linux': logs.extend([ self.docker_logs(names['worker_seed']), self.docker_logs(names['worker']), self.docker_logs(stopped_reader), ]) else: logs.append(self.windows_client.read_log()) self.assert_logs_safe(logs) return { 'bundle_sha256': sorted( entry['sha256'] for snapshot in restart_snapshots for entry in snapshot['bundles'].values() ), 'evidence': self.normalize_evidence(phase, self.phase_results[phase]), } def linux_snapshot_stopped(self, worker, volume, phase): reader = self.prefix + '-' + phase + '-stopped-reader' self.create_container( reader, phase, 'stopped-reader', ['--network', 'none', '--read-only', '--mount', f'type=volume,source={volume},target=/data', '--entrypoint', '/usr/local/bin/python3', self.args.linux_image, '-u', '-I', '-S', '-B', '/data/_inspect.py'], ) _, stdout, _ = self.runner.docker( 'run_stopped_reader', ['container', 'start', '--attach', reader], timeout=60, ) value = load_json_bytes(stdout, 'stopped_reader') require(isinstance(value, dict), 'stopped_reader_object') return value, reader def normalize_evidence(self, phase, value): expected_keys = { 'schema', 'phase', 'counts', 'claim_count', 'detectors', 'candidate_services', 'secret_hashes', 'commit_hashes', 'receipt_count', 'source_counts', 'planning_counts', 'capacity', } require(isinstance(value, dict) and set(value) == expected_keys, phase + '_evidence_shape') require( value['schema'] == 1 and value['phase'] == phase and value['claim_count'] == value['receipt_count'] == 4 and value['detectors'] == ['OpenAI'] and value['candidate_services'] == ['openai'] and value['source_counts'] == { 'gitlab': 2, 'dockerhub': 1, 'huggingface': 1, } and value['planning_counts'] == { 'exact_git_v1': 2, 'docker_direct_v1': 1, 'huggingface_space_v1': 1, }, phase + '_evidence_identity', ) expected_secret_hashes = sorted( sha256_bytes(token.encode('ascii')) for token in self.tokens.values() ) require(value['secret_hashes'] == expected_secret_hashes, phase + '_secret_hashes') require( value['commit_hashes'] == sorted(self.repository_heads.values()), phase + '_commit_hashes', ) normalized = dict(value) normalized.pop('phase') return normalized def clear_run_artifacts(self): require( self.run_root.parent == self.root / 'build' and re.fullmatch(r'pwe-[a-f0-9]{16}', self.run_root.name) and self.run_root.is_dir() and not self.run_root.is_symlink(), 'run_root_guard', ) def retry_writable(function, name, _error): details = os.lstat(name) if stat.S_ISLNK(details.st_mode): function(name) return mode = stat.S_IRUSR | stat.S_IWUSR if stat.S_ISDIR(details.st_mode): mode |= stat.S_IXUSR os.chmod(name, mode) function(name) for path in self.run_root.iterdir(): if path.is_symlink() or path.is_file(): try: path.unlink() except PermissionError: require(not path.is_symlink(), 'run_artifact_symlink_permission') os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) path.unlink() elif path.is_dir(): shutil.rmtree(path, onerror=retry_writable) else: raise Failure('run_artifact_type') require(not any(self.run_root.iterdir()), 'run_artifact_cleanup') def safe_evidence(self, value): content = json.dumps( value, ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii') + b'\n' require(len(content) <= MAX_SAFE_EVIDENCE, 'safe_evidence_bound') for secret in (*self.tokens.values(), *self.device_tokens.values()): require(secret.encode('ascii') not in content, 'secret_present_in_evidence') return content def retain_safe_evidence(self, name, value): content = self.safe_evidence(value) self.clear_run_artifacts() write_bytes(self.run_root / name, content) def write_summary(self, windows, linux, cleanup): summary = { 'schema': 1, 'status': 'passed', 'run_id': self.run_id, 'parallelism': PARALLELISM, 'targets': list(ALL_TARGETS), 'repository_heads': self.repository_heads, 'manifest_sha256': self.manifest_hashes, 'image_ids': self.image_details, 'bundle_sha256': { 'windows': windows['bundle_sha256'], 'linux': linux['bundle_sha256'], }, 'normalized_evidence_sha256': sha256_bytes(json.dumps( windows['evidence'], ensure_ascii=True, sort_keys=True, separators=(',', ':'), ).encode('ascii')), 'foreign_docker_state': 'unchanged', 'cleanup': cleanup, } self.retain_safe_evidence('summary.json', summary) return summary def run(self): self.preflight() windows = self.windows_phase() linux = self.linux_phase() require(windows['evidence'] == linux['evidence'], 'cross_platform_evidence_mismatch') cleanup = 'retained' if self.args.keep else 'complete' if not self.args.keep: self.tracker.cleanup() else: self.tracker.assert_foreign_unchanged() return self.write_summary(windows, linux, cleanup) def parse_args(argv=None): parser = argparse.ArgumentParser( description='Verify real packaged Windows and Linux workers across an API outage.', ) parser.add_argument('--windows-artifact', default=WINDOWS_ARTIFACT, help='portable Windows worker directory (default: %(default)s)') parser.add_argument('--linux-image', default=LINUX_IMAGE, help='already-built Linux worker image (default: %(default)s)') parser.add_argument('--test-image', default=TEST_IMAGE, help='already-built server/test image (default: %(default)s)') parser.add_argument('--wsl-distro', default='Ubuntu-24.04', help='WSL distribution that owns the Docker socket (default: %(default)s)') parser.add_argument('--timeout-seconds', type=int, default=1800, help='aggregate subprocess and test timeout (default: %(default)s)') parser.add_argument('--keep', action='store_true', help='retain labelled Docker resources after a successful run') args = parser.parse_args(argv) if not 300 <= args.timeout_seconds <= 7200: parser.error('--timeout-seconds must be between 300 and 7200') for name in ('linux_image', 'test_image', 'wsl_distro'): if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,255}', getattr(args, name)): parser.error('--' + name.replace('_', '-') + ' contains unsupported characters') return args def main(argv=None): verifier = None try: verifier = Verifier(parse_args(argv)) summary = verifier.run() except KeyboardInterrupt as exc: label = 'interrupted' failure_class = type(exc).__name__ exit_code = None except Failure as exc: label = str(exc) failure_class = type(exc).__name__ exit_code = getattr(exc, 'exit_code', None) except Exception as exc: label = 'unexpected_exception' failure_class = type(exc).__name__ exit_code = None else: print(json.dumps({ 'status': summary['status'], 'artifacts': os.fspath(verifier.run_root), 'cleanup': summary['cleanup'], }, ensure_ascii=True, sort_keys=True)) return 0 finally: if verifier is not None and verifier.windows_client is not None: verifier.windows_client.stop() if verifier is not None and verifier.loopback_proxy is not None: verifier.loopback_proxy.stop() if verifier is not None: verifier.runner.deadline = time.monotonic() + 720 verifier.tracker.guarded_stop() if not re.fullmatch(r'[a-z0-9_-]{1,160}', label): label = 'verifier_failure' if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,79}', failure_class): failure_class = 'Exception' if type(exit_code) is not int or not -(2 ** 31) <= exit_code < 2 ** 31: exit_code = None failure = { 'stage': label, 'class': failure_class, 'exit_code': exit_code, } if verifier is not None: try: if verifier.args.keep: write_bytes( verifier.run_root / 'failure.json', verifier.safe_evidence(failure), ) else: verifier.retain_safe_evidence('failure.json', failure) except Exception: pass print(json.dumps(failure, ensure_ascii=True, sort_keys=True), file=sys.stderr) return 1 if __name__ == '__main__': raise SystemExit(main())