from datetime import datetime, timedelta, timezone import json import os from pathlib import Path import sqlite3 import sys import tempfile from types import SimpleNamespace import unittest from unittest import mock ROOT = Path(__file__).resolve().parents[1] APP_DIR = ROOT / 'app' sys.path.insert(0, str(APP_DIR)) import console_runner from docker_depth_experiment import ( DOCKER_DEPTH_COLLECTION_GENERATION, DOCKER_DEPTH_SELECTOR_VERSION, canonical_ordered_query_hash, validate_docker_depth_config, ) import scanner_db from scanner_db import ScannerDB POLICY_SHA256 = 'a' * 64 class DockerDiscoveryProvenanceTests(unittest.TestCase): def setUp(self): self.environment = mock.patch.dict( os.environ, {'SCANNER_DB_URL': '', 'DATABASE_URL': ''}, ) self.environment.start() self.temp = tempfile.TemporaryDirectory() self.db = ScannerDB(db_path=os.path.join(self.temp.name, 'scanner.db')) self.run_id = self.db.start_run('test') def tearDown(self): self.db.close() self.temp.cleanup() self.environment.stop() def cycle(self, query, ordinal, count): return self.db.start_source_cycle( self.run_id, 'dockerhub', 'docker', 'search', query, ordinal + 1, count, ) @staticmethod def observation( queries, ordinal, cycle_id, *, page_number=1, per_page=100, pass_kind='deep', query_complete=False, total_count=None, ): if total_count is None: total_count = ( page_number * per_page if query_complete else (page_number + 1) * per_page ) return { 'cycle_id': cycle_id, 'query_ordinal': ordinal, 'query_count': len(queries), 'page_number': page_number, 'page_limit': 30, 'per_page': per_page, 'total_count': total_count, 'policy_sha256': POLICY_SHA256, 'pass_kind': pass_kind, 'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION, 'ordered_query_hash': canonical_ordered_query_hash(queries), 'query_complete': query_complete, } def scalar(self, sql, params=()): return self.db.conn.execute(sql, params).fetchone()[0] def test_current_collection_generation_overrides_old_runner_deep_marker(self): queries = ('generation-query',) ordered_hash = canonical_ordered_query_hash(queries) now = datetime.now(timezone.utc) state = console_runner.default_source_state() policies = {'generation-query': {'policy_sha256': POLICY_SHA256}} console_runner.mark_dockerhub_deep_dispatched( state, queries[0], POLICY_SHA256, now, ) state_path = Path(self.temp.name) / 'runner-state.json' state_path.write_text(json.dumps({ 'version': 1, 'sources': {'dockerhub': state}, }), encoding='utf-8') state = console_runner.load_state(str(state_path), { 'sources': {'dockerhub': {}}, })['sources']['dockerhub'] complete = self.db.dockerhub_discovery_generation_complete( 'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION, ordered_hash, 1, POLICY_SHA256, ) self.assertFalse(complete) self.assertTrue(console_runner.prepare_dockerhub_discovery_state( state, policies, queries[0], now + timedelta(hours=1), force_deep=not complete, )['deep']) timestamp = scanner_db.utc_now_iso() self.db.conn.execute( '''INSERT INTO docker_discovery_passes( pass_token, source, pass_kind, collection_generation, policy_sha256, ordered_queries_sha256, expected_query_count, completed_query_count, state, started_at, completed_at, created_at, updated_at ) VALUES ('legacy-generation-pass', 'dockerhub', 'deep', 'legacy', ?, ?, 1, 1, 'complete', ?, ?, ?, ?)''', (POLICY_SHA256, ordered_hash, timestamp, timestamp, timestamp, timestamp), ) self.db.conn.commit() self.assertFalse(self.db.dockerhub_discovery_generation_complete( 'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION, ordered_hash, 1, POLICY_SHA256, )) current_pass = self.db.conn.execute( '''INSERT INTO docker_discovery_passes( pass_token, source, pass_kind, collection_generation, policy_sha256, ordered_queries_sha256, expected_query_count, completed_query_count, state, started_at, completed_at, created_at, updated_at ) VALUES ('current-generation-pass', 'dockerhub', 'deep', ?, ?, ?, 1, 1, 'complete', ?, ?, ?, ?) RETURNING id''', ( DOCKER_DEPTH_COLLECTION_GENERATION, POLICY_SHA256, ordered_hash, timestamp, timestamp, timestamp, timestamp, ), ).fetchone() self.db.conn.execute( '''INSERT INTO docker_discovery_pages( pass_id, query, query_ordinal, page_number, result_count, total_count, admitted_count, query_complete, admission_kind, page_sha256, observed_at, created_at ) VALUES (?, 'generation-query', 0, 1, 0, 0, 0, 1, 'main', ?, ?, ?)''', (current_pass['id'], '7' * 64, timestamp, timestamp), ) self.db.conn.commit() complete = self.db.dockerhub_discovery_generation_complete( 'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION, ordered_hash, 1, POLICY_SHA256, ) self.assertTrue(complete) self.assertFalse(console_runner.prepare_dockerhub_discovery_state( state, policies, queries[0], now + timedelta(hours=1), force_deep=not complete, )['deep']) def test_main_provenance_failure_rolls_back_page_pass_and_queue_admission(self): queries = ('atomic-main',) cycle_id = self.cycle(queries[0], 0, 1) self.db.conn.execute( '''CREATE TRIGGER reject_fresh_provenance BEFORE INSERT ON docker_repository_query_provenance BEGIN SELECT RAISE(ABORT, 'forced provenance failure'); END''' ) self.db.conn.commit() with self.assertRaisesRegex(sqlite3.IntegrityError, 'forced provenance failure'): self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/atomic-main'], observation=self.observation( queries, 0, cycle_id, query_complete=True, ), ) for table in ( 'target_queue', 'docker_discovery_passes', 'docker_discovery_pages', 'docker_repository_query_provenance', 'docker_repository_query_observations', ): self.assertEqual(self.scalar(f'SELECT COUNT(*) FROM {table}'), 0, table) def test_main_pages_preserve_shared_query_relations_absolute_rank_and_replay(self): queries = ('first-query', 'second-query') first_cycle = self.cycle(queries[0], 0, len(queries)) second_cycle = self.cycle(queries[1], 1, len(queries)) first_page = self.observation(queries, 0, first_cycle) second_page = self.observation( queries, 0, first_cycle, page_number=2, query_complete=True, ) self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/page-one'], observation=first_page, ) page_two_repositories = [ 'owner/page-two', 'owner/shared', 'owner/shared', ] page_two = self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], page_two_repositories, observation=second_page, ) final = self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[1], ['owner/shared'], observation=self.observation( queries, 1, second_cycle, query_complete=True, ), ) self.assertEqual(page_two['pass_id'], final['pass_id']) self.assertTrue(final['pass_complete']) pages = self.db.conn.execute( '''SELECT query, source_cycle_id, retry_work_id, admission_kind FROM docker_discovery_pages WHERE pass_id = ? ORDER BY query_ordinal, page_number''', (final['pass_id'],), ).fetchall() self.assertEqual([tuple(row) for row in pages], [ ('first-query', first_cycle, None, 'main'), ('first-query', first_cycle, None, 'main'), ('second-query', second_cycle, None, 'main'), ]) shared_id = self.scalar( "SELECT id FROM target_queue WHERE normalized_target = 'owner/shared'" ) self.assertEqual(self.scalar( "SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/shared'" ), 1) self.assertEqual(self.scalar( '''SELECT COUNT(*) FROM docker_repository_query_provenance WHERE repository_queue_id = ?''', (shared_id,), ), 2) ranks = self.db.conn.execute( '''SELECT query, first_search_rank, best_search_rank, last_search_rank, observation_count, fresh_observation_count, fresh_complete_observation_count, fresh_coverage_eligible FROM docker_repository_query_provenance WHERE repository_queue_id = ? ORDER BY query''', (shared_id,), ).fetchall() self.assertEqual(tuple(ranks[0]), ( 'first-query', 102, 102, 102, 1, 1, 1, 1, )) self.assertEqual(tuple(ranks[1]), ( 'second-query', 1, 1, 1, 1, 1, 1, 1, )) replay = self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], page_two_repositories, observation=second_page, ) self.assertEqual(replay['pass_id'], final['pass_id']) self.assertEqual(replay['page_inserted_count'], 0) self.assertEqual(replay['observation_inserted_count'], 0) self.assertEqual(self.scalar( '''SELECT observation_count FROM docker_repository_query_provenance WHERE source = 'dockerhub' AND query = 'first-query' AND repository_queue_id = ?''', (shared_id,), ), 1) with self.assertRaisesRegex(RuntimeError, 'page evidence conflicts'): self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/changed'], observation=second_page, ) self.assertEqual(self.scalar( "SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/changed'" ), 0) def test_retry_pages_keep_original_pass_and_policy_under_the_lease(self): queries = ('retry-query',) origin_cycle = self.cycle(queries[0], 0, 1) enqueue = self.db.enqueue_discovery_retry( 'dockerhub', queries[0], POLICY_SHA256, 'deep', 'query', page_start=1, page_end=2, observation=self.observation(queries, 0, origin_cycle), ) later_cycle = self.cycle(queries[0], 0, 1) coalesced = self.db.enqueue_discovery_retry( 'dockerhub', queries[0], POLICY_SHA256, 'deep', 'query', page_start=1, page_end=2, observation=self.observation(queries, 0, later_cycle), ) self.assertEqual(coalesced['id'], enqueue['id']) self.assertEqual(coalesced['coalesced_count'], 1) self.assertEqual(coalesced['source_cycle_id'], origin_cycle) claim = self.db.claim_discovery_retries( 'dockerhub', {queries[0]: POLICY_SHA256}, 'retry-owner', )[0] self.assertEqual(claim['source_cycle_id'], origin_cycle) unrelated_current_cycle = self.cycle('unrelated-current-query', 0, 1) with self.assertRaisesRegex(RuntimeError, 'retry provenance changed'): self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/wrong-cycle'], retry_id=claim['id'], lease_owner=claim['lease_owner'], lease_token=claim['lease_token'], next_page=2, observation=self.observation( queries, 0, unrelated_current_cycle, ), ) self.assertEqual(self.scalar( 'SELECT next_page FROM discovery_retry_queue WHERE id = ?', (claim['id'],), ), 1) self.assertEqual(self.scalar( "SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/wrong-cycle'" ), 0) page_one = self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/retry-one'], retry_id=claim['id'], lease_owner=claim['lease_owner'], lease_token=claim['lease_token'], next_page=2, observation=self.observation(queries, 0, origin_cycle), ) page_two = self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/retry-two', 'owner/retry-ranked'], retry_id=claim['id'], lease_owner=claim['lease_owner'], lease_token=claim['lease_token'], complete=True, observation=self.observation( queries, 0, origin_cycle, page_number=2, query_complete=True, ), ) self.assertEqual(enqueue['pass_id'], page_one['pass_id']) self.assertEqual(page_one['pass_id'], page_two['pass_id']) self.assertTrue(page_two['pass_complete']) retry_row = self.db.conn.execute( '''SELECT status, source_cycle_id, lease_owner, lease_token, last_error_category, held_at FROM discovery_retry_queue WHERE id = ?''', (claim['id'],), ).fetchone() self.assertEqual(tuple(retry_row)[:5], ( 'held', origin_cycle, None, None, None, )) self.assertIsNotNone(retry_row['held_at']) pages = self.db.conn.execute( '''SELECT admission_kind, retry_work_id, source_cycle_id FROM docker_discovery_pages WHERE pass_id = ? ORDER BY page_number''', (page_two['pass_id'],), ).fetchall() self.assertEqual([tuple(row) for row in pages], [ ('retry', claim['id'], origin_cycle), ('retry', claim['id'], origin_cycle), ]) pass_row = self.db.conn.execute( '''SELECT policy_sha256, ordered_queries_sha256, pass_kind, state FROM docker_discovery_passes WHERE id = ?''', (page_two['pass_id'],), ).fetchone() self.assertEqual(tuple(pass_row), ( POLICY_SHA256, canonical_ordered_query_hash(queries), 'deep', 'complete', )) self.assertEqual(self.scalar( '''SELECT search_rank FROM docker_repository_query_observations WHERE repository_queue_id = ( SELECT id FROM target_queue WHERE normalized_target = 'owner/retry-ranked' )''' ), 102) def test_retry_provenance_failure_rolls_back_admission_and_retry_progress(self): queries = ('atomic-retry',) cycle_id = self.cycle(queries[0], 0, 1) retry = self.db.enqueue_discovery_retry( 'dockerhub', queries[0], POLICY_SHA256, 'deep', 'page', page_start=1, page_end=1, observation=self.observation(queries, 0, cycle_id), ) claim = self.db.claim_discovery_retries( 'dockerhub', {queries[0]: POLICY_SHA256}, 'retry-owner', )[0] self.db.conn.execute( '''CREATE TRIGGER reject_retry_provenance BEFORE INSERT ON docker_repository_query_provenance BEGIN SELECT RAISE(ABORT, 'forced retry provenance failure'); END''' ) self.db.conn.commit() with self.assertRaisesRegex(sqlite3.IntegrityError, 'forced retry provenance failure'): self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/atomic-retry'], retry_id=claim['id'], lease_owner=claim['lease_owner'], lease_token=claim['lease_token'], complete=True, observation=self.observation( queries, 0, cycle_id, query_complete=True, ), ) retry_row = self.db.conn.execute( '''SELECT status, next_page FROM discovery_retry_queue WHERE id = ?''', (retry['id'],), ).fetchone() self.assertEqual(tuple(retry_row), ('leased', 1)) self.assertEqual(self.scalar( "SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/atomic-retry'" ), 0) self.assertEqual(self.scalar('SELECT COUNT(*) FROM docker_discovery_pages'), 0) self.assertEqual(self.scalar( 'SELECT completed_query_count FROM docker_discovery_passes WHERE id = ?', (retry['pass_id'],), ), 0) def test_stale_retry_lease_rolls_back_page_and_preserves_prior_progress(self): queries = ('stale-retry',) origin_cycle = self.cycle(queries[0], 0, 1) retry = self.db.enqueue_discovery_retry( 'dockerhub', queries[0], POLICY_SHA256, 'deep', 'range', page_start=1, page_end=2, observation=self.observation(queries, 0, origin_cycle), ) stale_claim = self.db.claim_discovery_retries( 'dockerhub', {queries[0]: POLICY_SHA256}, 'stale-owner', )[0] self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/stale-first'], retry_id=stale_claim['id'], lease_owner=stale_claim['lease_owner'], lease_token=stale_claim['lease_token'], next_page=2, observation=self.observation(queries, 0, origin_cycle), ) self.db.conn.execute( '''UPDATE discovery_retry_queue SET lease_expires_at = '2000-01-01T00:00:00+00:00' WHERE id = ?''', (retry['id'],), ) self.db.conn.commit() current_claim = self.db.claim_discovery_retries( 'dockerhub', {queries[0]: POLICY_SHA256}, 'current-owner', )[0] with self.assertRaises(scanner_db.DiscoveryRetryLeaseError): self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/stale-second'], retry_id=stale_claim['id'], lease_owner=stale_claim['lease_owner'], lease_token=stale_claim['lease_token'], complete=True, observation=self.observation( queries, 0, origin_cycle, page_number=2, query_complete=True, ), ) current = self.db.conn.execute( '''SELECT status, next_page, lease_owner, lease_token, source_cycle_id FROM discovery_retry_queue WHERE id = ?''', (retry['id'],), ).fetchone() self.assertEqual(tuple(current), ( 'leased', 2, 'current-owner', current_claim['lease_token'], origin_cycle, )) self.assertEqual(self.scalar('SELECT COUNT(*) FROM docker_discovery_pages'), 1) self.assertEqual(self.scalar( "SELECT COUNT(*) FROM target_queue WHERE normalized_target = 'owner/stale-second'" ), 0) def test_only_complete_deep_pass_and_not_legacy_evidence_satisfies_coverage(self): queries = ('coverage-query',) deep_cycle = self.cycle(queries[0], 0, 1) ordinary_cycle = self.cycle(queries[0], 0, 1) self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/deep'], observation=self.observation( queries, 0, deep_cycle, query_complete=True, ), ) self.db.persist_dockerhub_discovery_page( 'dockerhub', queries[0], ['owner/ordinary'], observation=self.observation( queries, 0, ordinary_cycle, pass_kind='ordinary', query_complete=True, ), ) now = scanner_db.utc_now_iso() legacy_id = self.db.conn.execute( '''INSERT INTO target_queue( source, platform, query, target, normalized_target, status, resolver_state, created_at, updated_at ) VALUES ('dockerhub', 'docker', ?, 'owner/legacy', 'owner/legacy', 'deferred', 'pending', ?, ?) RETURNING id''', (queries[0], now, now), ).fetchone()['id'] self.db.conn.execute( '''INSERT INTO docker_repository_query_provenance( source, query, repository_queue_id, provenance_kind, first_observed_at, last_observed_at, created_at, updated_at ) VALUES ('dockerhub', ?, ?, 'legacy_queue', ?, ?, ?, ?)''', (queries[0], legacy_id, now, now, now, now), ) self.db.conn.commit() summary = self.db.dockerhub_discovery_coverage_summary( 'dockerhub', queries, canonical_ordered_query_hash(queries), {queries[0]: POLICY_SHA256}, required_repository_count=2, ) self.assertEqual(summary['state'], 'collecting') self.assertFalse(summary['planning_allowed']) self.assertEqual(summary['queries'][0]['eligible_repository_count'], 1) ordinary = self.db.conn.execute( '''SELECT fresh_complete_observation_count, fresh_coverage_eligible FROM docker_repository_query_provenance WHERE query = ? AND repository_queue_id = ( SELECT id FROM target_queue WHERE normalized_target = 'owner/ordinary' )''', (queries[0],), ).fetchone() self.assertEqual(tuple(ordinary), (0, 0)) def test_coverage_gate_requires_ten_eligible_rows_for_all_61_queries(self): queries = tuple(f'pinned-query-{index:02d}' for index in range(61)) ordered_hash = canonical_ordered_query_hash(queries) policies = {query: POLICY_SHA256 for query in queries} for ordinal, query in enumerate(queries): cycle_id = self.cycle(query, ordinal, len(queries)) repository_count = 16 if ordinal == 0 else 10 repositories = [ f'owner/q{ordinal:02d}-repo-{index:02d}' for index in range(repository_count) ] result = self.db.persist_dockerhub_discovery_page( 'dockerhub', query, repositories, observation=self.observation( queries, ordinal, cycle_id, query_complete=True, ), ) self.assertTrue(result['pass_complete']) pass_row = self.db.conn.execute( '''SELECT COUNT(*) AS count, MIN(completed_query_count) AS completed, MIN(state) AS state FROM docker_discovery_passes''' ).fetchone() self.assertEqual(tuple(pass_row), (1, 61, 'complete')) self.assertTrue(self.db.dockerhub_discovery_generation_complete( 'dockerhub', DOCKER_DEPTH_COLLECTION_GENERATION, ordered_hash, 61, POLICY_SHA256, )) rows = self.db.conn.execute( '''SELECT id, target FROM target_queue WHERE query = ? ORDER BY target''', (queries[0],), ).fetchall() excluded = rows[10:16] digest_target = excluded[0]['target'] + '@sha256:' + ('b' * 64) self.db.conn.execute( '''UPDATE target_queue SET target = ?, normalized_target = ? WHERE id = ?''', (digest_target, digest_target, excluded[0]['id']), ) for row, status in zip(excluded[1:5], ('done', 'failed', 'quarantined', 'cold')): self.db.conn.execute( 'UPDATE target_queue SET status = ? WHERE id = ?', (status, row['id']), ) historical = excluded[5] self.db.conn.execute( '''INSERT INTO target_scans( queue_id, source, query, target, normalized_target, status, created_at ) VALUES (?, 'dockerhub', ?, ?, ?, 'clean', ?)''', ( historical['id'], queries[0], historical['target'], historical['target'], scanner_db.utc_now_iso(), ), ) active_id = rows[0]['id'] self.db.conn.execute( '''UPDATE target_queue SET lease_owner = 'active-owner', lease_token = 'active-token' WHERE id = ?''', (active_id,), ) self.db.conn.commit() collecting = self.db.dockerhub_discovery_coverage_summary( 'dockerhub', queries, ordered_hash, policies, ) self.assertEqual(collecting['state'], 'collecting') self.assertFalse(collecting['planning_allowed']) self.assertEqual(collecting['covered_query_count'], 60) self.assertEqual(collecting['queries'][0]['eligible_repository_count'], 9) self.db.conn.execute( '''UPDATE target_queue SET lease_owner = NULL, lease_token = NULL WHERE id = ?''', (active_id,), ) self.db.conn.commit() complete = self.db.dockerhub_discovery_coverage_summary( 'dockerhub', queries, ordered_hash, policies, ) self.assertEqual(complete['state'], 'coverage_complete') self.assertTrue(complete['planning_allowed']) self.assertEqual(complete['covered_query_count'], 61) self.assertEqual(complete['minimum_eligible_repository_count'], 10) class DockerDiscoveryRunnerMetadataTests(unittest.TestCase): @staticmethod def config(): queries = [f'runner-query-{index:02d}' for index in range(61)] experiment = { 'experiment_key': 'runner-disabled-v1', 'enabled': False, 'queries': queries, 'repositories_per_query': 10, 'shallow_images_per_repository': 1, 'deep_repositories_per_query': 1, 'deep_images_per_repository': 10, 'target_limit': 1200, 'selector_version': DOCKER_DEPTH_SELECTOR_VERSION, } return { 'global': {'sync_file_queues': False}, 'sources': {'dockerhub': { 'queries': queries, 'mode': 'search', 'pages': 1, 'per_page': 100, 'require_digest': True, 'docker_platform_candidate_tags': 20, 'docker_images_per_repository': 3, 'docker_depth_experiment': experiment, }}, } def test_disabled_validated_experiment_still_supplies_page_observation_metadata(self): validated = validate_docker_depth_config(self.config()) self.assertFalse(validated.experiment.enabled) args = SimpleNamespace( platform='docker', mode='search', query=validated.experiment.queries[0], pages=1, per_page=100, docker_sort_by='updated_at', sort_order='desc', fetch_timeout=15, dockerhub_discovery_deep=True, ) policy = console_runner.dockerhub_discovery_policy(args) args.dockerhub_discovery_policy_sha256 = policy['policy_sha256'] console_runner.annotate_dockerhub_experiment_observation_args( args, validated.experiment, cycle_id=77, ) class DB: observation = None @staticmethod def runtime_control_state(): return {'effective_discovery_paused': False} def persist_dockerhub_discovery_page(self, _source, _query, repositories, **kwargs): self.observation = kwargs.get('observation') return { 'attempted_count': len(repositories), 'normalized_count': len(repositories), 'preexisting_count': 0, 'inserted_count': len(repositories), 'normalized_repositories': frozenset({'owner/runner'}), 'preexisting_repositories': frozenset(), } db = DB() response = { 'page': 1, 'repositories': [{'repo_name': 'owner/runner'}], 'total_count': 1, } with mock.patch.object( console_runner, 'fetch_dockerhub_search_page', return_value=response, ): console_runner.run_dockerhub_incremental_discovery(args, db) self.assertEqual(db.observation, { 'cycle_id': 77, 'query_ordinal': 0, 'query_count': 61, 'page_number': 1, 'page_limit': 1, 'per_page': 100, 'total_count': 1, 'policy_sha256': policy['policy_sha256'], 'pass_kind': 'deep', 'collection_generation': DOCKER_DEPTH_COLLECTION_GENERATION, 'ordered_query_hash': validated.experiment.ordered_query_hash, 'query_complete': True, }) def test_configured_source_integrates_disabled_experiment_before_collection(self): config = self.config() query = config['sources']['dockerhub']['queries'][0] state = { 'sources': {'dockerhub': console_runner.default_source_state()}, } built_args = SimpleNamespace(platform='docker', mode='search', query=query) seen = [] class Connection: is_postgres = True @staticmethod def rollback(): return None class DB: conn = Connection() @staticmethod def dockerhub_discovery_generation_complete(*_args, **_kwargs): return False @staticmethod def start_source_cycle(*_args, **_kwargs): return 77 @staticmethod def claim_discovery_retries(*_args, **_kwargs): return [] persist_dockerhub_discovery_page = staticmethod(lambda *_args, **_kwargs: {}) finish_discovery_retry = staticmethod(lambda *_args, **_kwargs: {}) renew_discovery_retry_lease = staticmethod(lambda *_args, **_kwargs: {}) update_discovery_retry = staticmethod(lambda *_args, **_kwargs: {}) hold_discovery_retry = staticmethod(lambda *_args, **_kwargs: {}) def run_cycle(args, *_args, **_kwargs): seen.append(args) return { 'cycle_status': 'completed', 'deep_dispatch_durable': True, 'source_failure_count': 0, 'scanned_count': 0, } with mock.patch.object(console_runner, 'save_state'), \ mock.patch.object(console_runner, 'select_auth_entry', return_value=None), \ mock.patch.object(console_runner, 'refresh_auth_summary'), \ mock.patch.object( console_runner, 'build_args_from_source_config', return_value=built_args, ), \ mock.patch.object(console_runner, 'configure_source_auth'), \ mock.patch.object(console_runner, 'persist_docker_auth_events'), \ mock.patch.object(console_runner, 'queue_counts_for_args', return_value={}), \ mock.patch.object(console_runner, 'run_cycle', side_effect=run_cycle): console_runner.run_configured_source( 'dockerhub', config, state, 'state.json', {}, DB(), run_id=3, ) self.assertEqual(len(seen), 1) self.assertTrue(seen[0].docker_depth_collection_only) self.assertEqual(seen[0].dockerhub_discovery_cycle_id, 77) self.assertEqual(seen[0].dockerhub_discovery_query_count, 61) self.assertEqual( seen[0].dockerhub_discovery_ordered_query_hash, canonical_ordered_query_hash( config['sources']['dockerhub']['queries'], ), ) if __name__ == '__main__': unittest.main()