sashatrask
138 lines
4.5 KiB
Python
138 lines
4.5 KiB
Python
import sys
|
|
|
|
sys.dont_write_bytecode = True
|
|
|
|
import argparse
|
|
import os
|
|
import sqlite3
|
|
|
|
from scanner_db import ScannerDB
|
|
|
|
|
|
TABLES = [
|
|
'runs',
|
|
'source_cycles',
|
|
'target_scans',
|
|
'target_queue',
|
|
'scan_publication_outbox',
|
|
'findings',
|
|
'errors',
|
|
'queue_snapshots',
|
|
'config_snapshots',
|
|
'package_repo_candidates',
|
|
'keycheck_results',
|
|
'keycheck_event_map',
|
|
'finding_uid_map',
|
|
]
|
|
|
|
|
|
def sqlite_connect_ro(path):
|
|
uri = 'file:' + os.path.abspath(path).replace('\\', '/') + '?mode=ro'
|
|
conn = sqlite3.connect(uri, uri=True)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
def sqlite_columns(conn, table):
|
|
return [row['name'] for row in conn.execute(f'PRAGMA table_info({table})').fetchall()]
|
|
|
|
|
|
def sqlite_count(conn, table):
|
|
return int(conn.execute(f'SELECT COUNT(*) AS count FROM {table}').fetchone()['count'])
|
|
|
|
|
|
def sqlite_row_estimate(conn, table, exact=False):
|
|
if exact:
|
|
return sqlite_count(conn, table)
|
|
try:
|
|
row = conn.execute('SELECT seq FROM sqlite_sequence WHERE name = ?', (table,)).fetchone()
|
|
if row and row['seq'] is not None:
|
|
return int(row['seq'])
|
|
except sqlite3.Error:
|
|
pass
|
|
return None
|
|
|
|
|
|
def pg_reset_identity(db, table):
|
|
db.conn.execute(f'''
|
|
SELECT setval(
|
|
pg_get_serial_sequence('{table}', 'id'),
|
|
COALESCE((SELECT MAX(id) FROM {table}), 1),
|
|
(SELECT MAX(id) IS NOT NULL FROM {table})
|
|
)
|
|
''')
|
|
|
|
|
|
def postgres_safe_value(value):
|
|
if isinstance(value, str) and '\x00' in value:
|
|
return value.replace('\x00', '\\u0000')
|
|
return value
|
|
|
|
|
|
def copy_table(source, target, table, batch_size, dry_run=False, exact_counts=False):
|
|
columns = sqlite_columns(source, table)
|
|
if not columns:
|
|
print(f'{table}: missing or empty schema in SQLite, skipped', flush=True)
|
|
return 0
|
|
total = sqlite_row_estimate(source, table, exact=exact_counts)
|
|
total_label = total if total is not None else 'unknown'
|
|
print(f'{table}: source_rows={total_label}' + ('' if exact_counts else ' estimated'), flush=True)
|
|
if dry_run or total == 0:
|
|
return int(total or 0)
|
|
|
|
column_sql = ', '.join(columns)
|
|
placeholders = ', '.join('?' for _ in columns)
|
|
conflict_sql = ' ON CONFLICT (id) DO NOTHING' if 'id' in columns else ''
|
|
insert_sql = f'INSERT INTO {table} ({column_sql}) VALUES ({placeholders}){conflict_sql}'
|
|
|
|
copied = 0
|
|
cursor = source.execute(f'SELECT {column_sql} FROM {table} ORDER BY id' if 'id' in columns else f'SELECT {column_sql} FROM {table}')
|
|
while True:
|
|
rows = cursor.fetchmany(batch_size)
|
|
if not rows:
|
|
break
|
|
values = [[postgres_safe_value(row[column]) for column in columns] for row in rows]
|
|
if target.conn.is_postgres:
|
|
with target.conn._conn.cursor() as pg_cursor:
|
|
with pg_cursor.copy(f'COPY {table} ({column_sql}) FROM STDIN') as copy:
|
|
for value_row in values:
|
|
copy.write_row(value_row)
|
|
else:
|
|
for value_row in values:
|
|
target.conn.execute(insert_sql, value_row)
|
|
copied += len(rows)
|
|
target.conn.commit()
|
|
print(f'{table}: copied={copied}/{total_label}', flush=True)
|
|
if 'id' in columns:
|
|
pg_reset_identity(target, table)
|
|
target.conn.commit()
|
|
return copied
|
|
|
|
|
|
def truncate_target(db):
|
|
table_sql = ', '.join(TABLES)
|
|
db.conn.execute(f'TRUNCATE TABLE {table_sql} RESTART IDENTITY CASCADE')
|
|
db.conn.commit()
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description='Offline migrate scanner observability SQLite DB to PostgreSQL.')
|
|
parser.add_argument('--sqlite', required=True, help='Path to scanner_active.db or archived SQLite DB')
|
|
parser.add_argument('--db-url', default=os.getenv('SCANNER_DB_URL') or os.getenv('DATABASE_URL'), help='PostgreSQL DSN')
|
|
parser.add_argument('--batch-size', type=int, default=1000)
|
|
parser.add_argument('--exact-counts', action='store_true', help='Use exact COUNT(*) per table; slow on multi-GB SQLite files')
|
|
parser.add_argument('--truncate', action='store_true', help='Delete existing Postgres observability rows before import')
|
|
parser.add_argument('--dry-run', action='store_true')
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
raise SystemExit(
|
|
'migrate_observability_db.py is retired for PostgreSQL. Use migrate_runtime_safety.py '
|
|
'--apply --sources-stopped with the bound cluster; perform legacy data import only with a reviewed, cluster-bound tool.'
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|