sashatrask
499 lines
19 KiB
Python
499 lines
19 KiB
Python
import sys
|
||
|
||
sys.dont_write_bytecode = True
|
||
|
||
import requests
|
||
import json
|
||
import os
|
||
import argparse
|
||
from itertools import cycle
|
||
from datetime import datetime, timezone
|
||
|
||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
try:
|
||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
||
except Exception:
|
||
pass
|
||
|
||
from keycheck_common import (
|
||
acquire_file_lock,
|
||
append_checked,
|
||
append_jsonl,
|
||
commit_status_transaction,
|
||
default_input_file,
|
||
default_proxy_file,
|
||
env_int,
|
||
ensure_output_files as ensure_private_output_files,
|
||
finding_detector_secret_hash,
|
||
iter_findings,
|
||
iter_bounded_text_lines,
|
||
keycheck_input_mode,
|
||
load_known_statuses,
|
||
load_checked_statuses,
|
||
mask_secret,
|
||
now_iso,
|
||
physical_jsonl_segments,
|
||
private_append_writer,
|
||
private_atomic_writer,
|
||
reconcile_keycheck_jsonl_segments,
|
||
record_validation_result,
|
||
recover_status_transaction,
|
||
release_file_lock,
|
||
repair_keycheck_jsonl_tail,
|
||
require_provider_authority,
|
||
rotate_jsonl_if_needed,
|
||
service_output_dir,
|
||
should_skip_key,
|
||
sha256_text,
|
||
write_keycheck_event,
|
||
)
|
||
from runtime_security import reject_reparse_components, require_private_file
|
||
|
||
# --- Конфигурация ---
|
||
SERVICE = "openrouter"
|
||
OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
|
||
INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
|
||
ALIVE_FILE = os.path.join(OUTPUT_DIR, "openrouterAlive.txt")
|
||
DEAD_FILE = os.path.join(OUTPUT_DIR, "openrouterDead.txt")
|
||
LIMITED_FILE = os.path.join(OUTPUT_DIR, "openrouterLimited.txt")
|
||
NO_BALANCE_FILE = os.path.join(OUTPUT_DIR, "openrouterNoBalance.txt")
|
||
NETWORK_FILE = os.path.join(OUTPUT_DIR, "openrouterNetwork.txt")
|
||
UNKNOWN_FILE = os.path.join(OUTPUT_DIR, "openrouterUnknown.txt")
|
||
CHECKED_FILE = os.path.join(OUTPUT_DIR, "openrouterChecked.txt")
|
||
RESULTS_FILE = os.path.join(OUTPUT_DIR, "openrouterResults.jsonl")
|
||
PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
|
||
STATUS_FILES = {
|
||
"VALID": ALIVE_FILE,
|
||
"NO_BALANCE": NO_BALANCE_FILE,
|
||
"DEAD": DEAD_FILE,
|
||
"LIMITED": LIMITED_FILE,
|
||
"NETWORK": NETWORK_FILE,
|
||
"UNKNOWN": UNKNOWN_FILE,
|
||
}
|
||
|
||
CREDITS_URL = "https://openrouter.ai/api/v1/credits"
|
||
|
||
# --- Вспомогательные функции ---
|
||
def load_set_from_file(filepath):
|
||
"""Загружает ключи из файла в set для быстрой проверки."""
|
||
if not os.path.exists(filepath):
|
||
return set()
|
||
return {line.strip().split(':')[0] for line in iter_bounded_text_lines(filepath) if line.strip()}
|
||
|
||
def ensure_output_files():
|
||
ensure_private_output_files((CHECKED_FILE, RESULTS_FILE, *STATUS_FILES.values()))
|
||
recover_status_transaction(CHECKED_FILE, STATUS_FILES)
|
||
|
||
|
||
def legacy_status_key(line):
|
||
value = line.strip()
|
||
if not value:
|
||
return None
|
||
if '\t' in value:
|
||
return value.split('\t', 1)[0].strip()
|
||
return value.split(':', 1)[0].strip()
|
||
|
||
|
||
def load_openrouter_keys(path):
|
||
if keycheck_input_mode() == 'postgres':
|
||
return set()
|
||
if not os.path.exists(path):
|
||
return set()
|
||
return {key for key in (legacy_status_key(line) for line in iter_bounded_text_lines(path)) if key}
|
||
|
||
|
||
def iter_plain_openrouter_keys(paths):
|
||
if keycheck_input_mode() == 'postgres':
|
||
return
|
||
seen = set()
|
||
items = []
|
||
for path in paths or []:
|
||
if not path or not os.path.exists(path):
|
||
continue
|
||
for line in iter_bounded_text_lines(path):
|
||
key = legacy_status_key(line)
|
||
if not key or key in seen:
|
||
continue
|
||
seen.add(key)
|
||
items.append({'raw': key, 'source': path, 'finding': {}})
|
||
for item in items:
|
||
yield item
|
||
|
||
|
||
def retry_plain_files(args):
|
||
if keycheck_input_mode() == 'postgres':
|
||
return []
|
||
files = list(args.plain or [])
|
||
if args.recheck_all:
|
||
files.extend(STATUS_FILES.values())
|
||
else:
|
||
if args.retry_valid:
|
||
files.append(ALIVE_FILE)
|
||
if args.retry_no_balance:
|
||
files.append(NO_BALANCE_FILE)
|
||
if args.retry_limited:
|
||
files.append(LIMITED_FILE)
|
||
if args.retry_network:
|
||
files.append(NETWORK_FILE)
|
||
if args.retry_unknown:
|
||
files.append(UNKNOWN_FILE)
|
||
out = []
|
||
seen = set()
|
||
for path in files:
|
||
if path and path not in seen:
|
||
seen.add(path)
|
||
out.append(path)
|
||
return out
|
||
|
||
|
||
def migrate_legacy_checked():
|
||
if keycheck_input_mode() == 'postgres':
|
||
return
|
||
checked = load_checked_statuses(CHECKED_FILE)
|
||
for key in sorted(load_openrouter_keys(ALIVE_FILE)):
|
||
if key not in checked:
|
||
write_keycheck_event(SERVICE, RESULTS_FILE, key, {'status': 'VALID', 'message': 'legacy alive status migration'}, 'legacy:openrouterAlive.txt', {}, 'OpenRouter', 'legacy_status')
|
||
append_checked(CHECKED_FILE, key, 'VALID')
|
||
checked[key] = 'VALID'
|
||
for key in sorted(load_openrouter_keys(DEAD_FILE)):
|
||
if key not in checked:
|
||
write_keycheck_event(SERVICE, RESULTS_FILE, key, {'status': 'DEAD', 'message': 'legacy dead status migration'}, 'legacy:openrouterDead.txt', {}, 'OpenRouter', 'legacy_status')
|
||
append_checked(CHECKED_FILE, key, 'DEAD')
|
||
checked[key] = 'DEAD'
|
||
|
||
|
||
def _legacy_alive_checked_at(path):
|
||
details = os.stat(path, follow_symlinks=False)
|
||
return datetime.fromtimestamp(details.st_mtime, timezone.utc).isoformat(timespec='seconds')
|
||
|
||
|
||
def _legacy_balance_event_payload(key, balance, checked_at):
|
||
balance_text = f'{balance:.6f}'
|
||
key_hash = sha256_text(key)
|
||
event_id = sha256_text('|'.join([
|
||
SERVICE,
|
||
'legacy_status',
|
||
'openrouterAlive.txt',
|
||
key_hash,
|
||
'NO_BALANCE',
|
||
balance_text,
|
||
]))
|
||
return {
|
||
'key_masked': mask_secret(key),
|
||
'key_hash': key_hash,
|
||
'secret_hash': key_hash,
|
||
'detector_secret_hash': finding_detector_secret_hash({}),
|
||
'finding_uid': '',
|
||
'detector': 'OpenRouter',
|
||
'source': 'legacy:openrouterAlive.txt',
|
||
'finding': {},
|
||
'checked_at': checked_at,
|
||
'result_source': 'legacy_status',
|
||
'status': 'NO_BALANCE',
|
||
'message': f'credits={balance_text}',
|
||
'event_id': event_id,
|
||
}
|
||
|
||
|
||
def _publish_legacy_no_balance(moved):
|
||
lock_path = f'{NO_BALANCE_FILE}.lock'
|
||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||
try:
|
||
require_private_file(NO_BALANCE_FILE)
|
||
existing = load_openrouter_keys(NO_BALANCE_FILE)
|
||
with private_append_writer(NO_BALANCE_FILE) as handle:
|
||
for key, balance in moved:
|
||
if key in existing:
|
||
continue
|
||
balance_text = f'{balance:.6f}'
|
||
handle.write(f'{key}\tNO_BALANCE\tcredits={balance_text}\tmigrated_from_alive\n')
|
||
existing.add(key)
|
||
finally:
|
||
release_file_lock(lock, lock_path)
|
||
|
||
|
||
def _existing_legacy_event_ids(expected):
|
||
found = set()
|
||
max_line_bytes = max(1024, env_int('KEYCHECK_INPUT_MAX_LINE_BYTES', 16 * 1024 * 1024))
|
||
max_file_bytes = max(
|
||
max_line_bytes,
|
||
max(1, env_int('KEYCHECK_INPUT_LIST_MAX_BYTES', 32 * 1024 * 1024)),
|
||
max(0, env_int('KEYCHECK_RESULTS_MAX_MB', 32)) * 1024 * 1024 + max_line_bytes,
|
||
)
|
||
paths = [path for _, path in physical_jsonl_segments(RESULTS_FILE)]
|
||
if os.path.isfile(RESULTS_FILE):
|
||
paths.append(os.path.abspath(RESULTS_FILE))
|
||
for path in paths:
|
||
reject_reparse_components(path)
|
||
if os.path.getsize(path) > max_file_bytes:
|
||
raise RuntimeError(f'OpenRouter result file exceeds the bounded migration scan size: {path}')
|
||
with open(path, 'rb') as handle:
|
||
while True:
|
||
raw_line = handle.readline(max_line_bytes + 1)
|
||
if not raw_line:
|
||
break
|
||
if len(raw_line) > max_line_bytes:
|
||
raise RuntimeError(f'OpenRouter result line exceeds the bounded migration scan size: {path}')
|
||
if not raw_line.endswith(b'\n'):
|
||
raise RuntimeError(f'torn OpenRouter result line during legacy migration: {path}')
|
||
try:
|
||
payload = json.loads(raw_line.decode('utf-8', errors='replace'))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
event_id = str(payload.get('event_id') or '') if isinstance(payload, dict) else ''
|
||
if event_id not in expected:
|
||
continue
|
||
wanted = expected[event_id]
|
||
for field in ('key_hash', 'status', 'source', 'result_source'):
|
||
if str(payload.get(field) or '') != str(wanted.get(field) or ''):
|
||
raise RuntimeError(f'conflicting OpenRouter legacy migration event: {event_id}')
|
||
found.add(event_id)
|
||
return found
|
||
|
||
|
||
def _publish_legacy_balance_events(moved, checked_at):
|
||
payloads = [_legacy_balance_event_payload(key, balance, checked_at) for key, balance in moved]
|
||
expected = {payload['event_id']: payload for payload in payloads}
|
||
lock_path = f'{RESULTS_FILE}.lock'
|
||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||
try:
|
||
require_private_file(RESULTS_FILE)
|
||
repair_keycheck_jsonl_tail(RESULTS_FILE)
|
||
reconcile_keycheck_jsonl_segments(RESULTS_FILE)
|
||
existing = _existing_legacy_event_ids(expected)
|
||
max_bytes = max(0, env_int('KEYCHECK_RESULTS_MAX_MB', 32)) * 1024 * 1024
|
||
for payload in payloads:
|
||
event_id = payload['event_id']
|
||
if event_id in existing:
|
||
continue
|
||
rotate_jsonl_if_needed(RESULTS_FILE, max_bytes)
|
||
with private_append_writer(RESULTS_FILE) as handle:
|
||
handle.write(json.dumps(payload, ensure_ascii=False, default=str) + '\n')
|
||
existing.add(event_id)
|
||
finally:
|
||
release_file_lock(lock, lock_path)
|
||
|
||
|
||
def _publish_legacy_checked(moved, checked_at):
|
||
lock_path = f'{CHECKED_FILE}.lock'
|
||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||
try:
|
||
require_private_file(CHECKED_FILE)
|
||
existing = set()
|
||
for line in iter_bounded_text_lines(CHECKED_FILE):
|
||
parts = line.rstrip('\r\n').split('\t')
|
||
if len(parts) >= 2:
|
||
existing.add((parts[0], parts[1]))
|
||
with private_append_writer(CHECKED_FILE) as handle:
|
||
for key, _ in moved:
|
||
identity = (key, 'NO_BALANCE')
|
||
if identity in existing:
|
||
continue
|
||
handle.write(f'{key}\tNO_BALANCE\t{checked_at}\n')
|
||
existing.add(identity)
|
||
finally:
|
||
release_file_lock(lock, lock_path)
|
||
|
||
|
||
def _rewrite_legacy_alive(keep):
|
||
with private_atomic_writer(ALIVE_FILE, binary=True, suffix='.legacy.tmp') as handle:
|
||
for line in keep:
|
||
handle.write(line.encode('utf-8'))
|
||
|
||
|
||
def migrate_legacy_alive_balances():
|
||
if keycheck_input_mode() == 'postgres':
|
||
return
|
||
lock_path = f'{ALIVE_FILE}.lock'
|
||
lock = acquire_file_lock(lock_path, timeout_sec=30)
|
||
try:
|
||
if not os.path.exists(ALIVE_FILE):
|
||
return
|
||
require_private_file(ALIVE_FILE)
|
||
checked_at = _legacy_alive_checked_at(ALIVE_FILE)
|
||
keep = []
|
||
moved = []
|
||
seen = set()
|
||
for line in iter_bounded_text_lines(ALIVE_FILE):
|
||
text = line.strip()
|
||
if not text:
|
||
keep.append(line)
|
||
continue
|
||
key = legacy_status_key(text)
|
||
balance = None
|
||
if ':' in text and '\t' not in text:
|
||
try:
|
||
balance = float(text.rsplit(':', 1)[1])
|
||
except ValueError:
|
||
balance = None
|
||
if key and balance is not None and balance <= 0:
|
||
if key not in seen:
|
||
moved.append((key, balance))
|
||
seen.add(key)
|
||
else:
|
||
keep.append(line)
|
||
if not moved:
|
||
return
|
||
_publish_legacy_no_balance(moved)
|
||
_publish_legacy_balance_events(moved, checked_at)
|
||
_publish_legacy_checked(moved, checked_at)
|
||
_rewrite_legacy_alive(keep)
|
||
finally:
|
||
release_file_lock(lock, lock_path)
|
||
|
||
|
||
def load_proxies(proxy_file=None):
|
||
"""Загружает и подготавливает прокси."""
|
||
proxy_file = proxy_file or PROXY_FILE
|
||
if not os.path.exists(proxy_file):
|
||
print("ℹ️ Файл proxy.txt не найден, запросы будут идти напрямую.")
|
||
return None
|
||
proxies = []
|
||
with open(proxy_file, 'r') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line: continue
|
||
try:
|
||
ip, port, login, password = line.split(':')
|
||
proxy_url = f"http://{login}:{password}@{ip}:{port}"
|
||
proxies.append({"http": proxy_url, "https": proxy_url})
|
||
except ValueError:
|
||
print(f"⚠️ Неверный формат прокси: '{line}'. Пропускаем.")
|
||
if not proxies:
|
||
print("⚠️ Файл proxy.txt пуст. Запросы будут идти напрямую.")
|
||
return None
|
||
print(f"✅ Загружено {len(proxies)} прокси.")
|
||
return cycle(proxies)
|
||
|
||
def write_result(key, result, source_line, finding=None, previous_status=None):
|
||
status = result.get('status') or 'UNKNOWN'
|
||
credits = result.get('remaining_credits')
|
||
extra = f"credits={credits:.6f}" if isinstance(credits, (int, float)) else source_line
|
||
checked_at = now_iso()
|
||
result = {**result, 'checked_at': result.get('checked_at') or checked_at}
|
||
write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source_line, finding, 'OpenRouter')
|
||
commit_status_transaction(
|
||
CHECKED_FILE, STATUS_FILES, key, status, result.get('message', ''), extra,
|
||
)
|
||
record_validation_result(SERVICE, key, result, source_line, finding, 'OpenRouter')
|
||
|
||
# --- Функция проверки ---
|
||
def check_openrouter_key(key, proxy):
|
||
"""Проверяет один ключ OpenRouter и возвращает normalized result."""
|
||
headers = {"Authorization": f"Bearer {key}"}
|
||
try:
|
||
resp = requests.get(CREDITS_URL, headers=headers, proxies=proxy, timeout=15)
|
||
except requests.exceptions.RequestException as e:
|
||
return {'status': 'NETWORK', 'message': str(e)}
|
||
|
||
if resp.status_code == 200:
|
||
try:
|
||
data = resp.json().get("data", {})
|
||
total = float(data.get("total_credits", 0.0) or 0.0)
|
||
used = float(data.get("total_usage", 0.0) or 0.0)
|
||
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
||
return {'status': 'UNKNOWN', 'http_status': resp.status_code, 'message': f'invalid credits response: {exc}'}
|
||
remaining = total - used
|
||
if remaining > 0:
|
||
return {'status': 'VALID', 'remaining_credits': remaining, 'message': f'credits={remaining:.6f}'}
|
||
return {'status': 'NO_BALANCE', 'remaining_credits': remaining, 'message': f'credits={remaining:.6f}'}
|
||
if resp.status_code in (401, 403):
|
||
return {'status': 'DEAD', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||
if resp.status_code == 429:
|
||
return {'status': 'LIMITED', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||
if 500 <= resp.status_code <= 599:
|
||
return {'status': 'NETWORK', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||
return {'status': 'UNKNOWN', 'http_status': resp.status_code, 'message': resp.text[:1000]}
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(description='OpenRouter key checker')
|
||
parser.add_argument('--input', default=INPUT_FILE)
|
||
parser.add_argument('--plain', action='append', default=[])
|
||
parser.add_argument('--proxy-file', default=PROXY_FILE)
|
||
parser.add_argument('--max-keys', type=int, default=0)
|
||
parser.add_argument('--retry-network', action='store_true')
|
||
parser.add_argument('--retry-limited', action='store_true')
|
||
parser.add_argument('--retry-unknown', action='store_true')
|
||
parser.add_argument('--retry-no-balance', action='store_true')
|
||
parser.add_argument('--retry-valid', action='store_true')
|
||
parser.add_argument('--recheck-all', action='store_true')
|
||
return parser.parse_args()
|
||
|
||
|
||
# --- Основной процесс ---
|
||
def main():
|
||
require_provider_authority(SERVICE)
|
||
args = parse_args()
|
||
ensure_output_files()
|
||
migrate_legacy_alive_balances()
|
||
migrate_legacy_checked()
|
||
print("--- 🚀 Запуск чекера ключей OpenRouter 🚀 ---")
|
||
proxy_cycler = load_proxies(args.proxy_file)
|
||
|
||
checked_statuses = load_checked_statuses(CHECKED_FILE)
|
||
known_statuses = load_known_statuses(CHECKED_FILE, STATUS_FILES)
|
||
known_keys = set(known_statuses)
|
||
for path in STATUS_FILES.values():
|
||
known_keys.update(load_openrouter_keys(path))
|
||
retry_statuses = set()
|
||
if args.retry_network:
|
||
retry_statuses.add('NETWORK')
|
||
if args.retry_limited:
|
||
retry_statuses.add('LIMITED')
|
||
if args.retry_unknown:
|
||
retry_statuses.add('UNKNOWN')
|
||
if args.retry_no_balance:
|
||
retry_statuses.add('NO_BALANCE')
|
||
if args.retry_valid:
|
||
retry_statuses.add('VALID')
|
||
print(f"📖 Загружено: {len(load_openrouter_keys(ALIVE_FILE))} живых ключей, {len(known_keys)} классифицированных ключей.")
|
||
|
||
if keycheck_input_mode() == 'jsonl' and not os.path.exists(args.input):
|
||
print(f"❌ Файл с секретами {args.input} не найден. Завершение.")
|
||
return
|
||
|
||
processed = 0
|
||
def candidates():
|
||
for item in iter_findings(args.input, ["OpenRouter"]):
|
||
key = item.get("raw") or ""
|
||
if key:
|
||
yield item
|
||
seen = set()
|
||
for item in iter_plain_openrouter_keys(retry_plain_files(args)):
|
||
key = item.get("raw") or ""
|
||
if key and key not in seen:
|
||
seen.add(key)
|
||
yield item
|
||
|
||
for item in candidates():
|
||
key = item.get("raw") or ""
|
||
if not key:
|
||
continue
|
||
|
||
source = item.get("source") or args.input
|
||
finding = item.get("finding") or {}
|
||
if should_skip_key(
|
||
key, checked_statuses, known_keys, args, retry_statuses,
|
||
service=SERVICE, source=source, finding=finding, detector='OpenRouter', known_statuses=known_statuses,
|
||
):
|
||
continue
|
||
if args.max_keys and processed >= args.max_keys:
|
||
break
|
||
processed += 1
|
||
|
||
print(f"\n[{processed}] 🎯 Новый кандидат: {key[:8]}...{key[-4:]} from {source}")
|
||
current_proxy = next(proxy_cycler) if proxy_cycler else None
|
||
result = check_openrouter_key(key, current_proxy)
|
||
print(f" STATUS: {result.get('status')} | {str(result.get('message', ''))[:200]}")
|
||
previous_status = known_statuses.get(key) or checked_statuses.get(key)
|
||
write_result(key, result, source, finding, previous_status)
|
||
known_keys.add(key)
|
||
checked_statuses[key] = result.get('status')
|
||
|
||
print("\n--- ✅ Проверка завершена. ---")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|