Initial server source import

This commit is contained in:
sashatrask
2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
+572
View File
@@ -0,0 +1,572 @@
import sys
sys.dont_write_bytecode = True
import requests
import json
import os
import argparse
import re
from itertools import cycle
import time
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from keycheck_common import (
commit_status_transaction,
default_input_file,
default_proxy_file,
ensure_output_files as ensure_private_output_files,
iter_findings,
iter_bounded_text_lines,
keycheck_input_mode,
load_known_statuses,
private_atomic_writer,
record_cached_keycheck_occurrence,
record_validation_result,
recover_status_transaction,
require_provider_authority,
service_output_dir,
should_skip_key,
write_keycheck_event,
)
try:
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
except Exception:
pass
# --- Конфигурация ---
SERVICE = "openai"
OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
ALIVE_FILE = os.path.join(OUTPUT_DIR, "openaiAlive.txt")
DEAD_FILE = os.path.join(OUTPUT_DIR, "openaiDead.txt")
NETWORK_FILE = os.path.join(OUTPUT_DIR, "openaiNetwork.txt")
LIMITED_FILE = os.path.join(OUTPUT_DIR, "openaiLimited.txt")
RESTRICTED_FILE = os.path.join(OUTPUT_DIR, "openaiRestricted.txt")
UNKNOWN_FILE = os.path.join(OUTPUT_DIR, "openaiUnknown.txt")
NO_TARGET_FILE = os.path.join(OUTPUT_DIR, "openaiNoTarget.txt")
CHECKED_FILE = os.path.join(OUTPUT_DIR, "openaiChecked.txt")
RESULTS_FILE = os.path.join(OUTPUT_DIR, "openaiResults.jsonl")
PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
MODEL_PRIORITY_FOR_TEST = (
'gpt-5.6-sol',
'gpt-5.6',
'gpt-5.6-luna',
'gpt-5.6-terra',
'gpt-5',
'o3-pro',
'o3',
)
TARGET_MODELS = {'gpt-5', 'o3-pro', 'o3'}
NON_CHAT_MODEL_MARKERS = (
'embedding', 'image', 'audio', 'tts', 'transcribe', 'realtime', 'search', 'moderation',
)
PROBE_MAX_COMPLETION_TOKENS = 16
STATUS_FILES = [ALIVE_FILE, DEAD_FILE, NETWORK_FILE, LIMITED_FILE, RESTRICTED_FILE, UNKNOWN_FILE, NO_TARGET_FILE]
STATUS_BY_FILE = {
ALIVE_FILE: 'ALIVE',
DEAD_FILE: 'DEAD',
NETWORK_FILE: 'NETWORK',
LIMITED_FILE: 'LIMITED',
RESTRICTED_FILE: 'RESTRICTED',
UNKNOWN_FILE: 'UNKNOWN',
NO_TARGET_FILE: 'NO_TARGET_MODELS',
}
OPENAI_KEY_REGEX = re.compile(r'sk-[A-Za-z0-9_-]{20,}')
# --- Вспомогательные функции ---
def key_from_line(line):
line = line.strip()
if not line:
return None
if '\t' in line:
return line.split('\t', 1)[0].strip()
if ':[' in line:
return line.split(':[', 1)[0].strip()
match = OPENAI_KEY_REGEX.search(line)
if match:
return match.group(0)
return line.split()[0].strip()
def load_set_from_file(filepath):
if keycheck_input_mode() == 'postgres':
return set()
if not os.path.exists(filepath): return set()
return {key for key in (key_from_line(line) for line in iter_bounded_text_lines(filepath)) if key}
def iter_plain_openai_keys(paths):
if keycheck_input_mode() == 'postgres':
return
seen = set()
items = []
for path in paths or []:
if not path or not os.path.exists(path):
continue
for line in iter_bounded_text_lines(path):
key = key_from_line(line)
if not key or key in seen:
continue
seen.add(key)
items.append({'key': key, 'source': path, 'finding': {}})
for item in items:
yield item
def retry_plain_files(args):
if keycheck_input_mode() == 'postgres':
return []
files = list(args.plain or [])
if args.recheck_all:
files.extend(STATUS_FILES)
else:
if args.retry_limited:
files.append(LIMITED_FILE)
if args.retry_network:
files.append(NETWORK_FILE)
if args.retry_unknown:
files.append(UNKNOWN_FILE)
if args.retry_restricted:
files.append(RESTRICTED_FILE)
if args.retry_no_balance:
files.append(LIMITED_FILE)
out = []
seen = set()
for path in files:
if path and path not in seen:
seen.add(path)
out.append(path)
return out
def load_checked_statuses():
if keycheck_input_mode() == 'postgres':
return {}
statuses = {}
if not os.path.exists(CHECKED_FILE):
return statuses
for line in iter_bounded_text_lines(CHECKED_FILE):
parts = line.rstrip('\n').split('\t')
if parts and parts[0]:
statuses[parts[0]] = parts[1] if len(parts) > 1 else 'UNKNOWN'
return statuses
def load_known_keys():
if keycheck_input_mode() == 'postgres':
return set()
known = set(load_checked_statuses().keys())
for path in STATUS_FILES:
known.update(load_set_from_file(path))
return known
def ensure_output_files():
ensure_private_output_files([CHECKED_FILE, RESULTS_FILE, *STATUS_FILES])
recover_status_transaction(CHECKED_FILE, STATUS_BY_FILE)
def compact_status_file(path):
if keycheck_input_mode() == 'postgres':
return
if not os.path.exists(path):
return
last_by_key = {}
order = []
for line in iter_bounded_text_lines(path):
key = key_from_line(line)
if not key:
continue
if key not in last_by_key:
order.append(key)
last_by_key[key] = line
for attempt in range(6):
try:
with private_atomic_writer(path) as f:
for key in order:
f.write(last_by_key[key])
except PermissionError:
if attempt == 5:
print(f"Warning: unable to compact {path}; leaving existing file as-is")
return
time.sleep(0.1 * (attempt + 1))
else:
return
def compact_all_status_files():
for path in [CHECKED_FILE, *STATUS_FILES]:
compact_status_file(path)
def backfill_checked_file():
if keycheck_input_mode() == 'postgres':
return
checked = load_checked_statuses()
changed = False
for path, status in STATUS_BY_FILE.items():
for key in load_set_from_file(path):
if key not in checked:
checked[key] = status
changed = True
if not changed:
return
with private_atomic_writer(CHECKED_FILE) as f:
for key, status in sorted(checked.items()):
f.write(f"{key}\t{status}\tbackfilled\n")
def load_proxies(proxy_file=None):
proxy_file = proxy_file or PROXY_FILE
if not os.path.exists(proxy_file): return None
proxies = []
with open(proxy_file, 'r') as f:
for line in f:
line = line.strip()
if not line: continue
try:
ip, port, login, password = line.split(':')
proxy_url = f"http://{login}:{password}@{ip}:{port}"
proxies.append({"http": proxy_url, "https": proxy_url})
except ValueError:
print(f"⚠️ Неверный формат прокси: '{line}'. Пропускаем.")
if not proxies:
print("⚠️ Файл proxy.txt пуст. Запросы будут идти напрямую.")
return None
print(f"✅ Загружено {len(proxies)} прокси.")
return cycle(proxies)
def move_key_to_alive(
key, available_target_models, service_tier, source='', finding=None,
model_inventory=None, probe_model='',
):
"""
Перемещает ключ из DEAD_FILE в ALIVE_FILE, записывая модели и service_tier.
"""
models_str = ",".join(sorted(list(available_target_models)))
tier_str = str(service_tier or 'unknown').replace('\r', ' ').replace('\n', ' ')[:1000]
print(f" -> ✅ Ключ рабочий! Модели: {models_str}, Тир: {tier_str}. Перемещаем в {ALIVE_FILE}")
model_inventory = sorted(set(model_inventory or available_target_models))
result = {
'status': 'ALIVE',
'models': sorted(list(available_target_models)),
'model_inventory': model_inventory,
'model_count': len(model_inventory),
'llm_probe_model': probe_model,
'llm_probe_status': 'GENERATION_OK',
'service_tier': service_tier,
'message': f'chat ping ok; model={probe_model}; models={len(model_inventory)}',
}
write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source, finding, 'OpenAI')
commit_status_transaction(
CHECKED_FILE,
STATUS_BY_FILE,
key,
'ALIVE',
status_line=f"{key}:[{models_str}]:{tier_str}\n",
checked_line=f"{key}\tALIVE\t{time.strftime('%Y-%m-%dT%H:%M:%S')}\n",
)
record_validation_result(SERVICE, key, result, source, finding, 'OpenAI')
def redact_message(message, key):
message = str(message).replace('\r', ' ').replace('\n', ' ')[:1000]
if key:
message = message.replace(key, '***REDACTED***')
return OPENAI_KEY_REGEX.sub('***REDACTED***', message)
def write_key_status(key, path, status, message='', source='', finding=None, metadata=None):
status_upper = status.upper()
projection_status = STATUS_BY_FILE.get(path)
if not projection_status:
raise ValueError(f'unknown OpenAI status projection: {path}')
message = redact_message(message, key)
result = {**(metadata or {}), 'status': status_upper, 'message': message}
write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source, finding, 'OpenAI')
commit_status_transaction(
CHECKED_FILE,
STATUS_BY_FILE,
key,
projection_status,
message,
source,
status_line=f"{key}\t{status}\t{message}\n",
checked_line=f"{key}\t{projection_status}\t{time.strftime('%Y-%m-%dT%H:%M:%S')}\n",
)
record_validation_result(SERVICE, key, result, source, finding, 'OpenAI')
def extract_openai_key(data):
if data.get("DetectorName") == "OpenAI":
return data.get("Raw") or data.get("RawV2")
if data.get("detector") == "OpenAI":
return data.get("raw") or data.get("raw_v2")
finding = data.get("finding")
if isinstance(finding, dict) and finding.get("DetectorName") == "OpenAI":
return finding.get("Raw") or finding.get("RawV2")
return None
# --- Функции проверки ---
def check_authentication(key, proxy):
print(f" [1/2] Проверка аутентификации...")
url = "https://api.openai.com/v1/models"
headers = {"Authorization": f"Bearer {key}"}
try:
response = requests.get(url, headers=headers, proxies=proxy, timeout=15)
if response.status_code == 200:
print(" -> Аутентификация пройдена.")
return 'valid', response.json().get('data', [])
elif response.status_code == 401:
print(" -> Ошибка 401: Ключ недействителен или отозван.")
return 'dead', response.text
elif response.status_code == 403:
print(f" -> Ошибка 403: ключ ограничен/заблокирован: {response.text[:300]}")
return 'restricted', response.text
elif response.status_code == 429:
print(f" -> Ошибка 429: rate limit / quota: {response.text[:300]}")
return 'limited', response.text
else:
print(f" -> Ошибка {response.status_code}: {response.text}")
return 'unknown', response.text
except requests.RequestException as e:
print(f" -> Ошибка сети: {e}")
return 'network', str(e)
def reportable_target_models(model_ids):
return {
model for model in model_ids
if model in TARGET_MODELS or model.startswith('gpt-5.6-') or model == 'gpt-5.6'
}
def choose_probe_model(model_ids):
models = [str(model or '') for model in model_ids if model]
by_lower = {model.lower(): model for model in models}
for model in MODEL_PRIORITY_FOR_TEST:
if model.lower() in by_lower:
return by_lower[model.lower()]
for model in models:
lowered = model.lower()
if lowered.startswith(('gpt-', 'o')) and not any(
marker in lowered for marker in NON_CHAT_MODEL_MARKERS
):
return model
return ''
def check_balance_and_tier(key, model_to_test, proxy):
"""
Проверяет баланс и возвращает service_tier в случае успеха.
"""
print(f" [2/2] Проверка баланса и тира...")
if not model_to_test:
print(" -> Не найдено подходящих моделей для теста баланса.")
return 'unknown', 'no chat-capable model from /models'
print(f" -> Используем модель для теста: {model_to_test}")
url = "https://api.openai.com/v1/chat/completions"
headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
payload = {
"model": model_to_test,
"messages": [{"role": "user", "content": "Reply with one digit."}],
"max_completion_tokens": PROBE_MAX_COMPLETION_TOKENS,
}
try:
response = requests.post(url, headers=headers, json=payload, proxies=proxy, timeout=20)
if response.status_code == 200:
# Успех, извлекаем service_tier
response_data = response.json()
service_tier = response_data.get('service_tier')
return 'ok', service_tier
elif response.status_code == 429:
print(f" -> Ошибка 429: Нет баланса или превышен лимит.")
return 'limited', response.text
elif response.status_code == 401:
print(f" -> Ошибка 401: ключ недействителен или отозван.")
return 'dead', response.text
elif response.status_code == 403:
print(f" -> Ошибка 403: ключ ограничен/заблокирован: {response.text[:300]}")
return 'restricted', response.text
else:
print(f" -> Ошибка {response.status_code}: {response.text}")
return 'unknown', response.text
except requests.RequestException as e:
print(f" -> Ошибка сети: {e}")
return 'network', str(e)
# --- Основной процесс ---
def parse_args():
parser = argparse.ArgumentParser(description='OpenAI key checker')
parser.add_argument('--input', default=INPUT_FILE)
parser.add_argument('--plain', action='append', default=[])
parser.add_argument('--proxy-file', default=PROXY_FILE)
parser.add_argument('--max-keys', type=int, default=0)
parser.add_argument('--retry-network', action='store_true')
parser.add_argument('--retry-limited', action='store_true')
parser.add_argument('--retry-unknown', action='store_true')
parser.add_argument('--retry-restricted', action='store_true')
parser.add_argument('--retry-no-balance', action='store_true')
parser.add_argument('--recheck-all', action='store_true')
return parser.parse_args()
def main():
require_provider_authority(SERVICE)
args = parse_args()
ensure_output_files()
compact_all_status_files()
backfill_checked_file()
print("--- 🚀 Запуск чекера ключей OpenAI 🚀 ---")
proxy_cycler = load_proxies(args.proxy_file)
checked_statuses = load_checked_statuses()
known_statuses = load_known_statuses(CHECKED_FILE, STATUS_BY_FILE)
known_keys = set(known_statuses)
alive_keys = load_set_from_file(ALIVE_FILE)
retry_statuses = set()
if args.retry_network:
retry_statuses.add('NETWORK')
if args.retry_limited:
retry_statuses.update(('LIMITED', 'LIMITED_OR_NO_BALANCE', 'LIMITED_OR_QUOTA'))
if args.retry_unknown:
retry_statuses.add('UNKNOWN')
if args.retry_restricted:
retry_statuses.add('RESTRICTED')
if args.retry_no_balance:
retry_statuses.update(('NO_BALANCE', 'NO_QUOTA', 'LIMITED_OR_NO_BALANCE', 'LIMITED_OR_QUOTA'))
print(f"📖 Загружено: {len(alive_keys)} живых ключей, {len(known_keys)} уже классифицированных ключей, {len(checked_statuses)} checked.")
if keycheck_input_mode() == 'jsonl' and not os.path.exists(args.input):
print(f"❌ Файл с секретами {args.input} не найден. Завершение.")
return
processed = 0
skipped = 0
seen_this_run = set()
def candidates():
for item in iter_findings(args.input, ['OpenAI']):
data = item.get('finding') or {}
key = item.get('raw') or extract_openai_key(data)
if key:
yield {'key': key, 'source': item.get('source') or args.input, 'finding': data}
seen_plain = set()
for item in iter_plain_openai_keys(retry_plain_files(args)):
key = item.get('key')
if key and key not in seen_plain:
seen_plain.add(key)
yield item
for item in candidates():
data = item.get('finding') or {}
source_line = item.get('source') or args.input
key = item.get('key')
if not key:
continue
if keycheck_input_mode() != 'postgres' and key in seen_this_run:
cached_status = checked_statuses.get(key) or known_statuses.get(key) or 'UNKNOWN'
record_cached_keycheck_occurrence(SERVICE, key, cached_status, source_line, data, 'OpenAI')
skipped += 1
continue
seen_this_run.add(key)
if should_skip_key(
key, checked_statuses, known_keys, args, retry_statuses,
service=SERVICE, source=source_line, finding=data, detector='OpenAI',
known_statuses=known_statuses,
):
skipped += 1
continue
if args.max_keys and processed >= args.max_keys:
break
processed += 1
print(f"\n[{processed}] 🎯 Новый кандидат: {key[:8]}...{key[-4:]} from {source_line}")
current_proxy = next(proxy_cycler) if proxy_cycler else None
auth_status, auth_data = check_authentication(key, current_proxy)
if auth_status == 'valid' and auth_data:
all_available_models_data = auth_data
all_model_ids = sorted({
str(model.get('id') or '') for model in all_available_models_data
if isinstance(model, dict) and model.get('id')
})
found_target_models = reportable_target_models(all_model_ids)
model_to_test = choose_probe_model(all_model_ids)
if not model_to_test:
print(" -> Ключ валиден, но не имеет подходящей chat-модели. Пропускаем.")
write_key_status(
key, NO_TARGET_FILE, 'no_target_models', ','.join(all_model_ids)[:500],
source_line, data, {
'models': sorted(found_target_models),
'model_inventory': all_model_ids,
'model_count': len(all_model_ids),
'llm_probe_status': 'NO_CONTEXT',
},
)
known_keys.add(key)
checked_statuses[key] = 'NO_TARGET_MODELS'
continue
balance_status, balance_data = check_balance_and_tier(key, model_to_test, current_proxy)
probe_metadata = {
'models': sorted(found_target_models or {model_to_test}),
'model_inventory': all_model_ids,
'model_count': len(all_model_ids),
'llm_probe_model': model_to_test,
'llm_probe_status': {
'ok': 'GENERATION_OK',
'limited': 'LIMITED',
'network': 'NETWORK',
'restricted': 'RESTRICTED',
'dead': 'DEAD',
}.get(balance_status, 'UNKNOWN'),
}
# Проверяем, что результат не None (успешная проверка баланса)
if balance_status == 'ok':
move_key_to_alive(
key, found_target_models or {model_to_test}, balance_data, source_line, data,
model_inventory=all_model_ids, probe_model=model_to_test,
)
alive_keys.add(key)
final_status = 'ALIVE'
elif balance_status == 'limited':
write_key_status(key, LIMITED_FILE, 'limited_or_no_balance', balance_data, source_line, data, probe_metadata)
final_status = 'LIMITED_OR_NO_BALANCE'
elif balance_status == 'network':
write_key_status(key, NETWORK_FILE, 'network_error', balance_data, source_line, data, probe_metadata)
final_status = 'NETWORK'
elif balance_status == 'restricted':
write_key_status(key, RESTRICTED_FILE, 'restricted', balance_data, source_line, data, probe_metadata)
final_status = 'RESTRICTED'
elif balance_status == 'dead':
write_key_status(key, DEAD_FILE, 'invalid_or_revoked', balance_data, source_line, data, probe_metadata)
final_status = 'DEAD'
else:
write_key_status(key, UNKNOWN_FILE, 'unknown', balance_data, source_line, data, probe_metadata)
final_status = 'UNKNOWN'
known_keys.add(key)
checked_statuses[key] = final_status
elif auth_status == 'network':
write_key_status(key, NETWORK_FILE, 'network_error', auth_data, source_line, data)
known_keys.add(key)
checked_statuses[key] = 'NETWORK'
elif auth_status == 'limited':
write_key_status(key, LIMITED_FILE, 'limited_or_quota', auth_data, source_line, data)
known_keys.add(key)
checked_statuses[key] = 'LIMITED'
elif auth_status == 'restricted':
write_key_status(key, RESTRICTED_FILE, 'restricted', auth_data, source_line, data)
known_keys.add(key)
checked_statuses[key] = 'RESTRICTED'
elif auth_status == 'dead':
write_key_status(key, DEAD_FILE, 'invalid_or_revoked', auth_data, source_line, data)
known_keys.add(key)
checked_statuses[key] = 'DEAD'
else:
write_key_status(key, UNKNOWN_FILE, 'unknown', auth_data, source_line, data)
known_keys.add(key)
checked_statuses[key] = 'UNKNOWN'
print("\n--- ✅ Проверка завершена. ---")
print(f"Processed={processed}, skipped={skipped}")
if __name__ == "__main__":
main()