Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,240 @@
|
||||
import sys
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from keycheck_common import (
|
||||
append_jsonl,
|
||||
commit_status_transaction,
|
||||
default_input_file,
|
||||
default_proxy_file,
|
||||
ensure_output_files,
|
||||
iter_findings,
|
||||
keycheck_input_mode,
|
||||
load_checked_statuses,
|
||||
load_known_keys,
|
||||
load_proxies,
|
||||
mask_secret,
|
||||
recover_status_transaction,
|
||||
request_error_message,
|
||||
record_validation_result,
|
||||
require_provider_authority,
|
||||
service_output_dir,
|
||||
should_skip_key,
|
||||
write_keycheck_event,
|
||||
)
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(SCRIPT_DIR)
|
||||
SERVICE = "dockerhub"
|
||||
OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
|
||||
|
||||
INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
|
||||
PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
|
||||
PLAIN_FILE = os.path.join(OUTPUT_DIR, "dockerhub.txt")
|
||||
CHECKED_FILE = os.path.join(OUTPUT_DIR, "dockerhubChecked.txt")
|
||||
RESULTS_FILE = os.path.join(OUTPUT_DIR, "dockerhubResults.jsonl")
|
||||
|
||||
STATUS_FILES = {
|
||||
"VALID": os.path.join(OUTPUT_DIR, "dockerhubAlive.txt"),
|
||||
"VALID_2FA": os.path.join(OUTPUT_DIR, "dockerhubAlive.txt"),
|
||||
"DEAD": os.path.join(OUTPUT_DIR, "dockerhubDead.txt"),
|
||||
"NO_USERNAME": os.path.join(OUTPUT_DIR, "dockerhubNoUsername.txt"),
|
||||
"RATE_LIMITED": os.path.join(OUTPUT_DIR, "dockerhubRateLimited.txt"),
|
||||
"NETWORK": os.path.join(OUTPUT_DIR, "dockerhubNetwork.txt"),
|
||||
"UNKNOWN": os.path.join(OUTPUT_DIR, "dockerhubUnknown.txt"),
|
||||
}
|
||||
|
||||
DOCKER_PAT_RE = re.compile(r"\bdckr_pat_[A-Za-z0-9_-]{27}\b")
|
||||
|
||||
|
||||
def ensure_files():
|
||||
ensure_output_files([CHECKED_FILE, RESULTS_FILE, *set(STATUS_FILES.values()), PLAIN_FILE])
|
||||
recover_status_transaction(CHECKED_FILE, STATUS_FILES)
|
||||
|
||||
|
||||
def candidate_key(username, token):
|
||||
return f"{username}:{token}" if username else token
|
||||
|
||||
|
||||
def parse_username_token(raw, raw_v2, finding):
|
||||
raw = raw or ""
|
||||
raw_v2 = raw_v2 or ""
|
||||
token = ""
|
||||
username = ""
|
||||
|
||||
if ":" in raw_v2:
|
||||
maybe_user, maybe_token = raw_v2.split(":", 1)
|
||||
if DOCKER_PAT_RE.fullmatch(maybe_token):
|
||||
username, token = maybe_user.strip(), maybe_token.strip()
|
||||
if not token:
|
||||
match = DOCKER_PAT_RE.search(raw_v2) or DOCKER_PAT_RE.search(raw)
|
||||
if match:
|
||||
token = match.group(0)
|
||||
|
||||
extra = finding.get("ExtraData") if isinstance(finding, dict) else {}
|
||||
analysis = finding.get("AnalysisInfo") if isinstance(finding, dict) else {}
|
||||
if isinstance(extra, dict):
|
||||
username = username or extra.get("hub_username") or ""
|
||||
if isinstance(analysis, dict):
|
||||
username = username or analysis.get("username") or ""
|
||||
return username, token
|
||||
|
||||
|
||||
def extract_candidates(input_file, plain_file):
|
||||
seen_plain = set()
|
||||
for item in iter_findings(input_file, ["Dockerhub"]):
|
||||
username, token = parse_username_token(item.get("raw"), item.get("raw_v2"), item.get("finding") or {})
|
||||
if not token:
|
||||
continue
|
||||
key = candidate_key(username, token)
|
||||
yield key, username, token, item["source"], item["finding"]
|
||||
|
||||
if keycheck_input_mode() == 'jsonl' and os.path.exists(plain_file):
|
||||
with open(plain_file, "r", encoding="utf-8") as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
username = ""
|
||||
token = ""
|
||||
if ":" in line:
|
||||
maybe_user, rest = line.split(":", 1)
|
||||
match = DOCKER_PAT_RE.search(rest)
|
||||
if match:
|
||||
username, token = maybe_user.strip(), match.group(0)
|
||||
else:
|
||||
match = DOCKER_PAT_RE.search(line)
|
||||
if match:
|
||||
token = match.group(0)
|
||||
if not token:
|
||||
continue
|
||||
key = candidate_key(username, token)
|
||||
if key not in seen_plain:
|
||||
seen_plain.add(key)
|
||||
yield key, username, token, f"{plain_file}:{line_num}", {}
|
||||
|
||||
|
||||
def decode_jwt_payload(jwt_token):
|
||||
try:
|
||||
payload = jwt_token.split(".")[1]
|
||||
payload += "=" * (-len(payload) % 4)
|
||||
return json.loads(base64.urlsafe_b64decode(payload.encode()).decode())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def check_token(username, token, proxy, timeout):
|
||||
if not username:
|
||||
return {"status": "NO_USERNAME", "message": "DockerHub PAT requires username/email for login check"}
|
||||
|
||||
url = "https://hub.docker.com/v2/users/login"
|
||||
payload = {"username": username, "password": token}
|
||||
try:
|
||||
response = requests.post(url, json=payload, proxies=proxy, timeout=timeout)
|
||||
except requests.RequestException as exc:
|
||||
return {"status": "NETWORK", "message": str(exc), "username": username}
|
||||
|
||||
message = request_error_message(response)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
hub_token = data.get("token", "")
|
||||
claims = decode_jwt_payload(hub_token) if hub_token else {}
|
||||
hub_claims = claims.get("https://hub.docker.com", {}) if isinstance(claims, dict) else {}
|
||||
return {
|
||||
"status": "VALID",
|
||||
"message": "login accepted",
|
||||
"username": username,
|
||||
"hub_username": hub_claims.get("username", username),
|
||||
"hub_email": hub_claims.get("email", ""),
|
||||
"scope": claims.get("scope", "") if isinstance(claims, dict) else "",
|
||||
}
|
||||
if response.status_code == 401:
|
||||
try:
|
||||
data = response.json()
|
||||
except ValueError:
|
||||
data = {}
|
||||
if data.get("login_2fa_token"):
|
||||
return {"status": "VALID_2FA", "message": "credentials accepted; 2FA required", "username": username}
|
||||
return {"status": "DEAD", "http_status": 401, "message": message, "username": username}
|
||||
if response.status_code == 429:
|
||||
return {"status": "RATE_LIMITED", "http_status": 429, "message": message, "username": username}
|
||||
if response.status_code >= 500:
|
||||
return {"status": "NETWORK", "http_status": response.status_code, "message": message, "username": username}
|
||||
return {"status": "UNKNOWN", "http_status": response.status_code, "message": message, "username": username}
|
||||
|
||||
|
||||
def write_result(key, result, source, finding):
|
||||
write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source, finding, "Dockerhub")
|
||||
extra = result.get("hub_username") or result.get("username") or source
|
||||
commit_status_transaction(
|
||||
CHECKED_FILE, STATUS_FILES, key, result["status"], result.get("message", ""), extra,
|
||||
)
|
||||
record_validation_result(SERVICE, key, result, source, finding, "Dockerhub")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="DockerHub PAT checker")
|
||||
parser.add_argument("--input", default=INPUT_FILE)
|
||||
parser.add_argument("--plain", default=PLAIN_FILE)
|
||||
parser.add_argument("--proxy-file", default=PROXY_FILE)
|
||||
parser.add_argument("--timeout", type=int, default=20)
|
||||
parser.add_argument("--max-keys", type=int, default=0)
|
||||
parser.add_argument("--retry-network", action="store_true")
|
||||
parser.add_argument("--retry-limited", action="store_true")
|
||||
parser.add_argument("--retry-unknown", action="store_true")
|
||||
parser.add_argument("--retry-no-username", action="store_true")
|
||||
parser.add_argument("--recheck-all", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
require_provider_authority(SERVICE)
|
||||
args = parse_args()
|
||||
ensure_files()
|
||||
proxy_cycler = load_proxies(args.proxy_file)
|
||||
checked = load_checked_statuses(CHECKED_FILE)
|
||||
known = load_known_keys(CHECKED_FILE, STATUS_FILES)
|
||||
retry_statuses = set()
|
||||
if args.retry_network:
|
||||
retry_statuses.add("NETWORK")
|
||||
if args.retry_limited:
|
||||
retry_statuses.add("RATE_LIMITED")
|
||||
if args.retry_unknown:
|
||||
retry_statuses.add("UNKNOWN")
|
||||
if args.retry_no_username:
|
||||
retry_statuses.add("NO_USERNAME")
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
for key, username, token, source, finding in extract_candidates(args.input, args.plain):
|
||||
if should_skip_key(key, checked, known, args, retry_statuses, service=SERVICE, source=source, finding=finding, detector="Dockerhub"):
|
||||
skipped += 1
|
||||
continue
|
||||
if args.max_keys and processed >= args.max_keys:
|
||||
break
|
||||
processed += 1
|
||||
print(f"\n[{processed}] DockerHub candidate {mask_secret(key)} from {source}")
|
||||
proxy = next(proxy_cycler) if proxy_cycler else None
|
||||
result = check_token(username, token, proxy, args.timeout)
|
||||
print(f" STATUS: {result['status']} | {str(result.get('message', ''))[:200]}")
|
||||
write_result(key, result, source, finding)
|
||||
known.add(key)
|
||||
checked[key] = result["status"]
|
||||
time.sleep(0.1)
|
||||
print(f"\nDone. Processed={processed}, skipped={skipped}, results={RESULTS_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user