Initial server source import

2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
@@ -0,0 +1,178 @@
+import sys
+
+sys.dont_write_bytecode = True
+
+import argparse
+import os
+import re
+import time
+
+import requests
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from keycheck_common import (
+    append_jsonl,
+    commit_status_transaction,
+    default_input_file,
+    default_proxy_file,
+    ensure_output_files,
+    iter_findings,
+    load_checked_statuses,
+    load_known_keys,
+    load_proxies,
+    mask_secret,
+    read_plain_keys,
+    recover_status_transaction,
+    request_error_message,
+    record_validation_result,
+    require_provider_authority,
+    service_output_dir,
+    should_skip_key,
+    write_keycheck_event,
+)
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = os.path.dirname(SCRIPT_DIR)
+SERVICE = "gitlab"
+OUTPUT_DIR = os.getenv("KEYCHECK_OUTPUT_DIR") or service_output_dir(SERVICE)
+
+INPUT_FILE = os.getenv("KEYCHECK_INPUT_FILE") or default_input_file()
+PROXY_FILE = os.getenv("KEYCHECK_PROXY_FILE") or default_proxy_file()
+PLAIN_FILE = os.path.join(OUTPUT_DIR, "gitlab.txt")
+CHECKED_FILE = os.path.join(OUTPUT_DIR, "gitlabChecked.txt")
+RESULTS_FILE = os.path.join(OUTPUT_DIR, "gitlabResults.jsonl")
+
+STATUS_FILES = {
+    "VALID": os.path.join(OUTPUT_DIR, "gitlabAlive.txt"),
+    "DEAD": os.path.join(OUTPUT_DIR, "gitlabDead.txt"),
+    "RESTRICTED": os.path.join(OUTPUT_DIR, "gitlabRestricted.txt"),
+    "RATE_LIMITED": os.path.join(OUTPUT_DIR, "gitlabRateLimited.txt"),
+    "NETWORK": os.path.join(OUTPUT_DIR, "gitlabNetwork.txt"),
+    "UNKNOWN": os.path.join(OUTPUT_DIR, "gitlabUnknown.txt"),
+}
+
+GITLAB_TOKEN_RE = re.compile(r"\b(?:glpat|gloas|glcbt|glimt|glrt|glft|glsoat)-[A-Za-z0-9_\-=]{20,}\b")
+
+
+def ensure_files():
+    ensure_output_files([CHECKED_FILE, RESULTS_FILE, *STATUS_FILES.values(), PLAIN_FILE])
+    recover_status_transaction(CHECKED_FILE, STATUS_FILES)
+
+
+def extract_candidates(input_file, plain_files):
+    seen_plain = set()
+    for item in iter_findings(input_file, ["Gitlab"]):
+        text = "\n".join(str(value or "") for value in [item.get("raw"), item.get("raw_v2")])
+        for match in GITLAB_TOKEN_RE.findall(text):
+            yield match, item["source"], item["finding"]
+
+    for item in read_plain_keys(plain_files, GITLAB_TOKEN_RE):
+        key = item["key"]
+        if key not in seen_plain:
+            seen_plain.add(key)
+            yield key, item["source"], {}
+
+
+def check_token(token, base_url, proxy, timeout):
+    base_url = base_url.rstrip("/")
+    url = f"{base_url}/api/v4/user"
+    headers = {"Authorization": f"Bearer {token}", "User-Agent": "local-keycheck-gitlab"}
+    try:
+        response = requests.get(url, headers=headers, proxies=proxy, timeout=timeout)
+    except requests.RequestException as exc:
+        return {"status": "NETWORK", "message": str(exc), "base_url": base_url}
+
+    message = request_error_message(response)
+    retry_after = response.headers.get("Retry-After", "")
+    rate_remaining = response.headers.get("RateLimit-Remaining") or response.headers.get("X-RateLimit-Remaining") or ""
+
+    if response.status_code == 200:
+        payload = response.json()
+        return {
+            "status": "VALID",
+            "message": "token accepted",
+            "username": payload.get("username"),
+            "name": payload.get("name"),
+            "user_id": payload.get("id"),
+            "base_url": base_url,
+            "rate_remaining": rate_remaining,
+        }
+    if response.status_code == 401:
+        return {"status": "DEAD", "http_status": 401, "message": message, "base_url": base_url}
+    if response.status_code == 403:
+        # TruffleHog treats 403 as a live token with insufficient scope or blocked account.
+        return {"status": "RESTRICTED", "http_status": 403, "message": message, "base_url": base_url}
+    if response.status_code == 429:
+        return {"status": "RATE_LIMITED", "http_status": 429, "message": message, "base_url": base_url, "retry_after": retry_after}
+    if response.status_code >= 500:
+        return {"status": "NETWORK", "http_status": response.status_code, "message": message, "base_url": base_url}
+    return {"status": "UNKNOWN", "http_status": response.status_code, "message": message, "base_url": base_url}
+
+
+def write_result(key, result, source, finding):
+    write_keycheck_event(SERVICE, RESULTS_FILE, key, result, source, finding)
+    extra = result.get("username") or result.get("user_id") or source
+    commit_status_transaction(
+        CHECKED_FILE, STATUS_FILES, key, result["status"], result.get("message", ""), extra,
+    )
+    record_validation_result(SERVICE, key, result, source, finding)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="GitLab token checker")
+    parser.add_argument("--input", default=INPUT_FILE)
+    parser.add_argument("--plain", action="append", default=[])
+    parser.add_argument("--proxy-file", default=PROXY_FILE)
+    parser.add_argument("--base-url", default="https://gitlab.com")
+    parser.add_argument("--timeout", type=int, default=20)
+    parser.add_argument("--max-keys", type=int, default=0)
+    parser.add_argument("--retry-network", action="store_true")
+    parser.add_argument("--retry-limited", action="store_true")
+    parser.add_argument("--retry-unknown", action="store_true")
+    parser.add_argument("--retry-restricted", action="store_true")
+    parser.add_argument("--recheck-all", action="store_true")
+    return parser.parse_args()
+
+
+def main():
+    require_provider_authority(SERVICE)
+    args = parse_args()
+    ensure_files()
+    proxy_cycler = load_proxies(args.proxy_file)
+    checked = load_checked_statuses(CHECKED_FILE)
+    known = load_known_keys(CHECKED_FILE, STATUS_FILES)
+    retry_statuses = set()
+    if args.retry_network:
+        retry_statuses.add("NETWORK")
+    if args.retry_limited:
+        retry_statuses.add("RATE_LIMITED")
+    if args.retry_unknown:
+        retry_statuses.add("UNKNOWN")
+    if args.retry_restricted:
+        retry_statuses.add("RESTRICTED")
+
+    plain_files = args.plain or [PLAIN_FILE]
+    processed = 0
+    skipped = 0
+    for key, source, finding in extract_candidates(args.input, plain_files):
+        if should_skip_key(key, checked, known, args, retry_statuses, service=SERVICE, source=source, finding=finding):
+            skipped += 1
+            continue
+        if args.max_keys and processed >= args.max_keys:
+            break
+        processed += 1
+        print(f"\n[{processed}] GitLab candidate {mask_secret(key)} from {source}")
+        proxy = next(proxy_cycler) if proxy_cycler else None
+        result = check_token(key, args.base_url, proxy, args.timeout)
+        print(f"  STATUS: {result['status']} | {str(result.get('message', ''))[:200]}")
+        write_result(key, result, source, finding)
+        known.add(key)
+        checked[key] = result["status"]
+        time.sleep(0.1)
+    print(f"\nDone. Processed={processed}, skipped={skipped}, results={RESULTS_FILE}")
+
+
+if __name__ == "__main__":
+    main()