#!/bin/bash
set -Eeuo pipefail
umask 077

MODE="${1:-}"
STAGE="${2:-}"
if [[ "$MODE" != plan && "$MODE" != apply ]]; then
    echo 'usage: deploy.sh plan|apply STAGE' >&2
    exit 64
fi
if [[ ! "$STAGE" =~ ^/var/lib/truf-deploy/stage/capacity50\.[A-Za-z0-9]+$ ]] || [[ ! -d "$STAGE" ]]; then
    echo 'invalid deployment stage' >&2
    exit 64
fi

readonly RELEASE_ID='capacity50-20260930'
readonly EXPECTED_IMAGE='sha256:46f1cf1b92d1a7d93d06f690309d8c7eca45f64dc45ca310861c70fb419bb035'
readonly EXPECTED_CONFIG_SHA256='12bd9a60cc56c3d6cbad18435523e8229b0cd8fdccc2d73922ea7e580ce7441c'
readonly CANDIDATE_TAG="truf-local:runtime-${RELEASE_ID}"
readonly ROLLBACK_TAG="truf-local:runtime-pre-${RELEASE_ID}"
readonly ACTIVE_CONFIG='/etc/truf/runtime/config.yaml'
readonly ACTIVE_SECRETS='/etc/truf/runtime/secrets.yaml'
readonly SOURCE_ROOT='/opt/truf'
readonly HISTORY="/var/lib/truf-deploy/history/${RELEASE_ID}"
readonly TEST_USER='operator-trace-windows-20260925'
readonly TEST_USER_ORIGINAL_CAP='2'
readonly APP_FILES=(
    capacity_model.py
    scanner_db.py
    worker_assignment.py
    worker_api.py
    jsonl_projector.py
    runtime_document.py
    lifecycle_authority.py
    config.linux.yaml
)
readonly COMPOSE=(
    docker compose
    --project-name truf-docker
    --project-directory /opt/truf
    --env-file /etc/truf-edge/edge.env
    --file /opt/truf/compose.yaml
    --file /opt/truf/compose.shared-host.yaml
)

PHASE='preflight'
MUTATED=0
PHASE_A_HEALTHY=0
SOURCE_INSTALLED=0
USER_CAP_CHANGED=0
DEPLOY_SUCCEEDED=0
RESUME=0

log() {
    printf '[%s] %s\n' "$RELEASE_ID" "$*"
}

runtime_id() {
    "${COMPOSE[@]}" ps --quiet runtime
}

psql() {
    local container
    container="$(runtime_id)"
    [[ -n "$container" ]] || return 1
    docker exec "$container" /usr/lib/postgresql/16/bin/psql \
        -h /run/truf-postgres -U truf -d truf -v ON_ERROR_STOP=1 -At "$@"
}

current_image() {
    docker image inspect --format '{{.Id}}' truf-local:runtime
}

config_sha256() {
    sha256sum "$ACTIVE_CONFIG" | cut -d' ' -f1
}

require_baseline() {
    [[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
        echo 'runtime image identity changed' >&2
        return 1
    }
    [[ "$(config_sha256)" == "$EXPECTED_CONFIG_SHA256" ]] || {
        echo 'active config identity changed' >&2
        return 1
    }
    local state
    state="$(psql -F '|' -c \
        "SELECT revision, discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
    if ((RESUME)); then
        [[ "$state" =~ ^[0-9]+\|0\|0\|(normal|drained)$ ]] || {
            echo "resumed runtime control is neither open nor drained: $state" >&2
            return 1
        }
    elif [[ ! "$state" =~ ^[0-9]+\|0\|0\|normal$ ]]; then
        echo "runtime control is not open: $state" >&2
        return 1
    fi
    local debt
    debt="$(psql -F '|' -c \
        "SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
    [[ "$debt" == '0|0|0|0|0|0|0|0|0' ]] || {
        echo "pipeline is not reconciled: $debt" >&2
        return 1
    }
    [[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}' AND disabled_at IS NULL;")" == "$TEST_USER_ORIGINAL_CAP" ]] || {
        echo 'temporary validation user identity changed' >&2
        return 1
    }
}

edge_value() {
    local name="$1"
    sed -n "s/^${name}=//p" /etc/truf-edge/edge.env
}

admin_material() {
    local marker host page token revision
    marker="$(edge_value TRUF_ADMIN_EDGE_MARKER)"
    host="$(edge_value TRUF_EDGE_HOST)"
    [[ "$marker" =~ ^[a-f0-9]{64}$ ]] || return 1
    [[ "$host" =~ ^[A-Za-z0-9.-]+$ ]] || return 1
    page="$(curl --fail --silent --show-error --max-time 20 \
        --header "X-Truf-Admin-Edge: ${marker}" \
        --header 'X-Truf-Admin-Operator: deploy-runtime' \
        http://127.0.0.1:8766/admin-internal/)"
    token="$(python3 -c \
        'import re,sys; values=set(re.findall(r"name=\"csrf_token\" value=\"([^\"]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
        <<<"$page")"
    revision="$(python3 -c \
        'import re,sys; values=set(re.findall(r"name=\"expected_revision\" value=\"([0-9]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
        <<<"$page")"
    [[ "$token" =~ ^[A-Za-z0-9_-]{32,128}$ && "$revision" =~ ^[0-9]+$ ]] || return 1
    printf '%s|%s|%s|%s\n' "$marker" "$host" "$token" "$revision"
}

admin_post() {
    local route="$1"
    shift
    local material marker host token revision operation
    material="$(admin_material)"
    IFS='|' read -r marker host token revision <<<"$material"
    operation="$(cat /proc/sys/kernel/random/uuid)"
    local arguments=(
        --fail --silent --show-error --max-time 30
        --request POST
        --header "X-Truf-Admin-Edge: ${marker}"
        --header 'X-Truf-Admin-Operator: deploy-runtime'
        --header "Origin: https://${host}"
        --header 'Content-Type: application/x-www-form-urlencoded'
        --data-urlencode "csrf_token=${token}"
        --data-urlencode "operation_id=${operation}"
    )
    if [[ "$route" == dispatch/* || "$route" == search/discovery/* ]]; then
        arguments+=(--data-urlencode "expected_revision=${revision}")
    fi
    while (($#)); do
        arguments+=(--data-urlencode "$1")
        shift
    done
    curl "${arguments[@]}" "http://127.0.0.1:8766/admin-internal/${route}" >/dev/null
}

wait_for_drain() {
    local deadline=$((SECONDS + 600)) state debt
    while ((SECONDS < deadline)); do
        state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")"
        debt="$(psql -F '|' -c \
            "SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
        if [[ "$state" == drained && "$debt" == '0|0|0|0|0|0|0|0|0' ]]; then
            return 0
        fi
        sleep 2
    done
    echo 'runtime did not drain within 600 seconds' >&2
    return 1
}

quiesce_pipeline_workers() {
    local source deadline active container
    for source in result-ingester jsonl-projector; do
        admin_post supervisor/sources/stop "source_id=${source}"
    done
    container="$(runtime_id)"
    [[ -n "$container" ]] || return 1
    docker exec --interactive "$container" /usr/local/bin/python3 -I -S -B - \
        <"$STAGE/release_stopped_pipeline_leases.py"
    deadline=$((SECONDS + 120))
    while ((SECONDS < deadline)); do
        active="$(psql -c \
            "SELECT count(*) FROM pipeline_leases WHERE state NOT IN ('released','failed');")"
        if [[ "$active" == 0 ]]; then
            return 0
        fi
        sleep 2
    done
    echo 'pipeline worker leases did not release within 120 seconds' >&2
    return 1
}

install_config() {
    local source="$1" temporary
    temporary="/etc/truf/runtime/.config.yaml.${RELEASE_ID}.tmp"
    install -o root -g root -m 0600 "$source" "$temporary"
    chown 10001:10001 "$temporary"
    mv -f "$temporary" "$ACTIVE_CONFIG"
}

stop_stack() {
    "${COMPOSE[@]}" stop --timeout 30 edge
    "${COMPOSE[@]}" stop --timeout 600 runtime
    local container state
    container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
    state="$(docker inspect --format '{{.State.Status}}|{{.State.ExitCode}}|{{.State.OOMKilled}}' "$container")"
    [[ "$state" == 'exited|0|false' ]] || {
        echo "runtime stop was not clean: $state" >&2
        return 1
    }
}

wait_runtime_health() {
    local deadline=$((SECONDS + 420)) container state status
    while ((SECONDS < deadline)); do
        container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
        if [[ -n "$container" ]]; then
            state="$(docker inspect --format '{{.State.Status}}' "$container")"
            [[ "$state" != exited && "$state" != dead ]] || return 1
            status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")"
            if [[ "$status" == healthy ]] && docker exec "$container" \
                /usr/local/bin/python3 -I -S -B /opt/truf/app/container_runtime.py \
                health --config /data/config/config.yaml --require-worker-api >/dev/null; then
                return 0
            fi
            [[ "$status" != unhealthy ]] || return 1
        fi
        sleep 3
    done
    echo 'runtime health timed out' >&2
    return 1
}

start_stack() {
    "${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate runtime
    wait_runtime_health
    "${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate edge
    sleep 3
    local edge_container edge_state host public_code
    edge_container="$("${COMPOSE[@]}" ps --quiet edge)"
    edge_state="$(docker inspect --format '{{.State.Status}}|{{.State.Running}}|{{.State.OOMKilled}}' "$edge_container")"
    [[ "$edge_state" == 'running|true|false' ]] || return 1
    host="$(edge_value TRUF_EDGE_HOST)"
    public_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
        --max-time 20 "https://${host}/")"
    [[ "$public_code" == 401 || "$public_code" == 404 ]] || {
        echo "unexpected public edge response: $public_code" >&2
        return 1
    }
}

validate_candidate_config() {
    local path="$1"
    docker run --rm --network none --read-only --user 10001:10001 \
        --cap-drop ALL --security-opt no-new-privileges:true \
        --tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \
        --volume "$path:/data/config/config.yaml:ro" \
        --volume "$ACTIVE_SECRETS:/data/config/secrets.yaml:ro" \
        --volume /etc/truf/worker-packages:/data/worker-packages:ro \
        --entrypoint /usr/local/bin/python3 "$CANDIDATE_TAG" -I -S -B -c \
        "import sys,sysconfig;sys.path.append(sysconfig.get_paths()['purelib']);sys.path.insert(0,'/opt/truf/app');from runtime_document_io import validate_managed_runtime_files;print(validate_managed_runtime_files('/data/config/config.yaml').config_sha256)" \
        >/dev/null
}

install_sources() {
    local name destination temporary
    for name in "${APP_FILES[@]}"; do
        destination="${SOURCE_ROOT}/app/${name}"
        temporary="${destination}.${RELEASE_ID}.tmp"
        install -o root -g root -m 0644 "$STAGE/payload/app/$name" "$temporary"
        mv -f "$temporary" "$destination"
    done
    SOURCE_INSTALLED=1
}

restore_sources() {
    local name
    for name in "${APP_FILES[@]}"; do
        if [[ -f "$HISTORY/source/$name" ]]; then
            install -o root -g root -m 0644 "$HISTORY/source/$name" "${SOURCE_ROOT}/app/$name"
        else
            rm -f "${SOURCE_ROOT}/app/$name"
        fi
    done
}

restore_user_cap() {
    if ((USER_CAP_CHANGED)); then
        admin_post users/cap "user_key=${TEST_USER}" "active_assignment_cap=${TEST_USER_ORIGINAL_CAP}" || true
        USER_CAP_CHANGED=0
    fi
}

cancel_drain() {
    local state
    state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;" 2>/dev/null || true)"
    if [[ "$state" == draining || "$state" == drained ]]; then
        admin_post dispatch/drain/cancel || return 1
    fi
}

rollback() {
    set +e
    log "rollback from phase ${PHASE}"
    restore_user_cap
    stop_stack
    if ((PHASE_A_HEALTHY)); then
        docker image tag "$CANDIDATE_TAG" truf-local:runtime
        install_config "$HISTORY/config.conservative.yaml"
    else
        docker image tag "$EXPECTED_IMAGE" truf-local:runtime
        install_config "$HISTORY/config.original.yaml"
    fi
    ((SOURCE_INSTALLED)) && restore_sources
    if start_stack; then
        cancel_drain
        log 'rollback restored a healthy runtime'
    else
        log 'rollback could not prove health; runtime remains contained' >&2
    fi
    set -e
}

on_exit() {
    local code=$?
    trap - EXIT ERR INT TERM
    if ((code != 0 && MUTATED && !DEPLOY_SUCCEEDED)); then
        rollback
    fi
    exit "$code"
}
trap on_exit EXIT

exec 9>/run/lock/truf-runtime-deploy.lock
flock -n 9 || {
    echo 'another runtime deployment is active' >&2
    exit 1
}

if [[ "$MODE" == apply && -d "$HISTORY" ]] \
    && docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1 \
    && [[ "$(docker image inspect --format '{{.Id}}' "$ROLLBACK_TAG" 2>/dev/null || true)" == "$EXPECTED_IMAGE" ]]; then
    RESUME=1
fi
require_baseline
available_kb="$(df -Pk /var/lib/docker | awk 'NR==2 {print $4}')"
[[ "$available_kb" =~ ^[0-9]+$ && "$available_kb" -ge 786432 ]] || {
    echo 'less than 768 MiB is available for the derived image' >&2
    exit 1
}
log "plan image=${EXPECTED_IMAGE#sha256:} config=${EXPECTED_CONFIG_SHA256} free_kib=${available_kb}"
if [[ "$MODE" == plan ]]; then
    log 'plan passed; no runtime state changed'
    exit 0
fi

if ((RESUME)); then
    log 'resuming a verified pre-cutover release'
    [[ "$(sha256sum "$HISTORY/config.original.yaml" | cut -d' ' -f1)" == "$EXPECTED_CONFIG_SHA256" ]] || exit 1
    [[ -f "$HISTORY/config.conservative.yaml" && -f "$HISTORY/config.final.yaml" ]] || exit 1
    validate_candidate_config "$HISTORY/config.conservative.yaml"
    validate_candidate_config "$HISTORY/config.final.yaml"
    if [[ "$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")" == normal ]]; then
        admin_post dispatch/drain/start
        MUTATED=1
        wait_for_drain
    else
        MUTATED=1
    fi
else
    install -d -o root -g root -m 0700 /var/lib/truf-deploy /var/lib/truf-deploy/history
    if [[ -e "$HISTORY" ]]; then
        echo 'release history already exists' >&2
        exit 1
    fi
    install -d -o root -g root -m 0700 "$HISTORY" "$HISTORY/source"
    install -o root -g root -m 0600 "$ACTIVE_CONFIG" "$HISTORY/config.original.yaml"
    for name in "${APP_FILES[@]}"; do
        if [[ -f "${SOURCE_ROOT}/app/$name" ]]; then
            install -o root -g root -m 0600 "${SOURCE_ROOT}/app/$name" "$HISTORY/source/$name"
        fi
    done

    python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
        --output "$HISTORY/config.conservative.yaml" --mode conservative \
        --expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
    python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
        --output "$HISTORY/config.final.yaml" --mode final \
        --expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
    chown 10001:10001 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
    chmod 0600 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"

    if docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1; then
        echo 'candidate image tag already exists' >&2
        exit 1
    fi
    PHASE='candidate-build'
    docker build --network none --build-arg BASE_IMAGE=truf-local:runtime \
        --file "$STAGE/Dockerfile" --tag "$CANDIDATE_TAG" "$STAGE"
    [[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
        echo 'runtime tag changed during candidate build' >&2
        exit 1
    }
    validate_candidate_config "$HISTORY/config.conservative.yaml"
    validate_candidate_config "$HISTORY/config.final.yaml"
    docker image tag "$EXPECTED_IMAGE" "$ROLLBACK_TAG"

    PHASE='drain'
    admin_post dispatch/drain/start
    MUTATED=1
    wait_for_drain
fi

PHASE='conservative-cutover'
quiesce_pipeline_workers
stop_stack
install_config "$HISTORY/config.conservative.yaml"
docker image tag "$CANDIDATE_TAG" truf-local:runtime
start_stack
schema_state="$(psql -F '|' -c \
    "SELECT (SELECT count(*) FROM information_schema.columns WHERE table_name='result_reservations' AND column_name='reserved_bundle_bytes'), (SELECT count(*) FROM runtime_schema_migrations WHERE version='20260930_33_remote_assignment_capacity');")"
[[ "$schema_state" == '1|1' ]] || {
    echo "capacity migration was not applied: $schema_state" >&2
    exit 1
}
PHASE_A_HEALTHY=1

PHASE='capacity50-cutover'
quiesce_pipeline_workers
stop_stack
install_config "$HISTORY/config.final.yaml"
start_stack

final_values="$(psql -F '|' -c \
    "SELECT bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
[[ "$final_values" == '0|0|0|0|0|0|0|0' ]] || {
    echo "post-deploy capacity is not reconciled: $final_values" >&2
    exit 1
}

PHASE='temporary-user-cap-validation'
admin_post users/cap "user_key=${TEST_USER}" 'active_assignment_cap=50'
USER_CAP_CHANGED=1
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == 50 ]] || exit 1
restore_user_cap
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == "$TEST_USER_ORIGINAL_CAP" ]] || exit 1

PHASE='source-install'
install_sources
for name in "${APP_FILES[@]}"; do
    cmp -s "$STAGE/payload/app/$name" "${SOURCE_ROOT}/app/$name" || exit 1
done

PHASE='resume'
cancel_drain
post_control="$(psql -F '|' -c \
    "SELECT discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
[[ "$post_control" == '0|0|normal' ]] || {
    echo "runtime control did not resume: $post_control" >&2
    exit 1
}

cat >"$HISTORY/result.txt" <<EOF
release=${RELEASE_ID}
runtime_image=$(current_image)
config_sha256=$(config_sha256)
schema=${schema_state}
capacity=${final_values}
control=${post_control}
EOF
chmod 0600 "$HISTORY/result.txt"
DEPLOY_SUCCEEDED=1
log "applied image=$(current_image) config=$(config_sha256)"