Files
2026-09-30 20:30:56 +03:00

482 lines
18 KiB
Bash

#!/bin/bash
set -Eeuo pipefail
umask 077
MODE="${1:-}"
STAGE="${2:-}"
if [[ "$MODE" != plan && "$MODE" != apply ]]; then
echo 'usage: deploy.sh plan|apply STAGE' >&2
exit 64
fi
if [[ ! "$STAGE" =~ ^/var/lib/truf-deploy/stage/capacity50\.[A-Za-z0-9]+$ ]] || [[ ! -d "$STAGE" ]]; then
echo 'invalid deployment stage' >&2
exit 64
fi
readonly RELEASE_ID='capacity50-20260930'
readonly EXPECTED_IMAGE='sha256:46f1cf1b92d1a7d93d06f690309d8c7eca45f64dc45ca310861c70fb419bb035'
readonly EXPECTED_CONFIG_SHA256='12bd9a60cc56c3d6cbad18435523e8229b0cd8fdccc2d73922ea7e580ce7441c'
readonly CANDIDATE_TAG="truf-local:runtime-${RELEASE_ID}"
readonly ROLLBACK_TAG="truf-local:runtime-pre-${RELEASE_ID}"
readonly ACTIVE_CONFIG='/etc/truf/runtime/config.yaml'
readonly ACTIVE_SECRETS='/etc/truf/runtime/secrets.yaml'
readonly SOURCE_ROOT='/opt/truf'
readonly HISTORY="/var/lib/truf-deploy/history/${RELEASE_ID}"
readonly TEST_USER='operator-trace-windows-20260925'
readonly TEST_USER_ORIGINAL_CAP='2'
readonly APP_FILES=(
capacity_model.py
scanner_db.py
worker_assignment.py
worker_api.py
jsonl_projector.py
runtime_document.py
lifecycle_authority.py
config.linux.yaml
)
readonly COMPOSE=(
docker compose
--project-name truf-docker
--project-directory /opt/truf
--env-file /etc/truf-edge/edge.env
--file /opt/truf/compose.yaml
--file /opt/truf/compose.shared-host.yaml
)
PHASE='preflight'
MUTATED=0
PHASE_A_HEALTHY=0
SOURCE_INSTALLED=0
USER_CAP_CHANGED=0
DEPLOY_SUCCEEDED=0
RESUME=0
log() {
printf '[%s] %s\n' "$RELEASE_ID" "$*"
}
runtime_id() {
"${COMPOSE[@]}" ps --quiet runtime
}
psql() {
local container
container="$(runtime_id)"
[[ -n "$container" ]] || return 1
docker exec "$container" /usr/lib/postgresql/16/bin/psql \
-h /run/truf-postgres -U truf -d truf -v ON_ERROR_STOP=1 -At "$@"
}
current_image() {
docker image inspect --format '{{.Id}}' truf-local:runtime
}
config_sha256() {
sha256sum "$ACTIVE_CONFIG" | cut -d' ' -f1
}
require_baseline() {
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
echo 'runtime image identity changed' >&2
return 1
}
[[ "$(config_sha256)" == "$EXPECTED_CONFIG_SHA256" ]] || {
echo 'active config identity changed' >&2
return 1
}
local state
state="$(psql -F '|' -c \
"SELECT revision, discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
if ((RESUME)); then
[[ "$state" =~ ^[0-9]+\|0\|0\|(normal|drained)$ ]] || {
echo "resumed runtime control is neither open nor drained: $state" >&2
return 1
}
elif [[ ! "$state" =~ ^[0-9]+\|0\|0\|normal$ ]]; then
echo "runtime control is not open: $state" >&2
return 1
fi
local debt
debt="$(psql -F '|' -c \
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
[[ "$debt" == '0|0|0|0|0|0|0|0|0' ]] || {
echo "pipeline is not reconciled: $debt" >&2
return 1
}
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}' AND disabled_at IS NULL;")" == "$TEST_USER_ORIGINAL_CAP" ]] || {
echo 'temporary validation user identity changed' >&2
return 1
}
}
edge_value() {
local name="$1"
sed -n "s/^${name}=//p" /etc/truf-edge/edge.env
}
admin_material() {
local marker host page token revision
marker="$(edge_value TRUF_ADMIN_EDGE_MARKER)"
host="$(edge_value TRUF_EDGE_HOST)"
[[ "$marker" =~ ^[a-f0-9]{64}$ ]] || return 1
[[ "$host" =~ ^[A-Za-z0-9.-]+$ ]] || return 1
page="$(curl --fail --silent --show-error --max-time 20 \
--header "X-Truf-Admin-Edge: ${marker}" \
--header 'X-Truf-Admin-Operator: deploy-runtime' \
http://127.0.0.1:8766/admin-internal/)"
token="$(python3 -c \
'import re,sys; values=set(re.findall(r"name=\"csrf_token\" value=\"([^\"]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
<<<"$page")"
revision="$(python3 -c \
'import re,sys; values=set(re.findall(r"name=\"expected_revision\" value=\"([0-9]+)\"",sys.stdin.read())); print(values.pop() if len(values)==1 else "")' \
<<<"$page")"
[[ "$token" =~ ^[A-Za-z0-9_-]{32,128}$ && "$revision" =~ ^[0-9]+$ ]] || return 1
printf '%s|%s|%s|%s\n' "$marker" "$host" "$token" "$revision"
}
admin_post() {
local route="$1"
shift
local material marker host token revision operation
material="$(admin_material)"
IFS='|' read -r marker host token revision <<<"$material"
operation="$(cat /proc/sys/kernel/random/uuid)"
local arguments=(
--fail --silent --show-error --max-time 30
--request POST
--header "X-Truf-Admin-Edge: ${marker}"
--header 'X-Truf-Admin-Operator: deploy-runtime'
--header "Origin: https://${host}"
--header 'Content-Type: application/x-www-form-urlencoded'
--data-urlencode "csrf_token=${token}"
--data-urlencode "operation_id=${operation}"
)
if [[ "$route" == dispatch/* || "$route" == search/discovery/* ]]; then
arguments+=(--data-urlencode "expected_revision=${revision}")
fi
while (($#)); do
arguments+=(--data-urlencode "$1")
shift
done
curl "${arguments[@]}" "http://127.0.0.1:8766/admin-internal/${route}" >/dev/null
}
wait_for_drain() {
local deadline=$((SECONDS + 600)) state debt
while ((SECONDS < deadline)); do
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")"
debt="$(psql -F '|' -c \
"SELECT (SELECT count(*) FROM result_reservations WHERE assignment_kind='remote' AND remote_resolved_at IS NULL), bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
if [[ "$state" == drained && "$debt" == '0|0|0|0|0|0|0|0|0' ]]; then
return 0
fi
sleep 2
done
echo 'runtime did not drain within 600 seconds' >&2
return 1
}
quiesce_pipeline_workers() {
local source deadline active container
for source in result-ingester jsonl-projector; do
admin_post supervisor/sources/stop "source_id=${source}"
done
container="$(runtime_id)"
[[ -n "$container" ]] || return 1
docker exec --interactive "$container" /usr/local/bin/python3 -I -S -B - \
<"$STAGE/release_stopped_pipeline_leases.py"
deadline=$((SECONDS + 120))
while ((SECONDS < deadline)); do
active="$(psql -c \
"SELECT count(*) FROM pipeline_leases WHERE state NOT IN ('released','failed');")"
if [[ "$active" == 0 ]]; then
return 0
fi
sleep 2
done
echo 'pipeline worker leases did not release within 120 seconds' >&2
return 1
}
install_config() {
local source="$1" temporary
temporary="/etc/truf/runtime/.config.yaml.${RELEASE_ID}.tmp"
install -o root -g root -m 0600 "$source" "$temporary"
chown 10001:10001 "$temporary"
mv -f "$temporary" "$ACTIVE_CONFIG"
}
stop_stack() {
"${COMPOSE[@]}" stop --timeout 30 edge
"${COMPOSE[@]}" stop --timeout 600 runtime
local container state
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
state="$(docker inspect --format '{{.State.Status}}|{{.State.ExitCode}}|{{.State.OOMKilled}}' "$container")"
[[ "$state" == 'exited|0|false' ]] || {
echo "runtime stop was not clean: $state" >&2
return 1
}
}
wait_runtime_health() {
local deadline=$((SECONDS + 420)) container state status
while ((SECONDS < deadline)); do
container="$("${COMPOSE[@]}" ps --all --quiet runtime)"
if [[ -n "$container" ]]; then
state="$(docker inspect --format '{{.State.Status}}' "$container")"
[[ "$state" != exited && "$state" != dead ]] || return 1
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")"
if [[ "$status" == healthy ]] && docker exec "$container" \
/usr/local/bin/python3 -I -S -B /opt/truf/app/container_runtime.py \
health --config /data/config/config.yaml --require-worker-api >/dev/null; then
return 0
fi
[[ "$status" != unhealthy ]] || return 1
fi
sleep 3
done
echo 'runtime health timed out' >&2
return 1
}
start_stack() {
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate runtime
wait_runtime_health
"${COMPOSE[@]}" up --detach --no-deps --no-build --pull never --force-recreate edge
sleep 3
local edge_container edge_state host public_code
edge_container="$("${COMPOSE[@]}" ps --quiet edge)"
edge_state="$(docker inspect --format '{{.State.Status}}|{{.State.Running}}|{{.State.OOMKilled}}' "$edge_container")"
[[ "$edge_state" == 'running|true|false' ]] || return 1
host="$(edge_value TRUF_EDGE_HOST)"
public_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
--max-time 20 "https://${host}/")"
[[ "$public_code" == 401 || "$public_code" == 404 ]] || {
echo "unexpected public edge response: $public_code" >&2
return 1
}
}
validate_candidate_config() {
local path="$1"
docker run --rm --network none --read-only --user 10001:10001 \
--cap-drop ALL --security-opt no-new-privileges:true \
--tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \
--volume "$path:/data/config/config.yaml:ro" \
--volume "$ACTIVE_SECRETS:/data/config/secrets.yaml:ro" \
--volume /etc/truf/worker-packages:/data/worker-packages:ro \
--entrypoint /usr/local/bin/python3 "$CANDIDATE_TAG" -I -S -B -c \
"import sys,sysconfig;sys.path.append(sysconfig.get_paths()['purelib']);sys.path.insert(0,'/opt/truf/app');from runtime_document_io import validate_managed_runtime_files;print(validate_managed_runtime_files('/data/config/config.yaml').config_sha256)" \
>/dev/null
}
install_sources() {
local name destination temporary
for name in "${APP_FILES[@]}"; do
destination="${SOURCE_ROOT}/app/${name}"
temporary="${destination}.${RELEASE_ID}.tmp"
install -o root -g root -m 0644 "$STAGE/payload/app/$name" "$temporary"
mv -f "$temporary" "$destination"
done
SOURCE_INSTALLED=1
}
restore_sources() {
local name
for name in "${APP_FILES[@]}"; do
if [[ -f "$HISTORY/source/$name" ]]; then
install -o root -g root -m 0644 "$HISTORY/source/$name" "${SOURCE_ROOT}/app/$name"
else
rm -f "${SOURCE_ROOT}/app/$name"
fi
done
}
restore_user_cap() {
if ((USER_CAP_CHANGED)); then
admin_post users/cap "user_key=${TEST_USER}" "active_assignment_cap=${TEST_USER_ORIGINAL_CAP}" || true
USER_CAP_CHANGED=0
fi
}
cancel_drain() {
local state
state="$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;" 2>/dev/null || true)"
if [[ "$state" == draining || "$state" == drained ]]; then
admin_post dispatch/drain/cancel || return 1
fi
}
rollback() {
set +e
log "rollback from phase ${PHASE}"
restore_user_cap
stop_stack
if ((PHASE_A_HEALTHY)); then
docker image tag "$CANDIDATE_TAG" truf-local:runtime
install_config "$HISTORY/config.conservative.yaml"
else
docker image tag "$EXPECTED_IMAGE" truf-local:runtime
install_config "$HISTORY/config.original.yaml"
fi
((SOURCE_INSTALLED)) && restore_sources
if start_stack; then
cancel_drain
log 'rollback restored a healthy runtime'
else
log 'rollback could not prove health; runtime remains contained' >&2
fi
set -e
}
on_exit() {
local code=$?
trap - EXIT ERR INT TERM
if ((code != 0 && MUTATED && !DEPLOY_SUCCEEDED)); then
rollback
fi
exit "$code"
}
trap on_exit EXIT
exec 9>/run/lock/truf-runtime-deploy.lock
flock -n 9 || {
echo 'another runtime deployment is active' >&2
exit 1
}
if [[ "$MODE" == apply && -d "$HISTORY" ]] \
&& docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1 \
&& [[ "$(docker image inspect --format '{{.Id}}' "$ROLLBACK_TAG" 2>/dev/null || true)" == "$EXPECTED_IMAGE" ]]; then
RESUME=1
fi
require_baseline
available_kb="$(df -Pk /var/lib/docker | awk 'NR==2 {print $4}')"
[[ "$available_kb" =~ ^[0-9]+$ && "$available_kb" -ge 786432 ]] || {
echo 'less than 768 MiB is available for the derived image' >&2
exit 1
}
log "plan image=${EXPECTED_IMAGE#sha256:} config=${EXPECTED_CONFIG_SHA256} free_kib=${available_kb}"
if [[ "$MODE" == plan ]]; then
log 'plan passed; no runtime state changed'
exit 0
fi
if ((RESUME)); then
log 'resuming a verified pre-cutover release'
[[ "$(sha256sum "$HISTORY/config.original.yaml" | cut -d' ' -f1)" == "$EXPECTED_CONFIG_SHA256" ]] || exit 1
[[ -f "$HISTORY/config.conservative.yaml" && -f "$HISTORY/config.final.yaml" ]] || exit 1
validate_candidate_config "$HISTORY/config.conservative.yaml"
validate_candidate_config "$HISTORY/config.final.yaml"
if [[ "$(psql -c "SELECT drain_state FROM runtime_operations_control WHERE id=1;")" == normal ]]; then
admin_post dispatch/drain/start
MUTATED=1
wait_for_drain
else
MUTATED=1
fi
else
install -d -o root -g root -m 0700 /var/lib/truf-deploy /var/lib/truf-deploy/history
if [[ -e "$HISTORY" ]]; then
echo 'release history already exists' >&2
exit 1
fi
install -d -o root -g root -m 0700 "$HISTORY" "$HISTORY/source"
install -o root -g root -m 0600 "$ACTIVE_CONFIG" "$HISTORY/config.original.yaml"
for name in "${APP_FILES[@]}"; do
if [[ -f "${SOURCE_ROOT}/app/$name" ]]; then
install -o root -g root -m 0600 "${SOURCE_ROOT}/app/$name" "$HISTORY/source/$name"
fi
done
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
--output "$HISTORY/config.conservative.yaml" --mode conservative \
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
python3 "$STAGE/render_config.py" --input "$ACTIVE_CONFIG" \
--output "$HISTORY/config.final.yaml" --mode final \
--expected-sha256 "$EXPECTED_CONFIG_SHA256" >/dev/null
chown 10001:10001 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
chmod 0600 "$HISTORY/config.conservative.yaml" "$HISTORY/config.final.yaml"
if docker image inspect "$CANDIDATE_TAG" >/dev/null 2>&1; then
echo 'candidate image tag already exists' >&2
exit 1
fi
PHASE='candidate-build'
docker build --network none --build-arg BASE_IMAGE=truf-local:runtime \
--file "$STAGE/Dockerfile" --tag "$CANDIDATE_TAG" "$STAGE"
[[ "$(current_image)" == "$EXPECTED_IMAGE" ]] || {
echo 'runtime tag changed during candidate build' >&2
exit 1
}
validate_candidate_config "$HISTORY/config.conservative.yaml"
validate_candidate_config "$HISTORY/config.final.yaml"
docker image tag "$EXPECTED_IMAGE" "$ROLLBACK_TAG"
PHASE='drain'
admin_post dispatch/drain/start
MUTATED=1
wait_for_drain
fi
PHASE='conservative-cutover'
quiesce_pipeline_workers
stop_stack
install_config "$HISTORY/config.conservative.yaml"
docker image tag "$CANDIDATE_TAG" truf-local:runtime
start_stack
schema_state="$(psql -F '|' -c \
"SELECT (SELECT count(*) FROM information_schema.columns WHERE table_name='result_reservations' AND column_name='reserved_bundle_bytes'), (SELECT count(*) FROM runtime_schema_migrations WHERE version='20260930_33_remote_assignment_capacity');")"
[[ "$schema_state" == '1|1' ]] || {
echo "capacity migration was not applied: $schema_state" >&2
exit 1
}
PHASE_A_HEALTHY=1
PHASE='capacity50-cutover'
quiesce_pipeline_workers
stop_stack
install_config "$HISTORY/config.final.yaml"
start_stack
final_values="$(psql -F '|' -c \
"SELECT bundle_items, bundle_bytes, projection_items, projection_bytes, keycheck_items, keycheck_bytes, quarantine_items, quarantine_bytes FROM pipeline_capacity WHERE id=1;")"
[[ "$final_values" == '0|0|0|0|0|0|0|0' ]] || {
echo "post-deploy capacity is not reconciled: $final_values" >&2
exit 1
}
PHASE='temporary-user-cap-validation'
admin_post users/cap "user_key=${TEST_USER}" 'active_assignment_cap=50'
USER_CAP_CHANGED=1
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == 50 ]] || exit 1
restore_user_cap
[[ "$(psql -c "SELECT active_assignment_cap FROM remote_worker_users WHERE user_key='${TEST_USER}';")" == "$TEST_USER_ORIGINAL_CAP" ]] || exit 1
PHASE='source-install'
install_sources
for name in "${APP_FILES[@]}"; do
cmp -s "$STAGE/payload/app/$name" "${SOURCE_ROOT}/app/$name" || exit 1
done
PHASE='resume'
cancel_drain
post_control="$(psql -F '|' -c \
"SELECT discovery_paused::int, dispatch_paused::int, drain_state FROM runtime_operations_control WHERE id=1;")"
[[ "$post_control" == '0|0|normal' ]] || {
echo "runtime control did not resume: $post_control" >&2
exit 1
}
cat >"$HISTORY/result.txt" <<EOF
release=${RELEASE_ID}
runtime_image=$(current_image)
config_sha256=$(config_sha256)
schema=${schema_state}
capacity=${final_values}
control=${post_control}
EOF
chmod 0600 "$HISTORY/result.txt"
DEPLOY_SUCCEEDED=1
log "applied image=$(current_image) config=$(config_sha256)"