fix(ops): rendre le healthcheck non intrusif
This commit is contained in:
@@ -1,21 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Vérifie les conteneurs Compose, leur healthcheck, leur redémarrage et leurs routes Traefik.
|
||||
# En cas d'écart, un redémarrage Compose ciblé est tenté une fois avant le prochain cycle.
|
||||
# Ce contrôle est volontairement non intrusif : Docker et manus-apps.service assurent les redémarrages.
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly STATE_DIR="/var/lib/manus-apps-health"
|
||||
readonly REPORT_FILE="${STATE_DIR}/latest.txt"
|
||||
readonly LOCK_FILE="/run/manus-apps-health.lock"
|
||||
readonly MODE="${1:---repair}"
|
||||
readonly MODE="${1:---check-only}"
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
exec 9>"$LOCK_FILE"
|
||||
flock -n 9 || exit 0
|
||||
|
||||
declare -A PROJECT_CONFIGS=()
|
||||
declare -A PROJECT_NAMES=()
|
||||
declare -A SEEN_HTTP_HOSTS=()
|
||||
declare -A FAILED_PROJECTS=()
|
||||
declare -a REPORT_LINES=()
|
||||
HAS_FAILURE=false
|
||||
|
||||
@@ -32,11 +29,6 @@ mark_failure() {
|
||||
log_line "KO" "$1"
|
||||
}
|
||||
|
||||
mark_project_failure() {
|
||||
FAILED_PROJECTS["$1"]=1
|
||||
mark_failure "$2"
|
||||
}
|
||||
|
||||
is_healthy_http_status() {
|
||||
[[ "$1" =~ ^2[0-9][0-9]$ || "$1" =~ ^3[0-9][0-9]$ ]]
|
||||
}
|
||||
@@ -44,7 +36,6 @@ is_healthy_http_status() {
|
||||
check_http_route() {
|
||||
local container_id="$1"
|
||||
local project="$2"
|
||||
local workdir="$3"
|
||||
local labels host_rule host status
|
||||
|
||||
labels="$(docker inspect -f '{{range $key, $value := .Config.Labels}}{{$key}}={{$value}}{{"\n"}}{{end}}' "$container_id")"
|
||||
@@ -57,12 +48,14 @@ check_http_route() {
|
||||
[[ -z "${SEEN_HTTP_HOSTS[$host]:-}" ]] || return 0
|
||||
SEEN_HTTP_HOSTS["$host"]=1
|
||||
|
||||
status="$(curl --silent --show-error --location --max-redirs 3 --connect-timeout 10 --max-time 25 \
|
||||
status="$(curl --silent --show-error --location --max-redirs 3 --connect-timeout 5 --max-time 8 \
|
||||
--output /dev/null --write-out '%{http_code}' "https://${host}" 2>/dev/null || true)"
|
||||
if is_healthy_http_status "$status"; then
|
||||
log_line "OK" "${project}: HTTPS ${host} → ${status}"
|
||||
else
|
||||
mark_project_failure "$workdir" "${project}: HTTPS ${host} → ${status:-erreur réseau}"
|
||||
# Un échec HTTPS peut dépendre de Traefik, DNS ou d’une redirection applicative.
|
||||
# Il déclenche une alerte, jamais un redémarrage Compose automatique.
|
||||
mark_failure "${project}: HTTPS ${host} → ${status:-erreur réseau}"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -84,64 +77,26 @@ check_containers() {
|
||||
health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")"
|
||||
restart_policy="$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$container_id")"
|
||||
|
||||
PROJECT_NAMES["$workdir"]="$project"
|
||||
PROJECT_CONFIGS["$workdir"]="$config_files"
|
||||
|
||||
if [[ "$status" != "running" ]]; then
|
||||
mark_project_failure "$workdir" "${project}/${name}: état Docker ${status}"
|
||||
mark_failure "${project}/${name}: état Docker ${status}"
|
||||
continue
|
||||
fi
|
||||
if [[ "$health" == "unhealthy" ]]; then
|
||||
mark_project_failure "$workdir" "${project}/${name}: healthcheck unhealthy"
|
||||
mark_failure "${project}/${name}: healthcheck unhealthy"
|
||||
continue
|
||||
fi
|
||||
if [[ "$restart_policy" != "unless-stopped" && "$restart_policy" != "always" ]]; then
|
||||
mark_project_failure "$workdir" "${project}/${name}: politique de redémarrage ${restart_policy:-none}"
|
||||
mark_failure "${project}/${name}: politique de redémarrage ${restart_policy:-none}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log_line "OK" "${project}/${name}: running, health=${health}, restart=${restart_policy}"
|
||||
check_http_route "$container_id" "$project" "$workdir"
|
||||
done
|
||||
}
|
||||
|
||||
repair_projects() {
|
||||
local workdir config_files project config
|
||||
for workdir in "${!FAILED_PROJECTS[@]}"; do
|
||||
project="${PROJECT_NAMES[$workdir]}"
|
||||
[[ -d "$workdir" ]] || continue
|
||||
config_files="${PROJECT_CONFIGS[$workdir]}"
|
||||
local -a compose_args=()
|
||||
if [[ -n "$config_files" ]]; then
|
||||
IFS=',' read -r -a configs <<< "$config_files"
|
||||
for config in "${configs[@]}"; do
|
||||
compose_args+=("-f" "$config")
|
||||
done
|
||||
elif [[ -f "$workdir/docker-compose.yml" ]]; then
|
||||
compose_args+=("-f" "$workdir/docker-compose.yml")
|
||||
else
|
||||
mark_failure "${project}: fichier Compose introuvable pour la réparation"
|
||||
continue
|
||||
fi
|
||||
|
||||
log_line "INFO" "${project}: tentative de réparation Compose"
|
||||
if (cd "$workdir" && docker compose "${compose_args[@]}" up -d --remove-orphans); then
|
||||
log_line "INFO" "${project}: réparation Compose demandée"
|
||||
else
|
||||
mark_failure "${project}: échec de réparation Compose"
|
||||
fi
|
||||
check_http_route "$container_id" "$project"
|
||||
done
|
||||
}
|
||||
|
||||
check_containers
|
||||
|
||||
if [[ "$HAS_FAILURE" == true && "$MODE" == "--repair" ]]; then
|
||||
repair_projects
|
||||
log_line "INFO" "Nouvelle vérification après réparation"
|
||||
sleep 15
|
||||
exec "$0" --check-only
|
||||
fi
|
||||
|
||||
{
|
||||
echo "Supervision Santinova / Itinova"
|
||||
echo "Généré : $(date -Is)"
|
||||
|
||||
Reference in New Issue
Block a user