manus-dashboard/ops/healthcheck-apps.sh

#!/usr/bin/env bash
# Vérifie les conteneurs Compose, leur healthcheck, leur redémarrage et leurs routes Traefik.
# En cas d'écart, un redémarrage Compose ciblé est tenté une fois avant le prochain cycle.
set -Eeuo pipefail

readonly STATE_DIR="/var/lib/manus-apps-health"
readonly REPORT_FILE="${STATE_DIR}/latest.txt"
readonly LOCK_FILE="/run/manus-apps-health.lock"
readonly MODE="${1:---repair}"

mkdir -p "$STATE_DIR"
exec 9>"$LOCK_FILE"
flock -n 9 || exit 0

declare -A PROJECT_CONFIGS=()
declare -A PROJECT_NAMES=()
declare -A SEEN_HTTP_HOSTS=()
declare -A FAILED_PROJECTS=()
declare -a REPORT_LINES=()
HAS_FAILURE=false

log_line() {
  local level="$1"
  local message="$2"
  local line="[$(date -Is)] [$level] $message"
  REPORT_LINES+=("$line")
  echo "$line"
}

mark_failure() {
  HAS_FAILURE=true
  log_line "KO" "$1"
}

mark_project_failure() {
  FAILED_PROJECTS["$1"]=1
  mark_failure "$2"
}

is_healthy_http_status() {
  [[ "$1" =~ ^2[0-9][0-9]$ || "$1" =~ ^3[0-9][0-9]$ ]]
}

check_http_route() {
  local container_id="$1"
  local project="$2"
  local workdir="$3"
  local labels host_rule host status

  labels="$(docker inspect -f '{{range $key, $value := .Config.Labels}}{{$key}}={{$value}}{{"\n"}}{{end}}' "$container_id")"
  host_rule="$(printf '%s\n' "$labels" | grep -oE 'Host\(`[^`]+`\)' | head -n 1 || true)"
  [[ -n "$host_rule" ]] || return 0

  host="${host_rule#Host(\`}"
  host="${host%\`)}"
  [[ -n "$host" ]] || return 0
  [[ -z "${SEEN_HTTP_HOSTS[$host]:-}" ]] || return 0
  SEEN_HTTP_HOSTS["$host"]=1

  status="$(curl --silent --show-error --location --max-redirs 3 --connect-timeout 10 --max-time 25 \
    --output /dev/null --write-out '%{http_code}' "https://${host}" 2>/dev/null || true)"
  if is_healthy_http_status "$status"; then
    log_line "OK" "${project}: HTTPS ${host} → ${status}"
  else
    mark_project_failure "$workdir" "${project}: HTTPS ${host} → ${status:-erreur réseau}"
  fi
}

check_containers() {
  local container_id name project workdir config_files status health restart_policy
  mapfile -t container_ids < <(docker ps -aq --filter label=com.docker.compose.project)

  if [[ "${#container_ids[@]}" -eq 0 ]]; then
    mark_failure "Aucun conteneur Docker Compose détecté"
    return
  fi

  for container_id in "${container_ids[@]}"; do
    name="$(docker inspect -f '{{.Name}}' "$container_id" | sed 's#^/##')"
    project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$container_id")"
    workdir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$container_id")"
    config_files="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.config_files"}}' "$container_id")"
    status="$(docker inspect -f '{{.State.Status}}' "$container_id")"
    health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")"
    restart_policy="$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$container_id")"

    PROJECT_NAMES["$workdir"]="$project"
    PROJECT_CONFIGS["$workdir"]="$config_files"

    if [[ "$status" != "running" ]]; then
      mark_project_failure "$workdir" "${project}/${name}: état Docker ${status}"
      continue
    fi
    if [[ "$health" == "unhealthy" ]]; then
      mark_project_failure "$workdir" "${project}/${name}: healthcheck unhealthy"
      continue
    fi
    if [[ "$restart_policy" != "unless-stopped" && "$restart_policy" != "always" ]]; then
      mark_project_failure "$workdir" "${project}/${name}: politique de redémarrage ${restart_policy:-none}"
      continue
    fi

    log_line "OK" "${project}/${name}: running, health=${health}, restart=${restart_policy}"
    check_http_route "$container_id" "$project" "$workdir"
  done
}

repair_projects() {
  local workdir config_files project config
  for workdir in "${!FAILED_PROJECTS[@]}"; do
    project="${PROJECT_NAMES[$workdir]}"
    [[ -d "$workdir" ]] || continue
    config_files="${PROJECT_CONFIGS[$workdir]}"
    local -a compose_args=()
    if [[ -n "$config_files" ]]; then
      IFS=',' read -r -a configs <<< "$config_files"
      for config in "${configs[@]}"; do
        compose_args+=("-f" "$config")
      done
    elif [[ -f "$workdir/docker-compose.yml" ]]; then
      compose_args+=("-f" "$workdir/docker-compose.yml")
    else
      mark_failure "${project}: fichier Compose introuvable pour la réparation"
      continue
    fi

    log_line "INFO" "${project}: tentative de réparation Compose"
    if (cd "$workdir" && docker compose "${compose_args[@]}" up -d --remove-orphans); then
      log_line "INFO" "${project}: réparation Compose demandée"
    else
      mark_failure "${project}: échec de réparation Compose"
    fi
  done
}

check_containers

if [[ "$HAS_FAILURE" == true && "$MODE" == "--repair" ]]; then
  repair_projects
  log_line "INFO" "Nouvelle vérification après réparation"
  sleep 15
  exec "$0" --check-only
fi

{
  echo "Supervision Santinova / Itinova"
  echo "Généré : $(date -Is)"
  echo "Statut : $([[ "$HAS_FAILURE" == true ]] && echo KO || echo OK)"
  printf '%s\n' "${REPORT_LINES[@]}"
} > "${REPORT_FILE}.tmp"
mv "${REPORT_FILE}.tmp" "$REPORT_FILE"

if [[ "$HAS_FAILURE" == true ]]; then
  logger -p daemon.err -t manus-apps-health "Anomalie détectée : consulter ${REPORT_FILE}"
  exit 1
fi

logger -p daemon.info -t manus-apps-health "Toutes les applications contrôlées sont opérationnelles"