From 5e8d4796d498a323357b94c2761983e9f1309858 Mon Sep 17 00:00:00 2001 From: Manus AI Date: Tue, 18 Aug 2026 07:31:02 +0000 Subject: [PATCH] feat(ops): ajouter supervision automatique des applications --- ops/README.md | 11 +++ ops/healthcheck-apps.sh | 158 ++++++++++++++++++++++++++++++++++ ops/install-healthcheck.sh | 14 +++ ops/manus-apps-health.service | 13 +++ ops/manus-apps-health.timer | 12 +++ 5 files changed, 208 insertions(+) create mode 100644 ops/README.md create mode 100644 ops/healthcheck-apps.sh create mode 100644 ops/install-healthcheck.sh create mode 100644 ops/manus-apps-health.service create mode 100644 ops/manus-apps-health.timer diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 0000000..c39ceef --- /dev/null +++ b/ops/README.md @@ -0,0 +1,11 @@ +# Supervision des applications + +Le script `healthcheck-apps.sh` est exécuté par le timer systemd toutes les cinq minutes et trois minutes après chaque démarrage serveur. Il vérifie les conteneurs Docker Compose, leur healthcheck, leur politique de redémarrage et les routes HTTPS Traefik déclarées. En cas d’anomalie, une remise en route Compose ciblée est tentée une fois. + +Le dernier rapport est écrit dans `/var/lib/manus-apps-health/latest.txt`. Les anomalies sont aussi consignées dans le journal système avec le tag `manus-apps-health`. + +Après mise à jour Git, installer ou actualiser le service avec : + +```bash +sudo ./ops/install-healthcheck.sh +``` diff --git a/ops/healthcheck-apps.sh b/ops/healthcheck-apps.sh new file mode 100644 index 0000000..fb5ecf1 --- /dev/null +++ b/ops/healthcheck-apps.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Vérifie les conteneurs Compose, leur healthcheck, leur redémarrage et leurs routes Traefik. +# En cas d'écart, un redémarrage Compose ciblé est tenté une fois avant le prochain cycle. +set -Eeuo pipefail + +readonly STATE_DIR="/var/lib/manus-apps-health" +readonly REPORT_FILE="${STATE_DIR}/latest.txt" +readonly LOCK_FILE="/run/manus-apps-health.lock" +readonly MODE="${1:---repair}" + +mkdir -p "$STATE_DIR" +exec 9>"$LOCK_FILE" +flock -n 9 || exit 0 + +declare -A PROJECT_CONFIGS=() +declare -A PROJECT_NAMES=() +declare -A SEEN_HTTP_HOSTS=() +declare -A FAILED_PROJECTS=() +declare -a REPORT_LINES=() +HAS_FAILURE=false + +log_line() { + local level="$1" + local message="$2" + local line="[$(date -Is)] [$level] $message" + REPORT_LINES+=("$line") + echo "$line" +} + +mark_failure() { + HAS_FAILURE=true + log_line "KO" "$1" +} + +mark_project_failure() { + FAILED_PROJECTS["$1"]=1 + mark_failure "$2" +} + +is_healthy_http_status() { + [[ "$1" =~ ^2[0-9][0-9]$ || "$1" =~ ^3[0-9][0-9]$ ]] +} + +check_http_route() { + local container_id="$1" + local project="$2" + local workdir="$3" + local labels host_rule host status + + labels="$(docker inspect -f '{{range $key, $value := .Config.Labels}}{{$key}}={{$value}}{{"\n"}}{{end}}' "$container_id")" + host_rule="$(printf '%s\n' "$labels" | grep -oE 'Host\(`[^`]+`\)' | head -n 1 || true)" + [[ -n "$host_rule" ]] || return 0 + + host="${host_rule#Host(\`}" + host="${host%\`)}" + [[ -n "$host" ]] || return 0 + [[ -z "${SEEN_HTTP_HOSTS[$host]:-}" ]] || return 0 + SEEN_HTTP_HOSTS["$host"]=1 + + status="$(curl --silent --show-error --location --max-redirs 3 --connect-timeout 10 --max-time 25 \ + --output /dev/null --write-out '%{http_code}' "https://${host}" 2>/dev/null || true)" + if is_healthy_http_status "$status"; then + log_line "OK" "${project}: HTTPS ${host} → ${status}" + else + mark_project_failure "$workdir" "${project}: HTTPS ${host} → ${status:-erreur réseau}" + fi +} + +check_containers() { + local container_id name project workdir config_files status health restart_policy + mapfile -t container_ids < <(docker ps -aq --filter label=com.docker.compose.project) + + if [[ "${#container_ids[@]}" -eq 0 ]]; then + mark_failure "Aucun conteneur Docker Compose détecté" + return + fi + + for container_id in "${container_ids[@]}"; do + name="$(docker inspect -f '{{.Name}}' "$container_id" | sed 's#^/##')" + project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$container_id")" + workdir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$container_id")" + config_files="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.config_files"}}' "$container_id")" + status="$(docker inspect -f '{{.State.Status}}' "$container_id")" + health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")" + restart_policy="$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$container_id")" + + PROJECT_NAMES["$workdir"]="$project" + PROJECT_CONFIGS["$workdir"]="$config_files" + + if [[ "$status" != "running" ]]; then + mark_project_failure "$workdir" "${project}/${name}: état Docker ${status}" + continue + fi + if [[ "$health" == "unhealthy" ]]; then + mark_project_failure "$workdir" "${project}/${name}: healthcheck unhealthy" + continue + fi + if [[ "$restart_policy" != "unless-stopped" && "$restart_policy" != "always" ]]; then + mark_project_failure "$workdir" "${project}/${name}: politique de redémarrage ${restart_policy:-none}" + continue + fi + + log_line "OK" "${project}/${name}: running, health=${health}, restart=${restart_policy}" + check_http_route "$container_id" "$project" "$workdir" + done +} + +repair_projects() { + local workdir config_files project config + for workdir in "${!FAILED_PROJECTS[@]}"; do + project="${PROJECT_NAMES[$workdir]}" + [[ -d "$workdir" ]] || continue + config_files="${PROJECT_CONFIGS[$workdir]}" + local -a compose_args=() + if [[ -n "$config_files" ]]; then + IFS=',' read -r -a configs <<< "$config_files" + for config in "${configs[@]}"; do + compose_args+=("-f" "$config") + done + elif [[ -f "$workdir/docker-compose.yml" ]]; then + compose_args+=("-f" "$workdir/docker-compose.yml") + else + mark_failure "${project}: fichier Compose introuvable pour la réparation" + continue + fi + + log_line "INFO" "${project}: tentative de réparation Compose" + if (cd "$workdir" && docker compose "${compose_args[@]}" up -d --remove-orphans); then + log_line "INFO" "${project}: réparation Compose demandée" + else + mark_failure "${project}: échec de réparation Compose" + fi + done +} + +check_containers + +if [[ "$HAS_FAILURE" == true && "$MODE" == "--repair" ]]; then + repair_projects + log_line "INFO" "Nouvelle vérification après réparation" + sleep 15 + exec "$0" --check-only +fi + +{ + echo "Supervision Santinova / Itinova" + echo "Généré : $(date -Is)" + echo "Statut : $([[ "$HAS_FAILURE" == true ]] && echo KO || echo OK)" + printf '%s\n' "${REPORT_LINES[@]}" +} > "${REPORT_FILE}.tmp" +mv "${REPORT_FILE}.tmp" "$REPORT_FILE" + +if [[ "$HAS_FAILURE" == true ]]; then + logger -p daemon.err -t manus-apps-health "Anomalie détectée : consulter ${REPORT_FILE}" + exit 1 +fi + +logger -p daemon.info -t manus-apps-health "Toutes les applications contrôlées sont opérationnelles" diff --git a/ops/install-healthcheck.sh b/ops/install-healthcheck.sh new file mode 100644 index 0000000..03fa80a --- /dev/null +++ b/ops/install-healthcheck.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Installe la supervision depuis une copie versionnée du dépôt manus-dashboard. +set -Eeuo pipefail + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +install -D -m 0750 "${SCRIPT_DIR}/healthcheck-apps.sh" /opt/manus-deploy/scripts/healthcheck-apps.sh +install -D -m 0644 "${SCRIPT_DIR}/manus-apps-health.service" /etc/systemd/system/manus-apps-health.service +install -D -m 0644 "${SCRIPT_DIR}/manus-apps-health.timer" /etc/systemd/system/manus-apps-health.timer + +systemctl daemon-reload +systemctl enable --now manus-apps-health.timer +systemctl start manus-apps-health.service +systemctl status manus-apps-health.service --no-pager diff --git a/ops/manus-apps-health.service b/ops/manus-apps-health.service new file mode 100644 index 0000000..77185ad --- /dev/null +++ b/ops/manus-apps-health.service @@ -0,0 +1,13 @@ +[Unit] +Description=Contrôle et réparation des applications Santinova / Itinova +Documentation=https://git.santinova-soft.org/manus-admin/manus-dashboard +After=docker.service network-online.target manus-apps.service +Requires=docker.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/opt/manus-deploy/scripts/healthcheck-apps.sh --repair +TimeoutStartSec=10min +StandardOutput=journal +StandardError=journal diff --git a/ops/manus-apps-health.timer b/ops/manus-apps-health.timer new file mode 100644 index 0000000..70388e9 --- /dev/null +++ b/ops/manus-apps-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Planification de la supervision des applications Santinova / Itinova + +[Timer] +OnBootSec=3min +OnUnitActiveSec=5min +RandomizedDelaySec=30s +Persistent=true +Unit=manus-apps-health.service + +[Install] +WantedBy=timers.target