feat(ops): ajouter supervision automatique des applications
This commit is contained in:
Manus AI
11
ops/README.md
Normal file
11
ops/README.md
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
# Supervision des applications
|
||||||
|
|
||||||
|
Le script `healthcheck-apps.sh` est exécuté par le timer systemd toutes les cinq minutes et trois minutes après chaque démarrage serveur. Il vérifie les conteneurs Docker Compose, leur healthcheck, leur politique de redémarrage et les routes HTTPS Traefik déclarées. En cas d’anomalie, une remise en route Compose ciblée est tentée une fois.
|
||||||
|
|
||||||
|
Le dernier rapport est écrit dans `/var/lib/manus-apps-health/latest.txt`. Les anomalies sont aussi consignées dans le journal système avec le tag `manus-apps-health`.
|
||||||
|
|
||||||
|
Après mise à jour Git, installer ou actualiser le service avec :
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./ops/install-healthcheck.sh
|
||||||
|
```
|
||||||
158
ops/healthcheck-apps.sh
Normal file
158
ops/healthcheck-apps.sh
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Vérifie les conteneurs Compose, leur healthcheck, leur redémarrage et leurs routes Traefik.
|
||||||
|
# En cas d'écart, un redémarrage Compose ciblé est tenté une fois avant le prochain cycle.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
readonly STATE_DIR="/var/lib/manus-apps-health"
|
||||||
|
readonly REPORT_FILE="${STATE_DIR}/latest.txt"
|
||||||
|
readonly LOCK_FILE="/run/manus-apps-health.lock"
|
||||||
|
readonly MODE="${1:---repair}"
|
||||||
|
|
||||||
|
mkdir -p "$STATE_DIR"
|
||||||
|
exec 9>"$LOCK_FILE"
|
||||||
|
flock -n 9 || exit 0
|
||||||
|
|
||||||
|
declare -A PROJECT_CONFIGS=()
|
||||||
|
declare -A PROJECT_NAMES=()
|
||||||
|
declare -A SEEN_HTTP_HOSTS=()
|
||||||
|
declare -A FAILED_PROJECTS=()
|
||||||
|
declare -a REPORT_LINES=()
|
||||||
|
HAS_FAILURE=false
|
||||||
|
|
||||||
|
log_line() {
|
||||||
|
local level="$1"
|
||||||
|
local message="$2"
|
||||||
|
local line="[$(date -Is)] [$level] $message"
|
||||||
|
REPORT_LINES+=("$line")
|
||||||
|
echo "$line"
|
||||||
|
}
|
||||||
|
|
||||||
|
mark_failure() {
|
||||||
|
HAS_FAILURE=true
|
||||||
|
log_line "KO" "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
mark_project_failure() {
|
||||||
|
FAILED_PROJECTS["$1"]=1
|
||||||
|
mark_failure "$2"
|
||||||
|
}
|
||||||
|
|
||||||
|
is_healthy_http_status() {
|
||||||
|
[[ "$1" =~ ^2[0-9][0-9]$ || "$1" =~ ^3[0-9][0-9]$ ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
check_http_route() {
|
||||||
|
local container_id="$1"
|
||||||
|
local project="$2"
|
||||||
|
local workdir="$3"
|
||||||
|
local labels host_rule host status
|
||||||
|
|
||||||
|
labels="$(docker inspect -f '{{range $key, $value := .Config.Labels}}{{$key}}={{$value}}{{"\n"}}{{end}}' "$container_id")"
|
||||||
|
host_rule="$(printf '%s\n' "$labels" | grep -oE 'Host\(`[^`]+`\)' | head -n 1 || true)"
|
||||||
|
[[ -n "$host_rule" ]] || return 0
|
||||||
|
|
||||||
|
host="${host_rule#Host(\`}"
|
||||||
|
host="${host%\`)}"
|
||||||
|
[[ -n "$host" ]] || return 0
|
||||||
|
[[ -z "${SEEN_HTTP_HOSTS[$host]:-}" ]] || return 0
|
||||||
|
SEEN_HTTP_HOSTS["$host"]=1
|
||||||
|
|
||||||
|
status="$(curl --silent --show-error --location --max-redirs 3 --connect-timeout 10 --max-time 25 \
|
||||||
|
--output /dev/null --write-out '%{http_code}' "https://${host}" 2>/dev/null || true)"
|
||||||
|
if is_healthy_http_status "$status"; then
|
||||||
|
log_line "OK" "${project}: HTTPS ${host} → ${status}"
|
||||||
|
else
|
||||||
|
mark_project_failure "$workdir" "${project}: HTTPS ${host} → ${status:-erreur réseau}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_containers() {
|
||||||
|
local container_id name project workdir config_files status health restart_policy
|
||||||
|
mapfile -t container_ids < <(docker ps -aq --filter label=com.docker.compose.project)
|
||||||
|
|
||||||
|
if [[ "${#container_ids[@]}" -eq 0 ]]; then
|
||||||
|
mark_failure "Aucun conteneur Docker Compose détecté"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
for container_id in "${container_ids[@]}"; do
|
||||||
|
name="$(docker inspect -f '{{.Name}}' "$container_id" | sed 's#^/##')"
|
||||||
|
project="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$container_id")"
|
||||||
|
workdir="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$container_id")"
|
||||||
|
config_files="$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project.config_files"}}' "$container_id")"
|
||||||
|
status="$(docker inspect -f '{{.State.Status}}' "$container_id")"
|
||||||
|
health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")"
|
||||||
|
restart_policy="$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' "$container_id")"
|
||||||
|
|
||||||
|
PROJECT_NAMES["$workdir"]="$project"
|
||||||
|
PROJECT_CONFIGS["$workdir"]="$config_files"
|
||||||
|
|
||||||
|
if [[ "$status" != "running" ]]; then
|
||||||
|
mark_project_failure "$workdir" "${project}/${name}: état Docker ${status}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$health" == "unhealthy" ]]; then
|
||||||
|
mark_project_failure "$workdir" "${project}/${name}: healthcheck unhealthy"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$restart_policy" != "unless-stopped" && "$restart_policy" != "always" ]]; then
|
||||||
|
mark_project_failure "$workdir" "${project}/${name}: politique de redémarrage ${restart_policy:-none}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_line "OK" "${project}/${name}: running, health=${health}, restart=${restart_policy}"
|
||||||
|
check_http_route "$container_id" "$project" "$workdir"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
repair_projects() {
|
||||||
|
local workdir config_files project config
|
||||||
|
for workdir in "${!FAILED_PROJECTS[@]}"; do
|
||||||
|
project="${PROJECT_NAMES[$workdir]}"
|
||||||
|
[[ -d "$workdir" ]] || continue
|
||||||
|
config_files="${PROJECT_CONFIGS[$workdir]}"
|
||||||
|
local -a compose_args=()
|
||||||
|
if [[ -n "$config_files" ]]; then
|
||||||
|
IFS=',' read -r -a configs <<< "$config_files"
|
||||||
|
for config in "${configs[@]}"; do
|
||||||
|
compose_args+=("-f" "$config")
|
||||||
|
done
|
||||||
|
elif [[ -f "$workdir/docker-compose.yml" ]]; then
|
||||||
|
compose_args+=("-f" "$workdir/docker-compose.yml")
|
||||||
|
else
|
||||||
|
mark_failure "${project}: fichier Compose introuvable pour la réparation"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_line "INFO" "${project}: tentative de réparation Compose"
|
||||||
|
if (cd "$workdir" && docker compose "${compose_args[@]}" up -d --remove-orphans); then
|
||||||
|
log_line "INFO" "${project}: réparation Compose demandée"
|
||||||
|
else
|
||||||
|
mark_failure "${project}: échec de réparation Compose"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
check_containers
|
||||||
|
|
||||||
|
if [[ "$HAS_FAILURE" == true && "$MODE" == "--repair" ]]; then
|
||||||
|
repair_projects
|
||||||
|
log_line "INFO" "Nouvelle vérification après réparation"
|
||||||
|
sleep 15
|
||||||
|
exec "$0" --check-only
|
||||||
|
fi
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "Supervision Santinova / Itinova"
|
||||||
|
echo "Généré : $(date -Is)"
|
||||||
|
echo "Statut : $([[ "$HAS_FAILURE" == true ]] && echo KO || echo OK)"
|
||||||
|
printf '%s\n' "${REPORT_LINES[@]}"
|
||||||
|
} > "${REPORT_FILE}.tmp"
|
||||||
|
mv "${REPORT_FILE}.tmp" "$REPORT_FILE"
|
||||||
|
|
||||||
|
if [[ "$HAS_FAILURE" == true ]]; then
|
||||||
|
logger -p daemon.err -t manus-apps-health "Anomalie détectée : consulter ${REPORT_FILE}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
logger -p daemon.info -t manus-apps-health "Toutes les applications contrôlées sont opérationnelles"
|
||||||
14
ops/install-healthcheck.sh
Normal file
14
ops/install-healthcheck.sh
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Installe la supervision depuis une copie versionnée du dépôt manus-dashboard.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
install -D -m 0750 "${SCRIPT_DIR}/healthcheck-apps.sh" /opt/manus-deploy/scripts/healthcheck-apps.sh
|
||||||
|
install -D -m 0644 "${SCRIPT_DIR}/manus-apps-health.service" /etc/systemd/system/manus-apps-health.service
|
||||||
|
install -D -m 0644 "${SCRIPT_DIR}/manus-apps-health.timer" /etc/systemd/system/manus-apps-health.timer
|
||||||
|
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable --now manus-apps-health.timer
|
||||||
|
systemctl start manus-apps-health.service
|
||||||
|
systemctl status manus-apps-health.service --no-pager
|
||||||
13
ops/manus-apps-health.service
Normal file
13
ops/manus-apps-health.service
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Contrôle et réparation des applications Santinova / Itinova
|
||||||
|
Documentation=https://git.santinova-soft.org/manus-admin/manus-dashboard
|
||||||
|
After=docker.service network-online.target manus-apps.service
|
||||||
|
Requires=docker.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/opt/manus-deploy/scripts/healthcheck-apps.sh --repair
|
||||||
|
TimeoutStartSec=10min
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
12
ops/manus-apps-health.timer
Normal file
12
ops/manus-apps-health.timer
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Planification de la supervision des applications Santinova / Itinova
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=3min
|
||||||
|
OnUnitActiveSec=5min
|
||||||
|
RandomizedDelaySec=30s
|
||||||
|
Persistent=true
|
||||||
|
Unit=manus-apps-health.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
Reference in New Issue
Block a user