106 lines
4.5 KiB
Bash
Executable File
106 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# L99 TOTAL WATCHDOG — catches EVERYTHING
|
|
# Cron: */15 * * * *
|
|
TS=$(date -Iseconds)
|
|
REPORT=/var/www/html/api/l99-watchdog.json
|
|
LOG=/var/log/l99-watchdog.log
|
|
echo "$TS === WATCHDOG ===" >> $LOG
|
|
|
|
PASS=0;FAIL=0;WARN=0;ISSUES=""
|
|
|
|
check() {
|
|
local CAT=$1 NAME=$2 CMD=$3 EXPECT=$4
|
|
RESULT=$(eval "$CMD" 2>/dev/null)
|
|
if echo "$RESULT"|grep -qiE "$EXPECT"; then
|
|
PASS=$((PASS+1))
|
|
else
|
|
FAIL=$((FAIL+1))
|
|
ISSUES="$ISSUES|$CAT:$NAME=$RESULT"
|
|
echo "$TS FAIL $CAT:$NAME got=$RESULT" >> $LOG
|
|
fi
|
|
}
|
|
|
|
# === INFRA ===
|
|
check "INFRA" "nginx" "systemctl is-active nginx" "^active"
|
|
check "INFRA" "php-fpm" "systemctl is-active php8.5-fpm" "^active"
|
|
check "INFRA" "deerflow" "systemctl is-active deerflow" "^active"
|
|
check "INFRA" "disk-pct" "df /|tail -1|awk '{print \$5}'|tr -d '%'" "^[0-8][0-9]$"
|
|
check "INFRA" "disk-inodes" "df -i /|tail -1|awk '{print \$5}'|tr -d '%'" "^[0-4][0-9]$"
|
|
check "INFRA" "ram-free" "free -m|grep Mem|awk '{print \$7}'" "^[0-9]"
|
|
check "INFRA" "load" "cat /proc/loadavg|awk '{print \$1}'|cut -d. -f1" "^[0-9]$"
|
|
check "INFRA" "ssl-valid" "echo|openssl s_client -servername weval-consulting.com -connect 127.0.0.1:443 2>/dev/null|openssl x509 -noout -checkend 604800 2>/dev/null && echo OK" "OK"
|
|
|
|
# === DOCKER (all 17) ===
|
|
for C in $(docker ps --format '{{.Names}}'); do
|
|
ST=$(docker inspect --format '{{.State.Status}}' $C 2>/dev/null)
|
|
check "DOCKER" "$C" "echo $ST" "running"
|
|
done
|
|
|
|
# === PAGES ROOT (all .html) ===
|
|
for P in $(find /var/www/html -maxdepth 1 -name '*.html' -exec basename {} \;|head -80); do
|
|
CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 3 "https://127.0.0.1/$P" -H 'Host: weval-consulting.com')
|
|
check "PAGE" "$P" "echo $CODE" "^(200|302)$"
|
|
done
|
|
|
|
# === DIRS WITH INDEX ===
|
|
for D in contact-us blog products service nearshore-it-maroc platform; do
|
|
CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 3 "https://127.0.0.1/$D/" -H 'Host: weval-consulting.com')
|
|
check "DIR" "$D" "echo $CODE" "^(200|302)$"
|
|
done
|
|
|
|
# === SUBDOMAINS ===
|
|
for S in crm paperclip langfuse deerflow analytics mm monitor n8n git; do
|
|
CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 15 "https://$S.weval-consulting.com/")
|
|
check "SUB" "$S" "echo $CODE" "^(200|302)$"
|
|
done
|
|
|
|
# === APIs ===
|
|
for A in nonreg-api.php?cat=all ecosystem-health.php wevia-deep-test.php wevia-master-autoheal.php architecture-index.json l99-state.json openclaw-proxy.php; do
|
|
CODE=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 15 "https://127.0.0.1/api/$A" -H 'Host: weval-consulting.com')
|
|
check "API" "$A" "echo $CODE" "^200$"
|
|
done
|
|
|
|
# === CHATBOT SEMANTIC ===
|
|
for EP in weval-ia-fast.php weval-ia; do
|
|
RESP=$(curl -sk --max-time 10 -X POST "https://127.0.0.1/api/$EP" -H "Host: weval-consulting.com" -H "Content-Type: application/json" -d '{"message":"Qui est WEVAL","provider":"cerebras"}' 2>/dev/null)
|
|
HAS=$(echo "$RESP"|grep -ciE "cabinet|casablanca|consulting")
|
|
BAD=$(echo "$RESP"|grep -ciE "musique|groupe|n.existe pas")
|
|
if [ "$HAS" -gt 0 ] && [ "$BAD" -eq 0 ]; then PASS=$((PASS+1))
|
|
elif [ ${#RESP} -lt 10 ]; then WARN=$((WARN+1))
|
|
else FAIL=$((FAIL+1));ISSUES="$ISSUES|SEMANTIC:$EP=hallucination";echo "$TS SEMANTIC FAIL $EP" >> $LOG; fi
|
|
done
|
|
|
|
# === PHP SYNTAX ===
|
|
PHP_ERR=$(find /var/www/html/api -name '*.php' -exec php -l {} \; 2>&1|grep -c "Parse error")
|
|
check "PHP" "syntax-errors" "echo $PHP_ERR" "^0$"
|
|
|
|
# === CRONS ===
|
|
CRON_N=$(crontab -l 2>/dev/null|grep -v '^#'|wc -l)
|
|
check "CRONS" "count>30" "echo $CRON_N" "^[3-9][0-9]"
|
|
|
|
# === OLLAMA ===
|
|
OLL=$(curl -s http://localhost:11435/api/tags 2>/dev/null|python3 -c "import sys,json;print(len(json.load(sys.stdin).get('models',[])))" 2>/dev/null)
|
|
check "AI" "ollama-models>=3" "echo ${OLL:-0}" "^[3-9]"
|
|
|
|
# === QDRANT ===
|
|
QD=$(curl -s http://localhost:6333/collections 2>/dev/null|python3 -c "import sys,json;print(len(json.load(sys.stdin).get('result',{}).get('collections',[])))" 2>/dev/null)
|
|
check "AI" "qdrant-collections>=3" "echo ${QD:-0}" "^[3-9]"
|
|
|
|
# === GIT ===
|
|
DIRTY=$(cd /var/www/html && git status --porcelain 2>/dev/null|wc -l)
|
|
check "GIT" "dirty<20" "echo $DIRTY" "^[0-1]?[0-9]$"
|
|
|
|
# === RESULT ===
|
|
TOTAL=$((PASS+FAIL+WARN))
|
|
SCORE=$(python3 -c "print(round($PASS/max(1,$TOTAL)*100,1))" 2>/dev/null)
|
|
|
|
python3 -c "
|
|
import json,time
|
|
d={'ts':'$TS','pass':$PASS,'fail':$FAIL,'warn':$WARN,'total':$TOTAL,'score':$SCORE,
|
|
'issues':'$ISSUES'.split('|')[1:] if '$ISSUES' else []}
|
|
json.dump(d,open('$REPORT','w'),indent=2)
|
|
"
|
|
|
|
echo "$TS DONE: ${PASS}/${TOTAL} (${SCORE}%) ${FAIL}F ${WARN}W" >> $LOG
|
|
echo "{\"pass\":$PASS,\"fail\":$FAIL,\"total\":$TOTAL,\"score\":$SCORE}"
|