122 lines
4.0 KiB
PHP
122 lines
4.0 KiB
PHP
<?php
|
|
// WEVAL Service Watchdog — runs every 3 minutes via cron
|
|
// Checks ALL critical services, auto-restarts if down
|
|
|
|
$log = "/var/log/weval-watchdog.log";
|
|
$ts = date('Y-m-d H:i:s');
|
|
$fixed = 0;
|
|
|
|
function check_http($url, $timeout=5) {
|
|
$ch = curl_init($url);
|
|
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>$timeout,
|
|
CURLOPT_SSL_VERIFYPEER=>false, CURLOPT_NOBODY=>true]);
|
|
curl_exec($ch);
|
|
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
curl_close($ch);
|
|
return $code;
|
|
}
|
|
|
|
function check_port($port) {
|
|
$conn = @fsockopen('127.0.0.1', $port, $errno, $errstr, 2);
|
|
if ($conn) { fclose($conn); return true; }
|
|
return false;
|
|
}
|
|
|
|
function wlog($msg) {
|
|
global $log, $ts;
|
|
file_put_contents($log, "[$ts] $msg\n", FILE_APPEND);
|
|
}
|
|
|
|
// ═══ 1. PAPERCLIP :3100 ═══
|
|
if (!check_port(3100)) {
|
|
wlog("PAPERCLIP DOWN → restart");
|
|
shell_exec("sudo pkill -9 -f paperclipai 2>/dev/null");
|
|
sleep(2);
|
|
shell_exec("sudo bash -c 'cd /opt/paperclip-weval && nohup env ANTHROPIC_BASE_URL=https://weval-consulting.com/api/wevia-openai.php/api/wevia-anthropic.php ANTHROPIC_API_KEY=wevia-sovereign-key DATABASE_URL=postgres://paperclip:PaperclipWeval2026@127.0.0.1:5432/paperclip PORT=3100 npx paperclipai run >> /var/log/paperclip.log 2>&1 &'");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 2. PHP-FPM ═══
|
|
$fpm = trim(shell_exec("systemctl is-active php8.5-fpm 2>/dev/null"));
|
|
if ($fpm !== 'active') {
|
|
wlog("PHP-FPM DOWN → restart");
|
|
shell_exec("sudo systemctl restart php8.5-fpm");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 3. NGINX ═══
|
|
$nginx = trim(shell_exec("systemctl is-active nginx 2>/dev/null"));
|
|
if ($nginx !== 'active') {
|
|
wlog("NGINX DOWN → restart");
|
|
shell_exec("sudo systemctl restart nginx");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 4. OLLAMA :11435 ═══
|
|
if (!check_port(11435)) {
|
|
wlog("OLLAMA DOWN → restart");
|
|
shell_exec("sudo systemctl restart ollama 2>/dev/null || nohup ollama serve >> /var/log/ollama.log 2>&1 &");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 5. DOCKER CONTAINERS ═══
|
|
$critical_containers = [
|
|
'plausible', 'plausible-db', 'plausible-events',
|
|
'authentik-db', 'authentik-redis',
|
|
'searxng', 'mattermost', 'uptime-kuma', 'qdrant', 'n8n',
|
|
'vaultwarden', 'twenty', 'twenty-redis', 'mirofish',
|
|
'open-webui', 'flowise', 'loki'
|
|
];
|
|
|
|
foreach ($critical_containers as $c) {
|
|
$status = trim(shell_exec("docker inspect -f '{{.State.Running}}' $c 2>/dev/null"));
|
|
if ($status !== 'true') {
|
|
wlog("DOCKER $c DOWN → restart");
|
|
shell_exec("docker start $c 2>/dev/null");
|
|
$fixed++;
|
|
}
|
|
}
|
|
|
|
// ═══ 6. DEERFLOW :3002 ═══
|
|
if (!check_port(3002)) {
|
|
wlog("DEERFLOW DOWN → restart");
|
|
shell_exec("cd /opt/deer-flow && nohup npm start >> /var/log/deerflow.log 2>&1 &");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 7. MIROFISH :3050 ═══
|
|
if (!check_port(3050)) {
|
|
wlog("MIROFISH DOWN → restart");
|
|
shell_exec("cd /opt/mirofish && nohup npm start >> /var/log/mirofish.log 2>&1 &");
|
|
$fixed++;
|
|
}
|
|
|
|
// ═══ 8. OPCACHE HEALTH ═══
|
|
if (function_exists('opcache_get_status')) {
|
|
$oc = opcache_get_status(false);
|
|
if ($oc && $oc['cache_full']) {
|
|
opcache_reset();
|
|
wlog("OPCACHE FULL → reset");
|
|
$fixed++;
|
|
}
|
|
}
|
|
|
|
// ═══ 9. DISK CHECK ═══
|
|
$disk = (int)trim(shell_exec("df / --output=pcent | tail -1 | tr -d ' %'"));
|
|
if ($disk > 90) {
|
|
wlog("DISK $disk% → cleanup");
|
|
shell_exec("sudo truncate -s 0 /var/log/*.log /var/log/nginx/*.log 2>/dev/null");
|
|
shell_exec("sudo journalctl --vacuum-size=50M 2>/dev/null");
|
|
shell_exec("sudo docker system prune -f 2>/dev/null");
|
|
$fixed++;
|
|
}
|
|
|
|
if ($fixed > 0) {
|
|
wlog("Fixed $fixed services");
|
|
// Telegram alert
|
|
$tg_token = '8544624912:AAEm9ttXK6JeFqAL-gcvB5sreCBhXzzQwrs';
|
|
$tg_chat = '7605775322';
|
|
$msg = urlencode("🔧 WATCHDOG: $fixed services restarted at $ts");
|
|
@file_get_contents("https://api.telegram.org/bot$tg_token/sendMessage?chat_id=$tg_chat&text=$msg");
|
|
}
|