import json, os, subprocess as sp, time from datetime import datetime LOG = '/var/log/wevia-l99-autofix.log' RESULT = '/var/www/html/api/l99-autofix-log.json' SAAS = '/var/www/html/l99-saas.html' def lg(m): ts = datetime.now().strftime('%H:%M') with open(LOG,'a') as f: f.write(f'{ts} {m}\n') print(f'{ts} {m}') def cmd(c, t=5): try: r = sp.run(c, shell=True, capture_output=True, text=True, timeout=t) return r.stdout.strip() except: return '' fixes = [] layers = {} def layer(name, total, passed, color='cy'): layers[name] = {'n':name, 't':total, 'p':passed, 'c':color} # ═══ 1. DOCKER ═══ lg('Check DOCKER...') expected = ['authentik-server','authentik-worker','twenty','plausible','n8n','uptime-kuma','mattermost','searxng','qdrant','vaultwarden','loki'] docker_ok = 0 docker_total = len(expected) for ct in expected: status = cmd(f"docker inspect -f '{{{{.State.Status}}}}' {ct} 2>/dev/null") if 'running' in status: docker_ok += 1 else: lg(f' Docker {ct}: {status or "NOT FOUND"} -> restarting') cmd(f'docker restart {ct} 2>/dev/null', 10) time.sleep(3) status2 = cmd(f"docker inspect -f '{{{{.State.Status}}}}' {ct} 2>/dev/null") if 'running' in status2: docker_ok += 1 fixes.append(f'Restarted {ct}') layer('DOCKER', docker_total, docker_ok, 'ro' if docker_ok < docker_total else 'cy') # ═══ 2. CAPABILITIES ═══ lg('Check CAPABILITIES...') try: r = sp.run(['curl','-sk','https://127.0.0.1/api/wevia-capabilities.php?cap=health','-H','Host: weval-consulting.com','--max-time','5'], capture_output=True, text=True, timeout=8) d = json.loads(r.stdout) svcs = d.get('services',{}) up = sum(1 for s in svcs.values() if s.get('up')) total = len(svcs) layer('CAPABILITIES', total, up, 'cy' if up >= 8 else 'am') except: layer('CAPABILITIES', 1, 0, 'ro') # ═══ 3. CRONS ═══ lg('Check CRONS...') www_data = int(cmd("sudo -u www-data crontab -l 2>/dev/null | grep -v '^#' | grep -v '^$' | wc -l") or '0') root_crons = int(cmd("crontab -l 2>/dev/null | grep -v '^#' | grep -v '^$' | wc -l") or '0') crond = int(cmd("ls /etc/cron.d/ 2>/dev/null | wc -l") or '0') cron_total = 3 cron_ok = 0 if www_data >= 20: cron_ok += 1 if root_crons >= 30: cron_ok += 1 if crond >= 5: cron_ok += 1 layer('CRONS', cron_total, cron_ok, 'cy' if cron_ok == cron_total else 'am') # ═══ 4. SYSTEMD ═══ lg('Check SYSTEMD...') systemd_svcs = ['nginx','php8.5-fpm','docker','postgresql'] sys_ok = 0 for svc in systemd_svcs: status = cmd(f'systemctl is-active {svc} 2>/dev/null') if status == 'active': sys_ok += 1 layer('SYSTEMD', len(systemd_svcs), sys_ok, 'cy') # ═══ 5. PORTS-S204 ═══ lg('Check PORTS...') ports = [(443,'HTTPS'),(5432,'PG'),(6333,'Qdrant'),(8888,'SearXNG'),(9095,"Prometheus"),(11434,'Ollama'),(5678,'n8n'),(8065,'MM')] port_ok = sum(1 for p,_ in ports if cmd(f'ss -tlnp | grep :{p} | wc -l') != '0') layer('PORTS-S204', len(ports), port_ok, 'lv') # ═══ 6. S95 ═══ lg('Check S95...') try: r = sp.run(['curl','-sf','--max-time','3','http://10.1.0.3:5890/api/sentinel-brain.php?action=exec&cmd=echo+OK'], capture_output=True, text=True, timeout=5) s95_ok = 'OK' in r.stdout except: s95_ok = False s95_ports = 0 for p in [25, 587, 5432]: if cmd(f'curl -sf --max-time 2 http://10.1.0.3:5890/api/sentinel-brain.php?action=exec&cmd=ss+-tlnp+|+grep+:{p} 2>/dev/null | grep -c {p}') != '0': s95_ports += 1 layer('S95-HEALTH', 3, (1 if s95_ok else 0) + min(s95_ports, 2), 'em') # ═══ 7. OLLAMA ═══ lg('Check OLLAMA...') try: r = sp.run(['curl','-sf','http://127.0.0.1:11434/api/tags'],capture_output=True,text=True,timeout=5) models = json.loads(r.stdout).get('models',[]) layer('SOVEREIGN', len(models), len(models), 'lv') except: layer('SOVEREIGN', 1, 0, 'ro') # ═══ 8. QDRANT ═══ lg('Check QDRANT...') try: r = sp.run(['curl','-sf','http://127.0.0.1:6333/collections'],capture_output=True,text=True,timeout=5) cols = json.loads(r.stdout).get('result',{}).get('collections',[]) layer('QDRANT', len(cols), len(cols), 'em') except: layer('QDRANT', 1, 0, 'ro') # ═══ 9. NONREG ═══ lg('Check NONREG...') try: nr = json.load(open('/var/www/html/api/nonreg-latest.json')) layer('NONREG', nr.get('total',153), nr.get('pass',0), 'em') except: layer('NONREG', 1, 0, 'ro') # ═══ UPDATE l99-saas.html with LIVE data ═══ lg('Updating l99-saas.html LAYERS...') layer_list = [layers.get(k, {'n':k,'t':1,'p':0,'c':'dm'}) for k in ['DOCKER','CAPABILITIES','CRONS','SYSTEMD','PORTS-S204','S95-HEALTH','SOVEREIGN','QDRANT','NONREG']] # Build JS js_layers = ','.join([f"{{n:'{l["n"]}',t:{l['t']},p:{l['p']},c:'{l['c']}'}}" for l in layer_list]) os.system(f'chattr -i {SAAS} 2>/dev/null') c = open(SAAS).read() # Replace LAYERS constant import re old = re.search(r'const LAYERS=\[.*?\];', c, re.DOTALL) if old: new_layers = f'const LAYERS=[{js_layers}];' c = c[:old.start()] + new_layers + c[old.end():] # Also update DATA total_t = sum(l['t'] for l in layer_list) total_p = sum(l['p'] for l in layer_list) total_f = total_t - total_p c = re.sub(r'let DATA=\{.*?\};', f'let DATA={{tests:{total_t},pass:{total_p},fail:{total_f},warn:0,layers:{len(layer_list)},ss:14,vid:32}};', c) open(SAAS, 'w').write(c) os.system(f'chattr +i {SAAS} 2>/dev/null') lg(f'LAYERS updated: {total_p}/{total_t} ({len(layer_list)} layers)') else: lg('LAYERS constant not found!') os.system(f'chattr +i {SAAS} 2>/dev/null') # Save autofix log result = { 'timestamp': datetime.now().isoformat(), 'layers': layers, 'fixes': fixes, 'total': sum(l['t'] for l in layers.values()), 'pass': sum(l['p'] for l in layers.values()) } json.dump(result, open(RESULT, 'w'), indent=2) lg(f'DONE: {len(fixes)} fixes | {result["pass"]}/{result["total"]}')