270 lines
10 KiB
Python
270 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
WEVAL OSS Master Pipeline v2.0
|
|
COMPLETE cycle: WIKI SCAN → SERVER SCAN → OSS → AGENTS → SKILLS → SYNC → VALIDATE → GIT → WIKI UPDATE
|
|
|
|
Stage 0: READ wiki (existing state, known tools, gaps)
|
|
Stage 1: SCAN servers (4 machines: S204, S95, S151, Blade)
|
|
Stage 2: OSS discovery (/opt/* new tools)
|
|
Stage 3: CREATE Paperclip agents (SQL direct)
|
|
Stage 4: SKILL sync check (Qdrant has its own cron)
|
|
Stage 5: DeerFlow symlinks
|
|
Stage 6: Enterprise-model sync
|
|
Stage 7: VALIDATE (Playwright + NonReg)
|
|
Stage 8: GIT commit + push
|
|
Stage 9: WIKI UPDATE (write back what changed)
|
|
|
|
Cron: 30 4 * * * (daily before benchmark at 5AM)
|
|
"""
|
|
import json, os, subprocess, urllib.request, uuid, glob
|
|
from datetime import datetime
|
|
|
|
LOG_FILE = "/tmp/oss-pipeline-v2.log"
|
|
WIKI_DIR = "/opt/weval-l99/wiki"
|
|
REGISTRY = "/opt/weval-l99/registry/registry.json"
|
|
PAPERCLIP = "http://127.0.0.1:3100"
|
|
COMPANY = "dd12987b-c774-45e7-95fd-d34003f91650"
|
|
QDRANT = "http://127.0.0.1:6333"
|
|
EM_FILE = "/var/www/html/enterprise-model.html"
|
|
LOG = []
|
|
|
|
def log(msg):
|
|
ts = datetime.now().strftime("%H:%M:%S")
|
|
line = f"[{ts}] {msg}"
|
|
LOG.append(line)
|
|
print(line)
|
|
|
|
def api(url, timeout=5):
|
|
try:
|
|
r = urllib.request.urlopen(url, timeout=timeout)
|
|
return json.loads(r.read())
|
|
except: return None
|
|
|
|
def cmd(c, timeout=15):
|
|
try:
|
|
r = subprocess.run(c, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return r.stdout.strip()
|
|
except: return ""
|
|
|
|
# ═══ STAGE 0: READ WIKI ═══
|
|
def stage_wiki_read():
|
|
log("=== STAGE 0: READ WIKI ===")
|
|
wiki = {}
|
|
for f in glob.glob(f"{WIKI_DIR}/*.json"):
|
|
try:
|
|
d = json.load(open(f))
|
|
wiki[os.path.basename(f)] = d
|
|
except: pass
|
|
log(f" Wiki entries: {len(wiki)}")
|
|
# Read registry if exists
|
|
reg = {}
|
|
if os.path.exists(REGISTRY):
|
|
try: reg = json.load(open(REGISTRY))
|
|
except: pass
|
|
log(f" Registry: {len(reg)} sections")
|
|
# Known agents from last run
|
|
agents_known = reg.get("agents", {}).get("total", 0)
|
|
skills_known = reg.get("skills", {}).get("qdrant_vectors", 0)
|
|
log(f" Known state: {agents_known} agents, {skills_known} skills")
|
|
return {"wiki": wiki, "registry": reg, "agents_before": agents_known, "skills_before": skills_known}
|
|
|
|
# ═══ STAGE 1: SCAN SERVERS ═══
|
|
def stage_server_scan():
|
|
log("=== STAGE 1: SCAN SERVERS ===")
|
|
servers = {}
|
|
# S204 (local)
|
|
docker = int(cmd("docker ps -q | wc -l") or 0)
|
|
ports = int(cmd("ss -tlnp | grep LISTEN | wc -l") or 0)
|
|
disk = cmd("df -h / | tail -1").split()
|
|
disk_pct = disk[4] if len(disk) > 4 else "?"
|
|
servers["S204"] = {"docker": docker, "ports": ports, "disk": disk_pct, "status": "UP"}
|
|
log(f" S204: {docker} Docker, {ports} ports, {disk_pct} disk")
|
|
# S95 (via Sentinel)
|
|
try:
|
|
r = urllib.request.urlopen("http://10.1.0.3:5890/api/sentinel-brain.php?action=exec&cmd=uptime", timeout=5)
|
|
servers["S95"] = {"status": "UP"}
|
|
log(f" S95: UP")
|
|
except:
|
|
servers["S95"] = {"status": "DOWN"}
|
|
log(f" S95: DOWN or unreachable")
|
|
# S151 (OVH)
|
|
r151 = cmd("ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no ubuntu@151.80.235.110 uptime 2>/dev/null")
|
|
servers["S151"] = {"status": "UP" if r151 else "UNREACHABLE"}
|
|
log(f" S151: {'UP' if r151 else 'UNREACHABLE'}")
|
|
return servers
|
|
|
|
# ═══ STAGE 2: OSS DISCOVERY ═══
|
|
def stage_oss_scan():
|
|
log("=== STAGE 2: OSS DISCOVERY ===")
|
|
OSS_MAP = {
|
|
"deepagent": ("orchestrator","Deep research agent"),
|
|
"HolyClaude": ("dev","Claude enhancement"),
|
|
"skillsmith": ("dev","Skill generator"),
|
|
"goose": ("dev","AI coding agent"),
|
|
"wazuh": ("security","SIEM monitoring"),
|
|
"claude-mem": ("dev","Persistent memory"),
|
|
"keyhacks": ("security","API key validation"),
|
|
"weval-radar": ("analyst","Technology radar"),
|
|
"weval-security": ("security","Security audit"),
|
|
"flowise-data": ("orchestrator","Visual AI builder"),
|
|
"antigravity": ("dev","4198 skills toolkit"),
|
|
"mirofish": ("ceo","Swarm Intelligence Engine"),
|
|
}
|
|
found = {}
|
|
for name, (role, desc) in OSS_MAP.items():
|
|
if os.path.isdir(f"/opt/{name}"):
|
|
found[name] = {"role": role, "desc": desc}
|
|
log(f" Found {len(found)} OSS tools")
|
|
return found
|
|
|
|
# ═══ STAGE 3: CREATE PAPERCLIP AGENTS (SQL) ═══
|
|
def stage_create_agents(oss_tools):
|
|
log("=== STAGE 3: CREATE PAPERCLIP AGENTS ===")
|
|
agents_raw = api(f"{PAPERCLIP}/api/companies/{COMPANY}/agents")
|
|
agents = agents_raw if isinstance(agents_raw, list) else (agents_raw or {}).get("data", [])
|
|
existing = {a.get("name","").lower() for a in agents}
|
|
log(f" Existing: {len(existing)} agents")
|
|
created = 0
|
|
for name, info in oss_tools.items():
|
|
agent_name = name.replace("-","").replace("_","").title()[:30]
|
|
if agent_name.lower() in existing: continue
|
|
uid = str(uuid.uuid4())
|
|
sql = f"INSERT INTO agents (id,company_id,name,role,status,capabilities,adapter_type,adapter_config,runtime_config,permissions,budget_monthly_cents,spent_monthly_cents,created_at,updated_at) VALUES ('{uid}'::uuid,'{COMPANY}'::uuid,'{agent_name}','{info['role']}','active','[OSS] {info['desc']}','process','{{}}'::jsonb,'{{}}'::jsonb,'{{}}'::jsonb,0,0,NOW(),NOW())"
|
|
r = subprocess.run(["sudo","-u","postgres","psql","-d","paperclip","-c",sql], capture_output=True, text=True, timeout=5)
|
|
if r.returncode == 0 and "INSERT" in r.stdout:
|
|
created += 1
|
|
log(f" Created: {agent_name}")
|
|
log(f" Total created: {created}")
|
|
return created
|
|
|
|
# ═══ STAGE 4: SKILLS CHECK ═══
|
|
def stage_skills_check():
|
|
log("=== STAGE 4: SKILLS CHECK ===")
|
|
d = api(f"{QDRANT}/collections/weval_skills")
|
|
count = d.get("result",{}).get("points_count",0) if d else 0
|
|
log(f" Qdrant: {count} vectors (sync runs on its own cron */6h)")
|
|
return count
|
|
|
|
# ═══ STAGE 5: DEERFLOW LINKS ═══
|
|
def stage_deerflow():
|
|
log("=== STAGE 5: DEERFLOW LINKS ===")
|
|
df_skills = "/opt/deer-flow/skills"
|
|
if not os.path.isdir(df_skills):
|
|
log(" DeerFlow skills dir missing")
|
|
return 0
|
|
links = len(os.listdir(df_skills))
|
|
log(f" DeerFlow: {links} skill dirs")
|
|
return links
|
|
|
|
# ═══ STAGE 6: ENTERPRISE-MODEL SYNC ═══
|
|
def stage_enterprise_sync():
|
|
log("=== STAGE 6: ENTERPRISE-MODEL SYNC ===")
|
|
sync = "/opt/weval-l99/enterprise-model-sync.py"
|
|
if os.path.exists(sync):
|
|
r = subprocess.run(["python3", sync], capture_output=True, text=True, timeout=30)
|
|
log(f" Sync: {r.stdout.strip()[:100]}")
|
|
return True
|
|
log(" enterprise-model-sync.py not found")
|
|
return False
|
|
|
|
# ═══ STAGE 7: VALIDATE ═══
|
|
def stage_validate():
|
|
log("=== STAGE 7: VALIDATE ===")
|
|
# NonReg
|
|
nr = api("https://weval-consulting.com/api/nonreg-api.php?cat=all")
|
|
if nr:
|
|
log(f" NonReg: {nr.get('pass')}/{nr.get('total')} ({nr.get('score')}%)")
|
|
# Chatbot
|
|
try:
|
|
req = urllib.request.Request("https://weval-consulting.com/api/weval-ia-fast.php",
|
|
data=json.dumps({"message":"test"}).encode(),
|
|
headers={"Content-Type":"application/json"})
|
|
r = urllib.request.urlopen(req, timeout=15)
|
|
d = json.loads(r.read())
|
|
log(f" Chatbot: {d.get('provider','?')} {len(d.get('response',''))}ch")
|
|
except Exception as e:
|
|
log(f" Chatbot: {e}")
|
|
return True
|
|
|
|
# ═══ STAGE 8: GIT ═══
|
|
def stage_git():
|
|
log("=== STAGE 8: GIT ===")
|
|
r = subprocess.run(["git","-C","/var/www/html","add","-A"], capture_output=True, text=True, timeout=10)
|
|
r2 = subprocess.run(["git","-C","/var/www/html","commit","-m","PIPELINE: auto-sync"], capture_output=True, text=True, timeout=10)
|
|
log(f" Git: {r2.stdout.strip()[:80]}")
|
|
return True
|
|
|
|
# ═══ STAGE 9: WIKI UPDATE ═══
|
|
def stage_wiki_update(state):
|
|
log("=== STAGE 9: WIKI UPDATE ===")
|
|
wiki_entry = {
|
|
"file": "pipeline-run-latest",
|
|
"type": "pipeline",
|
|
"timestamp": datetime.now().isoformat(),
|
|
"stages": {
|
|
"wiki_read": True,
|
|
"server_scan": state.get("servers", {}),
|
|
"oss_found": state.get("oss_count", 0),
|
|
"agents_created": state.get("agents_created", 0),
|
|
"agents_total": state.get("agents_total", 0),
|
|
"skills_qdrant": state.get("skills", 0),
|
|
"deerflow_links": state.get("deerflow", 0),
|
|
"enterprise_synced": state.get("enterprise", False),
|
|
"validated": state.get("validated", False),
|
|
"git": True,
|
|
},
|
|
"delta": {
|
|
"agents": f"{state.get('agents_before',0)} -> {state.get('agents_total',0)}",
|
|
"skills": f"{state.get('skills_before',0)} -> {state.get('skills',0)}",
|
|
}
|
|
}
|
|
with open(f"{WIKI_DIR}/pipeline-run-latest.json", "w") as f:
|
|
json.dump(wiki_entry, f, indent=2, default=str)
|
|
log(f" Wiki updated: pipeline-run-latest.json")
|
|
# Also update registry
|
|
subprocess.run(["python3","/opt/weval-l99/registry-master.py"], capture_output=True, text=True, timeout=30)
|
|
log(f" Registry refreshed")
|
|
return True
|
|
|
|
# ═══ MAIN ═══
|
|
def main():
|
|
log("=" * 60)
|
|
log("WEVAL OSS Master Pipeline v2.0")
|
|
log("=" * 60)
|
|
state = {}
|
|
# Stage 0: Wiki read
|
|
wiki_state = stage_wiki_read()
|
|
state["agents_before"] = wiki_state["agents_before"]
|
|
state["skills_before"] = wiki_state["skills_before"]
|
|
# Stage 1: Server scan
|
|
state["servers"] = stage_server_scan()
|
|
# Stage 2: OSS
|
|
oss = stage_oss_scan()
|
|
state["oss_count"] = len(oss)
|
|
# Stage 3: Agents
|
|
state["agents_created"] = stage_create_agents(oss)
|
|
agents_raw = api(f"{PAPERCLIP}/api/companies/{COMPANY}/agents")
|
|
agents = agents_raw if isinstance(agents_raw, list) else (agents_raw or {}).get("data", [])
|
|
state["agents_total"] = len(agents)
|
|
# Stage 4: Skills
|
|
state["skills"] = stage_skills_check()
|
|
# Stage 5: DeerFlow
|
|
state["deerflow"] = stage_deerflow()
|
|
# Stage 6: Enterprise
|
|
state["enterprise"] = stage_enterprise_sync()
|
|
# Stage 7: Validate
|
|
state["validated"] = stage_validate()
|
|
# Stage 8: Git
|
|
stage_git()
|
|
# Stage 9: Wiki update
|
|
stage_wiki_update(state)
|
|
log("=" * 60)
|
|
log(f"PIPELINE v2.0 COMPLETE: {json.dumps({k:v for k,v in state.items() if k != 'servers'})}")
|
|
log("=" * 60)
|
|
# Write log
|
|
with open(LOG_FILE, "w") as f:
|
|
f.write("\n".join(LOG))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|