From 61165e93c435d7228a3ce8c6009b802bf8e191da Mon Sep 17 00:00:00 2001 From: Opus-Yacine Date: Thu, 16 Apr 2026 23:31:21 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20top-IA=20V3=20=E2=80=94=207=20intents?= =?UTF-8?q?=20nl-priority=20(audit/rgpd/few-shot/dialectical/webhook/quota?= =?UTF-8?q?s)=20=E2=80=94=2020=20capacit=C3=A9s=20chat=20=E2=80=94=20E2E?= =?UTF-8?q?=207/7=20=E2=80=94=20NR=20153/153?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra-logs/fix-20260416-topia-v3.md | 36 ++++++++++++++ top-ia/audit_log.sh | 16 ++++++ top-ia/audit_query.sh | 27 +++++++++++ top-ia/dialectical.sh | 28 +++++++++++ top-ia/few_shot.sh | 23 +++++++++ top-ia/gpu_quotas.sh | 43 +++++++++++++++++ top-ia/rgpd_forget.sh | 75 +++++++++++++++++++++++++++++ top-ia/webhook_send.sh | 21 ++++++++ 8 files changed, 269 insertions(+) create mode 100644 infra-logs/fix-20260416-topia-v3.md create mode 100755 top-ia/audit_log.sh create mode 100755 top-ia/audit_query.sh create mode 100755 top-ia/dialectical.sh create mode 100755 top-ia/few_shot.sh create mode 100755 top-ia/gpu_quotas.sh create mode 100755 top-ia/rgpd_forget.sh create mode 100755 top-ia/webhook_send.sh diff --git a/infra-logs/fix-20260416-topia-v3.md b/infra-logs/fix-20260416-topia-v3.md new file mode 100644 index 000000000..e8ec035e3 --- /dev/null +++ b/infra-logs/fix-20260416-topia-v3.md @@ -0,0 +1,36 @@ +# Extension 16 avril 2026 (23h30) — 7 intents top-IA V3 + +## Intents ajoutés +- top_ia_audit → audit_log.sh (append-only JSONL) +- top_ia_audit_query → audit_query.sh (search audit trail) +- top_ia_rgpd_forget → rgpd_forget.sh (Qdrant+proc purge, dry-run par défaut) +- top_ia_few_shot → few_shot.sh (top 3 similar from memory+kb 768dim) +- top_ia_dialectical → dialectical.sh (pro/con/arbiter 3 providers) +- top_ia_webhook → webhook_send.sh (POST + signature SHA256) +- top_ia_quotas → gpu_quotas.sh (8 providers status) + +## Validation E2E chat +7/7 matchent et exécutent réellement. + +## Feuille de route couverte +J+15: +- Few-shot dynamique ✅ (Qdrant top3) +- Raisonnement contradictoire ✅ (pro/con/arb) +J+30: +- Webhooks sortants ✅ +J+60: +- Webhooks sortants ✅ +J+90: +- Audit trail ✅ +- Droit à l'oubli ✅ (RGPD Art.17) +Bonus: +- GPU scheduler monitor ✅ (8 providers) + +## Total top-IA now +20 intents nl-priority opus-intents.php (13 V1/V2 + 7 V3) +49 intents total opus-intents (was 42) + +## RGPD +Audit trail immutable (Art.30), right to forget dry-run par défaut depuis chat (safety). + +## NR 153/153 préservé. diff --git a/top-ia/audit_log.sh b/top-ia/audit_log.sh new file mode 100755 index 000000000..2c236144b --- /dev/null +++ b/top-ia/audit_log.sh @@ -0,0 +1,16 @@ +#!/bin/bash +ACTOR="${1:-unknown}" +ACTION="${2:-none}" +TARGET="${3:-}" +PAYLOAD="${4:-{}}" +FILE=/var/log/weval/audit/audit.jsonl +export A_ACTOR="$ACTOR" A_ACTION="$ACTION" A_TARGET="$TARGET" A_PAYLOAD="$PAYLOAD" +python3 <<'PY' >> $FILE +import json, os, datetime, socket +try: payload=json.loads(os.environ.get('A_PAYLOAD','{}')) +except: payload={"raw":os.environ.get('A_PAYLOAD','')} +entry={"ts":datetime.datetime.now().isoformat(),"actor":os.environ.get('A_ACTOR'),"action":os.environ.get('A_ACTION'),"target":os.environ.get('A_TARGET'),"host":socket.gethostname(),"payload":payload} +print(json.dumps(entry,ensure_ascii=False)) +PY +COUNT=$(wc -l < $FILE) +echo "{\"logged\":true,\"count\":$COUNT}" diff --git a/top-ia/audit_query.sh b/top-ia/audit_query.sh new file mode 100755 index 000000000..6f43142a2 --- /dev/null +++ b/top-ia/audit_query.sh @@ -0,0 +1,27 @@ +#!/bin/bash +Q="$*" +FILE=/var/log/weval/audit/audit.jsonl +[ ! -f $FILE ] && { echo '{"error":"no audit log"}'; exit 1; } +if [ -z "$Q" ]; then + TOTAL=$(wc -l < $FILE) + python3 -c " +import json +with open('$FILE') as f: lines=[l.strip() for l in f if l.strip()] +# Filter only valid JSON lines +valid=[] +for l in lines[-10:]: + try: json.loads(l); valid.append(l) + except: pass +print(json.dumps({'total_lines':len(lines),'valid_entries':len(valid),'last_5':[json.loads(v) for v in valid[-5:]]}, ensure_ascii=False)) +" +else + python3 -c " +import json, sys +with open('$FILE') as f: lines=[l.strip() for l in f if l.strip() and '$Q'.lower() in l.lower()] +valid=[] +for l in lines[-10:]: + try: valid.append(json.loads(l)) + except: pass +print(json.dumps({'query':'$Q','matches':valid}, ensure_ascii=False)) +" +fi diff --git a/top-ia/dialectical.sh b/top-ia/dialectical.sh new file mode 100755 index 000000000..28cc9fe39 --- /dev/null +++ b/top-ia/dialectical.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Raisonnement contradictoire: 2 agents (pro/con) + arbitre +Q="$*" +[ -z "$Q" ] && { echo '{"error":"need question"}'; exit 1; } +source /etc/weval/secrets.env 2>/dev/null +export Q GROQ_KEY CEREBRAS_API_KEY NVIDIA_NIM_KEY +python3 <<'PY' +import os, json, urllib.request +q = os.environ['Q'] + +def ask(provider, url, key, model, prompt): + try: + body = json.dumps({"model":model,"messages":[{"role":"user","content":prompt}],"max_tokens":180}).encode() + req = urllib.request.Request(url, data=body, headers={"Authorization":"Bearer "+key,"Content-Type":"application/json"}) + d = json.loads(urllib.request.urlopen(req, timeout=15).read()) + return d.get('choices',[{}])[0].get('message',{}).get('content','')[:400] + except Exception as e: + return f"ERR: {str(e)[:60]}" + +pro = ask("groq","https://api.groq.com/openai/v1/chat/completions",os.environ.get('GROQ_KEY',''),"llama-3.1-8b-instant", + f"Defend this position with strong arguments (max 100 words): {q}") +con = ask("nvidia","https://integrate.api.nvidia.com/v1/chat/completions",os.environ.get('NVIDIA_NIM_KEY',''),"meta/llama-3.1-8b-instruct", + f"Argue AGAINST this position with strong counter-arguments (max 100 words): {q}") +arb = ask("cerebras","https://api.cerebras.ai/v1/chat/completions",os.environ.get('CEREBRAS_API_KEY',''),"llama3.1-8b", + f"PRO: {pro}\n\nCON: {con}\n\nAs neutral arbiter, synthesize the most nuanced position (max 100 words):") + +print(json.dumps({"question":q,"pro":pro,"con":con,"arbiter":arb}, ensure_ascii=False)) +PY diff --git a/top-ia/few_shot.sh b/top-ia/few_shot.sh new file mode 100755 index 000000000..8d8ff9c31 --- /dev/null +++ b/top-ia/few_shot.sh @@ -0,0 +1,23 @@ +#!/bin/bash +Q="$*" +[ -z "$Q" ] && { echo '{"error":"need query"}'; exit 1; } +export Q +python3 <<'PY' +import os, json, urllib.request +q = os.environ['Q'] +try: + req = urllib.request.Request("http://localhost:11434/api/embeddings", data=json.dumps({"model":"nomic-embed-text","prompt":q}).encode(), headers={"Content-Type":"application/json"}) + emb = json.loads(urllib.request.urlopen(req, timeout=8).read()).get("embedding",[]) +except Exception as e: + print(json.dumps({"error":"embed: "+str(e)[:80]})); exit() +fs = [] +for col in ["wevia_memory_768","wevia_kb_768"]: + try: + body = json.dumps({"vector":emb,"limit":3,"with_payload":True}) + req = urllib.request.Request("http://localhost:6333/collections/"+col+"/points/search", data=body.encode(), headers={"Content-Type":"application/json"}) + for m in json.loads(urllib.request.urlopen(req, timeout=5).read()).get("result",[]): + fs.append({"col":col,"score":round(m.get("score",0),3),"text":str(m.get("payload",{}).get("text",m.get("payload",{})))[:250]}) + except: pass +top3 = sorted(fs, key=lambda x:-x["score"])[:3] +print(json.dumps({"query":q,"few_shot":top3},ensure_ascii=False)) +PY diff --git a/top-ia/gpu_quotas.sh b/top-ia/gpu_quotas.sh new file mode 100755 index 000000000..d00ccf28b --- /dev/null +++ b/top-ia/gpu_quotas.sh @@ -0,0 +1,43 @@ +#!/bin/bash +source /etc/weval/secrets.env 2>/dev/null +export HF_TOKEN GEMINI_KEY CEREBRAS_API_KEY GROQ_KEY NVIDIA_NIM_KEY +python3 <<'PY' +import os, json, urllib.request +result = {} + +def probe(name, url, headers=None, method="GET", body=None): + try: + req = urllib.request.Request(url, headers=headers or {}, method=method) + if body: req.data = body + resp = urllib.request.urlopen(req, timeout=8) + return {"status":resp.status, "up":True} + except urllib.error.HTTPError as e: + return {"status":e.code, "up": e.code in [200,400,401,403,405]} + except Exception as e: + return {"status":"err", "msg":str(e)[:60], "up":False} + +# HF Inference +if os.environ.get('HF_TOKEN'): + result["huggingface"] = probe("hf", "https://api-inference.huggingface.co/status", {"Authorization":"Bearer "+os.environ['HF_TOKEN']}) +# Gemini +if os.environ.get('GEMINI_KEY'): + result["gemini"] = probe("gem", f"https://generativelanguage.googleapis.com/v1beta/models?key={os.environ['GEMINI_KEY']}") +# Groq +if os.environ.get('GROQ_KEY'): + result["groq"] = probe("groq","https://api.groq.com/openai/v1/models",{"Authorization":"Bearer "+os.environ['GROQ_KEY']}) +# Cerebras +if os.environ.get('CEREBRAS_API_KEY'): + result["cerebras"] = probe("cer","https://api.cerebras.ai/v1/models",{"Authorization":"Bearer "+os.environ['CEREBRAS_API_KEY']}) +# NVIDIA +if os.environ.get('NVIDIA_NIM_KEY'): + result["nvidia"] = probe("nv","https://integrate.api.nvidia.com/v1/models",{"Authorization":"Bearer "+os.environ['NVIDIA_NIM_KEY']}) +# Ollama local +result["ollama"] = probe("oll","http://localhost:11434/api/tags") +# Qdrant +result["qdrant"] = probe("qd","http://localhost:6333/collections") +# SearXNG +result["searxng"] = probe("sxg","http://localhost:8888/healthz") + +up_count = sum(1 for v in result.values() if v.get("up")) +print(json.dumps({"providers":result,"up":up_count,"total":len(result)}, ensure_ascii=False)) +PY diff --git a/top-ia/rgpd_forget.sh b/top-ia/rgpd_forget.sh new file mode 100755 index 000000000..8f2af652c --- /dev/null +++ b/top-ia/rgpd_forget.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# RGPD Right to be Forgotten: purge data by identifier (email/phone/name) +# Usage: rgpd_forget.sh [--dry-run] +ID="$1" +DRYRUN="${2:-}" +[ -z "$ID" ] && { echo '{"error":"need identifier"}'; exit 1; } + +# Safety: require at least 5 chars to avoid purging too broad +if [ "${#ID}" -lt 5 ]; then + echo '{"error":"identifier too short (min 5 chars for safety)"}' + exit 1 +fi + +# Audit the request +/opt/weval-ops/top-ia/audit_log.sh "rgpd" "forget_request" "$ID" "{\"dry_run\":\"$DRYRUN\"}" >/dev/null 2>&1 + +python3 </dev/null 2>&1 diff --git a/top-ia/webhook_send.sh b/top-ia/webhook_send.sh new file mode 100755 index 000000000..518f1ca8c --- /dev/null +++ b/top-ia/webhook_send.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Usage: webhook_send.sh [json_payload] +URL="$1" +EVENT="${2:-event}" +PAYLOAD="${3:-{}}" +[ -z "$URL" ] && { echo '{"error":"need url"}'; exit 1; } +# Audit +/opt/weval-ops/top-ia/audit_log.sh "webhook" "send" "$URL" "{\"event\":\"$EVENT\"}" >/dev/null 2>&1 +# Send +START=$(date +%s%N) +RESP=$(curl -sS -o /tmp/wh_resp -w "%{http_code}|%{time_total}" --max-time 10 \ + -X POST "$URL" \ + -H "Content-Type: application/json" \ + -H "X-WEVIA-Event: $EVENT" \ + -H "X-WEVIA-Signature: $(echo -n "$EVENT$PAYLOAD" | sha256sum | cut -d' ' -f1)" \ + -d "{\"event\":\"$EVENT\",\"payload\":$PAYLOAD,\"source\":\"wevia-master\",\"ts\":\"$(date -Iseconds)\"}" 2>&1) +CODE=$(echo "$RESP" | cut -d'|' -f1) +TIME=$(echo "$RESP" | cut -d'|' -f2) +BODY=$(head -c 200 /tmp/wh_resp 2>/dev/null) +rm -f /tmp/wh_resp +echo "{\"url\":\"$URL\",\"http\":\"$CODE\",\"time\":\"$TIME\",\"body\":\"$(echo $BODY | head -c 150)\"}"