MAINTENANCE: PG logrotate, seo-monitor cron, ethica log cleanup
This commit is contained in:
8
cron/gpu-model-downloader.sh
Normal file
8
cron/gpu-model-downloader.sh
Normal file
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
LOG=/opt/wevads/logs/gpu-model-downloads.log
|
||||
echo "=== GPU Model Downloads Started $(date) ===" >> $LOG
|
||||
|
||||
MODELS="llama3.3:70b qwen2.5:72b qwen2.5-coder:32b codellama:70b mixtral:8x22b deepseek-coder-v2:16b gemma2:27b phi3:14b starcoder2:15b command-r:35b"
|
||||
|
||||
for MODEL in $MODELS; do
|
||||
echo "[$(date)] Pulling $MODEL..." >
|
||||
19
cron/gpu-pull.sh
Executable file
19
cron/gpu-pull.sh
Executable file
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
LOG=/opt/wevads/logs/gpu-pull.log
|
||||
echo "=== START $(date) ===" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"llama3.3:70b","stream":false}' >> $LOG 2>&1
|
||||
echo "llama3.3:70b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"qwen2.5:72b","stream":false}' >> $LOG 2>&1
|
||||
echo "qwen2.5:72b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"qwen2.5-coder:32b","stream":false}' >> $LOG 2>&1
|
||||
echo "qwen2.5-coder:32b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"codellama:70b","stream":false}' >> $LOG 2>&1
|
||||
echo "codellama:70b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"mixtral:8x22b","stream":false}' >> $LOG 2>&1
|
||||
echo "mixtral:8x22b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"gemma2:27b","stream":false}' >> $LOG 2>&1
|
||||
echo "gemma2:27b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"starcoder2:15b","stream":false}' >> $LOG 2>&1
|
||||
echo "starcoder2:15b done $(date)" >> $LOG
|
||||
curl -s http://88.198.4.195:11434/api/pull -d '{"name":"command-r:35b","stream":false}' >> $LOG 2>&1
|
||||
echo "=== ALL DONE $(date) ===" >> $LOG
|
||||
166
cron/kb-auto-enrichment.php
Executable file
166
cron/kb-auto-enrichment.php
Executable file
@@ -0,0 +1,166 @@
|
||||
<?php
|
||||
/**
|
||||
* WEVAL Brain — KB Auto-Enrichment Cron
|
||||
* Runs every 2 hours via crontab
|
||||
* Generates new KB entries via AI providers then injects into DB
|
||||
*/
|
||||
|
||||
$LOG_FILE = '/opt/wevads/logs/kb-enrichment.log';
|
||||
function logMsg($msg) { global $LOG_FILE; file_put_contents($LOG_FILE, date('Y-m-d H:i:s')." $msg\n", FILE_APPEND); }
|
||||
|
||||
// DB connection
|
||||
try {
|
||||
$pdo = new PDO("pgsql:host=localhost;dbname=adx_system", "admin", "admin123");
|
||||
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
||||
} catch (Exception $e) {
|
||||
logMsg("DB ERROR: " . $e->getMessage());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// AI Provider config — ordered by priority
|
||||
$providers = [
|
||||
['name' => 'groq', 'url' => 'https://api.groq.com/openai/v1/chat/completions', 'model' => 'llama-3.3-70b-versatile'],
|
||||
['name' => 'cerebras', 'url' => 'https://api.cerebras.ai/v1/chat/completions', 'model' => 'llama3.1-8b'],
|
||||
['name' => 'ollama', 'url' => 'http://88.198.4.195:11434/api/chat', 'model' => 'deepseek-r1:32b'],
|
||||
];
|
||||
|
||||
// Get API keys from DB
|
||||
$keys = [];
|
||||
$rows = $pdo->query("SELECT provider_name, api_key FROM hamid_providers WHERE is_active = true AND api_key IS NOT NULL AND api_key != ''")->fetchAll(PDO::FETCH_ASSOC);
|
||||
foreach ($rows as $r) $keys[strtolower($r['provider_name'])] = $r['api_key'];
|
||||
|
||||
// Topics to generate KB for — randomly pick 3 each run
|
||||
$topics = [
|
||||
'SAP S/4HANA implementation best practices and common pitfalls',
|
||||
'Email deliverability optimization for B2B campaigns',
|
||||
'Pharmaceutical marketing compliance GDPR and local regulations',
|
||||
'Cloud architecture patterns for high availability systems',
|
||||
'Cybersecurity zero trust implementation in enterprise',
|
||||
'Data engineering pipeline design with Apache Spark and Kafka',
|
||||
'Machine learning model deployment and MLOps best practices',
|
||||
'Supply chain optimization using AI and digital twins',
|
||||
'Aerospace quality management AS9100 and lean manufacturing',
|
||||
'Automotive IATF 16949 quality system implementation',
|
||||
'React and TypeScript advanced patterns for frontend development',
|
||||
'PostgreSQL query optimization and performance tuning',
|
||||
'Docker and Kubernetes production deployment strategies',
|
||||
'Financial risk management and derivatives pricing',
|
||||
'Agile project management Scrum and Kanban at scale',
|
||||
'Digital marketing attribution and conversion optimization',
|
||||
'Human resources talent acquisition and retention strategies',
|
||||
'Moroccan digital economy and tech startup ecosystem',
|
||||
'Renewable energy technologies solar wind storage',
|
||||
'Biotechnology drug development and clinical trials',
|
||||
'IoT architecture and industrial automation Industry 4.0',
|
||||
'Blockchain smart contracts and DeFi protocols',
|
||||
'Natural language processing and transformer architectures',
|
||||
'Python advanced patterns async generators decorators',
|
||||
'Linux system administration and DevOps automation',
|
||||
'Business intelligence and data visualization best practices',
|
||||
'API design REST GraphQL gRPC best practices',
|
||||
'Microservices communication patterns and service mesh',
|
||||
'Database design normalization indexing sharding strategies',
|
||||
'CI/CD pipeline design and GitOps deployment models',
|
||||
];
|
||||
|
||||
shuffle($topics);
|
||||
$selected = array_slice($topics, 0, 3);
|
||||
|
||||
$system_prompt = "Tu es un expert technique et consultant senior. Genere exactement 3 entries de knowledge base au format JSON strict. Chaque entry doit avoir: title (court), category (un mot en snake_case), content (250-400 caracteres, dense en information technique, pas de phrases vides). Reponds UNIQUEMENT avec le JSON array, sans markdown ni explication. Format: [{\"title\":\"...\",\"category\":\"...\",\"content\":\"...\"}]";
|
||||
|
||||
$injected = 0;
|
||||
|
||||
foreach ($selected as $topic) {
|
||||
$user_msg = "Genere 3 entries knowledge base sur: $topic";
|
||||
$response = null;
|
||||
|
||||
foreach ($providers as $provider) {
|
||||
$key = $keys[$provider['name']] ?? null;
|
||||
|
||||
if ($provider['name'] === 'ollama') {
|
||||
// Ollama uses different API format
|
||||
$payload = json_encode([
|
||||
'model' => $provider['model'],
|
||||
'messages' => [
|
||||
['role' => 'system', 'content' => $system_prompt],
|
||||
['role' => 'user', 'content' => $user_msg]
|
||||
],
|
||||
'stream' => false,
|
||||
'options' => ['temperature' => 0.7]
|
||||
]);
|
||||
} else {
|
||||
if (!$key) continue;
|
||||
$payload = json_encode([
|
||||
'model' => $provider['model'],
|
||||
'messages' => [
|
||||
['role' => 'system', 'content' => $system_prompt],
|
||||
['role' => 'user', 'content' => $user_msg]
|
||||
],
|
||||
'max_tokens' => 2000,
|
||||
'temperature' => 0.7
|
||||
]);
|
||||
}
|
||||
|
||||
$ch = curl_init($provider['url']);
|
||||
$headers = ['Content-Type: application/json'];
|
||||
if ($key && $provider['name'] !== 'ollama') {
|
||||
$headers[] = "Authorization: Bearer $key";
|
||||
}
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => $payload,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 120,
|
||||
]);
|
||||
|
||||
$result = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($httpCode === 200 && $result) {
|
||||
$data = json_decode($result, true);
|
||||
if ($provider['name'] === 'ollama') {
|
||||
$text = $data['message']['content'] ?? '';
|
||||
} else {
|
||||
$text = $data['choices'][0]['message']['content'] ?? '';
|
||||
}
|
||||
|
||||
// Extract JSON from response
|
||||
if (preg_match('/\[.*\]/s', $text, $matches)) {
|
||||
$entries = json_decode($matches[0], true);
|
||||
if (is_array($entries)) {
|
||||
$response = $entries;
|
||||
logMsg("Provider {$provider['name']} OK for: $topic");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
logMsg("Provider {$provider['name']} failed (HTTP $httpCode) for: $topic");
|
||||
}
|
||||
|
||||
if ($response) {
|
||||
$stmt = $pdo->prepare("INSERT INTO knowledge_base(title, category, content, source, created_at) VALUES(?, ?, ?, ?, NOW())");
|
||||
foreach ($response as $entry) {
|
||||
if (!empty($entry['title']) && !empty($entry['category']) && !empty($entry['content'])) {
|
||||
try {
|
||||
$stmt->execute([
|
||||
substr($entry['title'], 0, 200),
|
||||
strtolower(preg_replace('/[^a-z0-9_]/', '_', strtolower($entry['category']))),
|
||||
substr($entry['content'], 0, 2000),
|
||||
'auto_enrichment_cron'
|
||||
]);
|
||||
$injected++;
|
||||
} catch (Exception $e) {
|
||||
logMsg("Insert error: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sleep(2); // Rate limiting between topics
|
||||
}
|
||||
|
||||
logMsg("Enrichment complete: $injected entries injected from " . count($selected) . " topics");
|
||||
echo "OK: $injected entries injected\n";
|
||||
?>
|
||||
@@ -9,6 +9,7 @@
|
||||
# --- Original Guardian/Sentinel (from snapshot) ---
|
||||
# --- Warmup ---
|
||||
# === WEVADS ADX Production Crons ===
|
||||
# KB Auto-Scraper - Deployed 2026-03-02
|
||||
# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/guardian-scan.php?action=full_scan > /var/log/guardian-scan.log 2>&1
|
||||
# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/sentinel-brain.php?action=scan&fix=1 >> /var/log/sentinel.log 2>&1
|
||||
# Migrated from CCX33 - All additions, no ADX changes
|
||||
@@ -22,6 +23,7 @@
|
||||
# STANDBY-SEND warmup-advance: 0 0 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=advance_day' >> /opt/wevads/logs/warmup-advance.log 2>&1
|
||||
# STANDBY-SEND warmup-execute: */15 6-22 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=execute_warmup&batch=50' >> /opt/wevads/logs/warmup-execute.log 2>&1
|
||||
# STANDBY-SEND warmup-reset: 0 0 * * * sudo -u postgres psql adx_system -c "UPDATE admin.warmup_accounts SET sent_today=0 WHERE status='warming'" >> /opt/wevads/logs/warmup-reset.log 2>&1
|
||||
# WEVIA AI Innovation Learner - Every 6h - Deployed 2026-03-02
|
||||
*/1 * * * * crontab -l > /opt/wevads/crontab-sync.txt 2>/dev/null
|
||||
*/10 * * * * /usr/local/bin/weval-watchdog
|
||||
*/15 * * * * curl -s http://localhost:5821/api/bounce-processor.php >> /var/log/wevads/bounce-cron.log 2>&1
|
||||
@@ -33,10 +35,12 @@
|
||||
*/30 * * * * curl -s "http://127.0.0.1:5821/api/conversions-collector.php?action=pull" >> /opt/wevads/logs/conversions-pull.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/public/api/brain-graph-api.php send >> /var/log/wevads/brain-graph.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/scripts/brain-pipeline.php full >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/scripts/ethica/ethica-google-verify-v2.php 50 >> /opt/wevads/logs/ethica-google-verify.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/scripts/ethica/ethica-validator.php 500 >> /opt/wevads/logs/ethica-validator.log 2>&1
|
||||
*/5 * * * * /opt/wevads/scripts/security-sentinel.sh >> /opt/wevads/logs/security-sentinel-cron.log 2>&1
|
||||
*/5 * * * * pg_isready -h 127.0.0.1 -p 5432 > /dev/null 2>&1 || sudo systemctl restart postgresql >> /opt/wevads/logs/pg-watchdog.log 2>&1
|
||||
*/5 * * * * php /opt/wevads/public/api/brain-pmta-send.php send >> /var/log/wevads/brain-pmta.log 2>&1
|
||||
*/5 * * * * php /opt/wevads/scripts/ethica/ethica-enricher.php 10000 >> /opt/wevads/logs/ethica-enricher.log 2>&1
|
||||
0 * * * * PGPASSWORD=admin123 psql -h /var/run/postgresql -U admin adx_system -c "REFRESH MATERIALIZED VIEW admin.v_affiliate_funnel; REFRESH MATERIALIZED VIEW admin.v_channel_summary;" > /dev/null 2>&1
|
||||
0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-hamid-check.php' >> /opt/wevads/logs/hamid-affiliate.log 2>&1
|
||||
0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-monitor.php?action=compare&from=2026-02-10&to=2026-02-11' >> /opt/wevads/logs/affiliate-monitor.log 2>&1
|
||||
@@ -44,6 +48,9 @@
|
||||
0 */2 * * * php /opt/wevads/public/api/kb-sync-cron.php >> /var/log/wevads/claude_kb_sync.log 2>&1
|
||||
0 */4 * * * /opt/wevads/scripts/brain-optimizer-cron.sh
|
||||
0 */4 * * * php /opt/wevads/scripts/scrape-to-sendcontacts.php >> /opt/wevads/logs/scrape-feeder.log 2>&1
|
||||
0 */6 * * * /opt/wevads/crons/wevia-ai-learner.sh
|
||||
0 */6 * * * /opt/wevads/kb-auto-enrich.sh
|
||||
0 */6 * * * /opt/wevads/scripts/rsync-backup-s88.sh
|
||||
0 */6 * * * curl -s http://127.0.0.1:5890/api/vault-guard.php?action=cron >> /opt/wevads/logs/vault-guard-cron.log 2>&1
|
||||
0 */6 * * * php /opt/wevads/public/api/brain-graph-api.php discover >> /var/log/wevads/brain-graph.log 2>&1
|
||||
0 */6 * * * php /opt/wevads/scripts/brain-creative-engine.php full_cycle >> /opt/wevads/logs/brain-creative-engine.log 2>&1
|
||||
@@ -51,17 +58,12 @@
|
||||
0 0 * * * PGPASSWORD=admin123 psql -U admin -d adx_system -c "UPDATE ethica.sms_providers SET sent_today=0" >> /opt/wevads/logs/ethica-sms-reset.log 2>&1
|
||||
0 0 * * * curl -s http://127.0.0.1:5821/api/ia-provider-factory.php?action=rotate > /dev/null 2>&1
|
||||
0 0 * * * php -r "\$d=new PDO('pgsql:host=localhost;dbname=adx_system','admin','admin123');\$d->exec('UPDATE ethica.senders SET sent_today=0');" >> /opt/wevads/logs/ethica-reset.log 2>&1
|
||||
0 22 * * * /opt/wevads/crons/kb-scraper-cron.sh
|
||||
0 3 * * * /opt/wevads/scripts/pg-backup-s88.sh >> /var/log/wevads/pg-backup.log 2>&1
|
||||
0 3 1,15 * * php /opt/wevads/scripts/ethica/ethica-scraper.php all all >> /opt/wevads/logs/ethica-scraper.log 2>&1
|
||||
0 6 * * * php /opt/wevads/scripts/brain-pipeline.php factory >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
15,45 * * * * php /opt/wevads/public/api/brain-graph-api.php check >> /var/log/wevads/brain-graph.log 2>&1
|
||||
15,45 * * * * php /opt/wevads/scripts/brain-pipeline.php check >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
30 */6 * * * php /opt/wevads/scripts/ethica/ethica-mega-scraper.php all all directories >> /opt/wevads/logs/ethica-scraper-continuous.log 2>&1
|
||||
30 0 */3 * * /opt/wevads/scripts/offer-refresh-cron.sh
|
||||
*/5 * * * * php /opt/wevads/scripts/ethica/ethica-enricher.php 10000 >> /opt/wevads/logs/ethica-enricher.log 2>&1
|
||||
# KB Auto-Scraper - Deployed 2026-03-02
|
||||
0 22 * * * /opt/wevads/crons/kb-scraper-cron.sh
|
||||
# WEVIA AI Innovation Learner - Every 6h - Deployed 2026-03-02
|
||||
0 */6 * * * /opt/wevads/crons/wevia-ai-learner.sh
|
||||
0 3 * * * /opt/wevads/scripts/pg-backup-s88.sh >> /var/log/wevads/pg-backup.log 2>&1
|
||||
0 */6 * * * /opt/wevads/scripts/rsync-backup-s88.sh
|
||||
0 */6 * * * /opt/wevads/kb-auto-enrich.sh
|
||||
0 */2 * * * php /opt/wevads/cron/kb-auto-enrichment.php >> /opt/wevads/logs/kb-enrichment.log 2>&1
|
||||
|
||||
146
scripts/kb-auto-enrich.php
Normal file
146
scripts/kb-auto-enrich.php
Normal file
@@ -0,0 +1,146 @@
|
||||
<?php
|
||||
/**
|
||||
* WEVAL Brain — KB Auto-Enrichment Cron
|
||||
* Runs every 2 hours to generate new KB entries via AI providers
|
||||
* Injects into knowledge_base + brain_knowledge
|
||||
* Uses WEVAL Brain multi-provider failover
|
||||
*/
|
||||
|
||||
$LOG_FILE = '/opt/wevads/logs/kb-auto-enrich.log';
|
||||
function logMsg($msg) { global $LOG_FILE; file_put_contents($LOG_FILE, date('Y-m-d H:i:s') . " $msg\n", FILE_APPEND); }
|
||||
|
||||
logMsg("=== KB AUTO-ENRICHMENT START ===");
|
||||
|
||||
// DB connection
|
||||
include("/opt/wevads/config/credentials.php");
|
||||
$pdo = get_pdo();
|
||||
|
||||
// AI Provider configs (failover order)
|
||||
$providers = [
|
||||
['name' => 'groq', 'url' => 'https://api.groq.com/openai/v1/chat/completions', 'model' => 'llama-3.3-70b-versatile', 'key_env' => 'GROQ_API_KEY'],
|
||||
['name' => 'cerebras', 'url' => 'https://api.cerebras.ai/v1/chat/completions', 'model' => 'llama3.1-8b', 'key_env' => 'CEREBRAS_API_KEY'],
|
||||
['name' => 'ollama', 'url' => 'http://88.198.4.195:11434/v1/chat/completions', 'model' => 'deepseek-r1:32b', 'key_env' => ''],
|
||||
];
|
||||
|
||||
// Get API keys from DB
|
||||
$keys = [];
|
||||
$stmt = $pdo->query("SELECT LOWER(provider_name) as name, api_key FROM hamid_providers WHERE is_active=true AND api_key != ''");
|
||||
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
|
||||
$keys[strtolower($row['name'])] = $row['api_key'];
|
||||
}
|
||||
|
||||
// Topics to cycle through (one per run)
|
||||
$topics = [
|
||||
'coding' => ['PHP design patterns avancés', 'JavaScript async patterns', 'Python data structures', 'SQL optimization techniques', 'API security best practices', 'Microservices communication patterns', 'Database indexing strategies', 'CI/CD pipeline optimization', 'Container orchestration patterns', 'Serverless architecture patterns'],
|
||||
'consulting' => ['Digital transformation KPIs', 'Stakeholder management strategies', 'Business process reengineering', 'Strategic planning frameworks', 'Organizational design patterns', 'Post-merger integration', 'IT governance frameworks', 'Vendor management best practices'],
|
||||
'ia_ml' => ['LLM prompt engineering techniques', 'Vector database optimization', 'ML model deployment strategies', 'AI ethics and governance', 'Computer vision applications industrielles', 'NLP for business intelligence', 'Reinforcement learning applications'],
|
||||
'pharma' => ['Drug development lifecycle', 'Pharmacovigilance automation', 'Clinical trial design', 'GMP compliance digital', 'Pharmaceutical supply chain', 'Biosimilar development', 'Real-world evidence'],
|
||||
'cybersecurite' => ['Incident response playbooks', 'Cloud security posture', 'API security testing', 'Identity governance', 'Threat modeling methodologies', 'Security automation SOAR'],
|
||||
'sap' => ['SAP BTP extension development', 'S/4HANA data migration', 'SAP Fiori development patterns', 'SAP Integration Suite patterns', 'ABAP Cloud programming model'],
|
||||
'sciences' => ['Quantum computing algorithms', 'Climate modeling techniques', 'CRISPR applications 2026', 'Neuroscience discoveries', 'Materials science innovations'],
|
||||
'medecine' => ['Precision medicine advances', 'Digital health technologies', 'AI in medical imaging', 'Antimicrobial stewardship', 'Telemedicine best practices'],
|
||||
'email_marketing' => ['Deliverability monitoring automation', 'Email authentication advanced', 'ISP relationship management', 'Inbox placement optimization', 'Email personalization at scale'],
|
||||
'cloud' => ['Multi-cloud networking', 'Cloud cost optimization FinOps', 'Serverless patterns avancés', 'Cloud-native security', 'Platform engineering'],
|
||||
'philosophie' => ['Ethics of artificial intelligence', 'Philosophy of consciousness', 'Epistemology and data science', 'Bioethics contemporary debates'],
|
||||
'art_culture' => ['Digital art and NFTs', 'Architecture durable', 'Patrimoine culturel numerique', 'Design systems principles'],
|
||||
'finance' => ['Algorithmic trading strategies', 'ESG investment frameworks', 'Decentralized finance risks', 'Islamic finance principles'],
|
||||
'energie' => ['Smart grid technologies', 'Carbon capture innovations', 'Nuclear fusion progress 2026', 'Green hydrogen economics'],
|
||||
];
|
||||
|
||||
// Pick random category and topic
|
||||
$categories = array_keys($topics);
|
||||
$cat = $categories[array_rand($categories)];
|
||||
$topic = $topics[$cat][array_rand($topics[$cat])];
|
||||
logMsg("Selected: [$cat] $topic");
|
||||
|
||||
// Build prompt
|
||||
$prompt = "Generate 3 knowledge base entries about: $topic
|
||||
Category: $cat
|
||||
|
||||
For each entry, provide:
|
||||
1. A concise title (max 80 chars)
|
||||
2. A detailed content paragraph (200-400 chars) with specific facts, numbers, frameworks, or techniques
|
||||
|
||||
Respond ONLY in this JSON format, no markdown:
|
||||
[{\"title\":\"...\",\"content\":\"...\"},{\"title\":\"...\",\"content\":\"...\"},{\"title\":\"...\",\"content\":\"...\"}]";
|
||||
|
||||
// Call AI provider with failover
|
||||
$result = null;
|
||||
foreach ($providers as $prov) {
|
||||
$key = $keys[$prov['name']] ?? ($prov['key_env'] ? getenv($prov['key_env']) : 'dummy');
|
||||
if (!$key && $prov['name'] !== 'ollama') continue;
|
||||
|
||||
$headers = ['Content-Type: application/json'];
|
||||
if ($prov['name'] !== 'ollama') $headers[] = "Authorization: Bearer $key";
|
||||
|
||||
$payload = json_encode([
|
||||
'model' => $prov['model'],
|
||||
'messages' => [
|
||||
['role' => 'system', 'content' => 'You are a knowledge base generator. Respond ONLY with valid JSON arrays. No markdown, no explanations.'],
|
||||
['role' => 'user', 'content' => $prompt]
|
||||
],
|
||||
'max_tokens' => 1500,
|
||||
'temperature' => 0.7
|
||||
]);
|
||||
|
||||
$ch = curl_init($prov['url']);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => $payload,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 120,
|
||||
]);
|
||||
$resp = curl_exec($ch);
|
||||
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($code === 200 && $resp) {
|
||||
$data = json_decode($resp, true);
|
||||
$text = $data['choices'][0]['message']['content'] ?? '';
|
||||
// Clean potential markdown fences
|
||||
$text = preg_replace('/```json\s*/', '', $text);
|
||||
$text = preg_replace('/```\s*/', '', $text);
|
||||
$text = trim($text);
|
||||
$entries = json_decode($text, true);
|
||||
if (is_array($entries) && count($entries) > 0) {
|
||||
$result = $entries;
|
||||
logMsg("Provider {$prov['name']} OK: " . count($entries) . " entries");
|
||||
break;
|
||||
}
|
||||
}
|
||||
logMsg("Provider {$prov['name']} failed (HTTP $code)");
|
||||
}
|
||||
|
||||
if (!$result) {
|
||||
logMsg("ERROR: All providers failed");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Insert into knowledge_base + brain_knowledge
|
||||
$kb_inserted = 0;
|
||||
$bk_inserted = 0;
|
||||
foreach ($result as $entry) {
|
||||
$title = substr($entry['title'] ?? '', 0, 200);
|
||||
$content = $entry['content'] ?? '';
|
||||
if (!$title || !$content) continue;
|
||||
|
||||
// knowledge_base
|
||||
$stmt = $pdo->prepare("INSERT INTO knowledge_base (title, category, content, author, source, created_at) VALUES (?, ?, ?, 'WEVAL Brain Auto', 'auto-enrichment', NOW()) ON CONFLICT DO NOTHING");
|
||||
$stmt->execute([$title, $cat, $content]);
|
||||
if ($stmt->rowCount() > 0) $kb_inserted++;
|
||||
|
||||
// brain_knowledge (key = slugified title)
|
||||
$key = strtolower(preg_replace('/[^a-z0-9]+/i', '_', $title));
|
||||
$key = substr($key, 0, 100);
|
||||
$stmt = $pdo->prepare("INSERT INTO brain_knowledge (category, key, value, confidence, last_updated) VALUES (?, ?, ?, 0.85, NOW()) ON CONFLICT (category, key) DO UPDATE SET value=EXCLUDED.value, last_updated=NOW()");
|
||||
$stmt->execute([$cat, $key, substr($content, 0, 500)]);
|
||||
$bk_inserted++;
|
||||
}
|
||||
|
||||
$total_kb = $pdo->query("SELECT count(*) FROM knowledge_base")->fetchColumn();
|
||||
$total_bk = $pdo->query("SELECT count(*) FROM brain_knowledge")->fetchColumn();
|
||||
logMsg("INJECTED: KB+$kb_inserted BK+$bk_inserted | TOTALS: KB=$total_kb BK=$total_bk");
|
||||
logMsg("=== KB AUTO-ENRICHMENT END ===");
|
||||
echo "OK: KB+$kb_inserted BK+$bk_inserted | TOTALS: KB=$total_kb BK=$total_bk\n";
|
||||
?>
|
||||
@@ -363,3 +363,4 @@
|
||||
196.206.88.151
|
||||
196.206.88.151
|
||||
196.206.88.151
|
||||
196.206.88.151
|
||||
|
||||
Reference in New Issue
Block a user