383 lines
17 KiB
PHP
Executable File
383 lines
17 KiB
PHP
Executable File
#!/usr/bin/env php
|
|
<?php
|
|
/**
|
|
* BRAIN PIPELINE ORCHESTRATOR
|
|
* Runs the complete cycle: Send Tests → IMAP Check → Score → Promote Winners
|
|
* Cron: every-30-min * * * * php /opt/wevads/scripts/brain-pipeline.php >> /var/log/wevads/brain-pipeline.log 2>&1
|
|
*/
|
|
$LOG = fopen('/var/log/wevads/brain-pipeline.log', 'a');
|
|
function logMsg($m) { global $LOG; $t = date('Y-m-d H:i:s'); fwrite($LOG, "[$t] $m\n"); echo "[$t] $m\n"; }
|
|
|
|
$pdo = new PDO("pgsql:host=localhost;dbname=adx_system", "admin", "admin123");
|
|
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
|
|
|
$phase = $argv[1] ?? 'full';
|
|
logMsg("=== BRAIN PIPELINE START (phase=$phase) ===");
|
|
|
|
// ============================================================
|
|
// PHASE 1: SEND TEST EMAILS (via Postfix SMTP port 25)
|
|
// ============================================================
|
|
if (in_array($phase, ['full', 'send'])) {
|
|
logMsg("--- PHASE 1: SEND TESTS ---");
|
|
|
|
$maxTests = 20;
|
|
$jobId = 'PIPE_' . date('YmdHis');
|
|
|
|
// Pick untested or least-tested active configs WITH credentials
|
|
$configs = $pdo->query("
|
|
SELECT * FROM admin.brain_send_configs
|
|
WHERE status='active' AND smtp_user IS NOT NULL AND smtp_user != ''
|
|
ORDER BY COALESCE(inbox_rate,0) ASC, RANDOM()
|
|
LIMIT 3
|
|
")->fetchAll(PDO::FETCH_ASSOC);
|
|
|
|
// Also pick from brain_configs (PMTA path)
|
|
$pmtaConfigs = $pdo->query("
|
|
SELECT * FROM admin.brain_configs
|
|
WHERE is_active=true
|
|
ORDER BY total_sent ASC, RANDOM()
|
|
LIMIT 25
|
|
")->fetchAll(PDO::FETCH_ASSOC);
|
|
|
|
$pdo->prepare("INSERT INTO admin.brain_test_jobs (job_id, status, total_configs, started_at, created_at) VALUES (?,?,?,NOW(),NOW())")
|
|
->execute([$jobId, 'running', count($configs) + count($pmtaConfigs)]);
|
|
|
|
$sent = 0; $failed = 0;
|
|
|
|
$ispMapping = [
|
|
'T-ONLINE'=>"'T-ONLINE','TONLINE'", 'GMX'=>"'GMX'", 'GMAIL'=>"'GMAIL','GOOGLE'",
|
|
'OUTLOOK'=>"'HOTMAIL','MICROSOFT','OUTLOOK'", 'HOTMAIL'=>"'HOTMAIL','MICROSOFT'",
|
|
'ZIGGO'=>"'ZIGGO'", 'ALICE'=>"'ALICE','LIBERO'", 'YAHOO'=>"'YAHOO'",
|
|
'ORANGE'=>"'ORANGE'", 'SFR'=>"'SFR'", 'FREE'=>"'FREE'", 'WEB.DE'=>"'GMX','WEBDE'",
|
|
'Yahoo'=>"'YAHOO'", 'Gmail'=>"'GMAIL','GOOGLE'", 'Outlook'=>"'HOTMAIL','MICROSOFT','OUTLOOK'",
|
|
'T-Online'=>"'T-ONLINE','TONLINE'", 'Ziggo'=>"'ZIGGO'", 'Orange'=>"'ORANGE'"
|
|
];
|
|
|
|
// --- O365 SMTP PATH ---
|
|
foreach ($configs as $c) {
|
|
$ispTarget = strtoupper($c['isp_target']);
|
|
$ispList = $ispMapping[$ispTarget] ?? $ispMapping[$c['isp_target']] ?? "'$ispTarget'";
|
|
|
|
$seed = $pdo->query("SELECT id, email, isp FROM admin.brain_seeds WHERE is_active=true AND UPPER(isp) IN ($ispList) AND password IS NOT NULL ORDER BY RANDOM() LIMIT 1")->fetch();
|
|
if (!$seed) $seed = $pdo->query("SELECT id, email, isp FROM admin.brain_seeds WHERE is_active=true AND password IS NOT NULL ORDER BY RANDOM() LIMIT 1")->fetch();
|
|
if (!$seed) continue;
|
|
|
|
$subject = 'Update #' . rand(1000,9999) . ' - ' . date('H:i');
|
|
$fromDomain = explode('@', $c['smtp_user'])[1] ?? 'onmicrosoft.com';
|
|
$body = "<html><body><p>Newsletter update " . date('Y-m-d') . "</p><p>Config test #{$c['id']}</p></body></html>";
|
|
|
|
// Send via O365 SMTP (PHPMailer)
|
|
require_once '/opt/wevads/vendor/phpmailer/phpmailer/src/PHPMailer.php';
|
|
require_once '/opt/wevads/vendor/phpmailer/phpmailer/src/SMTP.php';
|
|
require_once '/opt/wevads/vendor/phpmailer/phpmailer/src/Exception.php';
|
|
$success = false;
|
|
$error = '';
|
|
try {
|
|
$mail = new PHPMailer\PHPMailer\PHPMailer(true);
|
|
$mail->isSMTP();
|
|
$mail->Host = 'smtp.office365.com';
|
|
$mail->SMTPAuth = true;
|
|
$mail->Username = $c['smtp_user'];
|
|
$mail->Password = $c['smtp_pass'];
|
|
$mail->SMTPSecure = PHPMailer\PHPMailer\PHPMailer::ENCRYPTION_STARTTLS;
|
|
$mail->Port = 587;
|
|
$mail->Timeout = 15;
|
|
$mail->setFrom($c['smtp_user'], 'Newsletter');
|
|
$mail->addAddress($seed['email']);
|
|
$mail->Subject = $subject;
|
|
$mail->isHTML(true);
|
|
$mail->Body = $body;
|
|
if ($c['x_mailer']) $mail->XMailer = $c['x_mailer'];
|
|
else $mail->XMailer = ' ';
|
|
$mail->send();
|
|
$success = true;
|
|
} catch (\Exception $e) {
|
|
$error = $e->getMessage();
|
|
}
|
|
|
|
|
|
if ($success) {
|
|
$sent++;
|
|
logMsg(" O365 ✅ config#{$c['id']} {$c['isp_target']} → {$seed['email']}");
|
|
} else {
|
|
$failed++;
|
|
logMsg(" O365 ❌ config#{$c['id']} {$c['isp_target']}: $error");
|
|
}
|
|
usleep(500000);
|
|
}
|
|
|
|
// --- PMTA PATH ---
|
|
foreach ($pmtaConfigs as $c) {
|
|
$ispTarget = strtoupper($c['isp_target']);
|
|
$ispList = $ispMapping[$ispTarget] ?? "'$ispTarget'";
|
|
|
|
$seed = $pdo->query("SELECT id, email, isp FROM admin.brain_seeds WHERE is_active=true AND UPPER(isp) IN ($ispList) AND password IS NOT NULL ORDER BY RANDOM() LIMIT 1")->fetch();
|
|
if (!$seed) continue;
|
|
|
|
$domain = $c['domain_used'] ?: 'wevads.com';
|
|
$fromEmail = $c['from_email'] ?: "test@$domain";
|
|
$subject = ($c['subject_template'] ?: 'Update') . ' #' . rand(100,999);
|
|
$body = $c['body_template'] ?: "<html><body><p>Test #{$c['id']} - {$c['isp_target']}</p></body></html>";
|
|
|
|
$sock = @fsockopen('127.0.0.1', 25, $en, $es, 5);
|
|
if (!$sock) { $failed++; continue; }
|
|
|
|
fgets($sock);
|
|
foreach(["EHLO $domain\r\n","MAIL FROM:<$fromEmail>\r\n","RCPT TO:<{$seed['email']}>\r\n","DATA\r\n"] as $cmd) {
|
|
fwrite($sock, $cmd); fgets($sock);
|
|
}
|
|
$msg = "From: Newsletter <$fromEmail>\r\nTo: {$seed['email']}\r\nSubject: $subject\r\nContent-Type: text/html\r\n\r\n$body\r\n.\r\n";
|
|
fwrite($sock, $msg);
|
|
$resp = fgets($sock);
|
|
fwrite($sock, "QUIT\r\n");
|
|
fclose($sock);
|
|
|
|
$ok = strpos($resp, '250') !== false;
|
|
$pdo->prepare("INSERT INTO admin.brain_test_results (job_id, config_id, seed_id, send_status, send_time, subject_used, created_at) VALUES (?,?,?,?,NOW(),?,NOW())")
|
|
->execute([$jobId, $c['id'], $seed['id'], $ok ? 'sent' : 'failed', $subject]);
|
|
|
|
if ($ok) { $sent++; logMsg(" PMTA ✅ config#{$c['id']} {$c['isp_target']} → {$seed['email']}"); }
|
|
else { $failed++; logMsg(" PMTA ❌ config#{$c['id']}"); }
|
|
usleep(300000);
|
|
}
|
|
|
|
$pdo->prepare("UPDATE admin.brain_test_jobs SET status='completed', tested_count=?, inbox_count=?, completed_at=NOW() WHERE job_id=?")->execute([$sent+$failed, $sent, $jobId]);
|
|
logMsg("PHASE 1 DONE: $sent sent, $failed failed (job=$jobId)");
|
|
}
|
|
|
|
// ============================================================
|
|
// PHASE 2: IMAP CHECK (check inbox/spam for sent tests)
|
|
// ============================================================
|
|
if (in_array($phase, ['full', 'check'])) {
|
|
logMsg("--- PHASE 2: IMAP CHECK ---");
|
|
|
|
// Get recent test results that were sent but not yet checked
|
|
$tests = $pdo->query("
|
|
SELECT tr.id, tr.config_id, tr.seed_id, tr.subject_used, tr.send_time,
|
|
bs.email as seed_email, bs.password as seed_pass, bs.imap_host, bs.imap_port, bs.isp, COALESCE(bs.check_method,'imap') as check_method
|
|
FROM admin.brain_test_results tr
|
|
JOIN admin.brain_seeds bs ON bs.id = tr.seed_id
|
|
WHERE tr.send_status = 'sent'
|
|
AND tr.inbox_status IS NULL
|
|
AND tr.send_time > NOW() - INTERVAL '24 hours'
|
|
AND bs.password IS NOT NULL AND bs.password != ''
|
|
AND bs.imap_host IS NOT NULL
|
|
ORDER BY tr.send_time DESC
|
|
LIMIT 30
|
|
")->fetchAll(PDO::FETCH_ASSOC);
|
|
|
|
$inbox = 0; $spam = 0; $notfound = 0; $errors = 0;
|
|
|
|
foreach ($tests as $t) {
|
|
$subjectSearch = substr($t['subject_used'], 0, 30);
|
|
|
|
// Hybrid: Graph API or IMAP
|
|
if (($t['check_method'] ?? 'imap') === 'graph') {
|
|
$cmd = "timeout 15 python3 /opt/wevads/scripts/graph-inbox-check.py " . escapeshellarg($t['seed_email']) . " " . escapeshellarg($t['seed_pass']) . " " . escapeshellarg($subjectSearch) . " 2>&1";
|
|
$result = trim(shell_exec($cmd));
|
|
logMsg(" Graph check {$t['seed_email']}: $result");
|
|
} else {
|
|
// Legacy IMAP check
|
|
$cmd = "python3 -c \"
|
|
import imaplib, sys
|
|
try:
|
|
m = imaplib.IMAP4_SSL('{$t['imap_host']}', {$t['imap_port']})
|
|
m.login('{$t['seed_email']}', '{$t['seed_pass']}')
|
|
m.select('INBOX')
|
|
_, d = m.search(None, 'SUBJECT', '\\\"" . addslashes($subjectSearch) . "\\\"')
|
|
if d[0]:
|
|
print('INBOX')
|
|
else:
|
|
for f in ['Spam','Junk','Junk E-mail','[Gmail]/Spam','Courrier ind&AOk-sirable']:
|
|
try:
|
|
m.select(f)
|
|
_, d = m.search(None, 'SUBJECT', '\\\"" . addslashes($subjectSearch) . "\\\"')
|
|
if d[0]:
|
|
print('SPAM')
|
|
break
|
|
except: continue
|
|
else:
|
|
print('NOTFOUND')
|
|
m.logout()
|
|
except Exception as e:
|
|
print(f'ERROR:{e}')
|
|
\" 2>&1";
|
|
$result = trim(shell_exec($cmd));
|
|
} // end IMAP else
|
|
|
|
if ($result === 'INBOX') {
|
|
$pdo->exec("UPDATE admin.brain_test_results SET inbox_status='inbox', checked_at=NOW() WHERE id={$t['id']}");
|
|
$inbox++;
|
|
} elseif ($result === 'SPAM') {
|
|
$pdo->exec("UPDATE admin.brain_test_results SET inbox_status='spam', checked_at=NOW() WHERE id={$t['id']}");
|
|
$spam++;
|
|
} elseif ($result === 'NOTFOUND') {
|
|
$pdo->exec("UPDATE admin.brain_test_results SET inbox_status='pending', checked_at=NOW() WHERE id={$t['id']}");
|
|
$notfound++;
|
|
} else {
|
|
$errors++;
|
|
logMsg(" IMAP error {$t['seed_email']}: $result");
|
|
}
|
|
|
|
usleep(200000);
|
|
}
|
|
logMsg("PHASE 2 DONE: inbox=$inbox spam=$spam notfound=$notfound errors=$errors");
|
|
}
|
|
|
|
// ============================================================
|
|
// PHASE 3: SCORE CONFIGS + PROMOTE WINNERS
|
|
// ============================================================
|
|
if (in_array($phase, ['full', 'score'])) {
|
|
logMsg("--- PHASE 3: SCORE & PROMOTE ---");
|
|
|
|
// Calculate inbox_rate per config
|
|
$scored = $pdo->exec("
|
|
UPDATE admin.brain_send_configs SET
|
|
inbox_rate = sub.rate,
|
|
updated_at = NOW()
|
|
FROM (
|
|
SELECT config_id,
|
|
ROUND(COUNT(CASE WHEN inbox_status='inbox' THEN 1 END)::numeric / NULLIF(COUNT(CASE WHEN inbox_status IN ('inbox','spam') THEN 1 END), 0) * 100, 2) as rate
|
|
FROM admin.brain_test_results
|
|
WHERE send_status='sent' AND inbox_status IS NOT NULL
|
|
GROUP BY config_id
|
|
) sub
|
|
WHERE admin.brain_send_configs.id = sub.config_id
|
|
");
|
|
|
|
// Also score brain_configs (PMTA path)
|
|
$pdo->exec("
|
|
UPDATE admin.brain_configs SET
|
|
inbox_rate = sub.rate,
|
|
inbox_count = sub.inbox_ct,
|
|
spam_count = sub.spam_ct
|
|
FROM (
|
|
SELECT config_id,
|
|
ROUND(COUNT(CASE WHEN inbox_status='inbox' THEN 1 END)::numeric / NULLIF(COUNT(CASE WHEN inbox_status IN ('inbox','spam') THEN 1 END), 0) * 100, 2) as rate,
|
|
COUNT(CASE WHEN inbox_status='inbox' THEN 1 END) as inbox_ct,
|
|
COUNT(CASE WHEN inbox_status='spam' THEN 1 END) as spam_ct
|
|
FROM admin.brain_test_results
|
|
WHERE send_status='sent' AND inbox_status IS NOT NULL
|
|
GROUP BY config_id
|
|
) sub
|
|
WHERE admin.brain_configs.id = sub.config_id
|
|
");
|
|
|
|
// Promote winners (inbox_rate >= 85% with at least 3 tests)
|
|
$winners = $pdo->query("
|
|
SELECT bsc.id, bsc.send_method, bsc.isp_target, bsc.smtp_user, bsc.x_mailer, bsc.content_type, bsc.inbox_rate,
|
|
COUNT(tr.id) as total_tests
|
|
FROM admin.brain_send_configs bsc
|
|
JOIN admin.brain_test_results tr ON tr.config_id = bsc.id AND tr.send_status='sent' AND tr.inbox_status IS NOT NULL
|
|
WHERE bsc.inbox_rate >= 85
|
|
GROUP BY bsc.id, bsc.send_method, bsc.isp_target, bsc.smtp_user, bsc.x_mailer, bsc.content_type, bsc.inbox_rate
|
|
HAVING COUNT(tr.id) >= 3
|
|
")->fetchAll(PDO::FETCH_ASSOC);
|
|
|
|
$promoted = 0;
|
|
foreach ($winners as $w) {
|
|
$domain = explode('@', $w['smtp_user'] ?? '')[1] ?? '';
|
|
$stmt = $pdo->prepare("INSERT INTO public.brain_winners (send_method, from_name, from_email, from_domain, x_mailer, content_type, inbox_rate, total_tests, target_isp, created_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?,NOW()) ON CONFLICT DO NOTHING");
|
|
$stmt->execute([$w['send_method'], 'Newsletter', $w['smtp_user'], $domain, $w['x_mailer'] ?: '', $w['content_type'], $w['inbox_rate'], $w['total_tests'], $w['isp_target']]);
|
|
if ($stmt->rowCount()) $promoted++;
|
|
}
|
|
|
|
logMsg("PHASE 3 DONE: scored=$scored, new_winners=$promoted");
|
|
}
|
|
|
|
// ============================================================
|
|
// PHASE 4: SEED FACTORY - Verify + Promote
|
|
// ============================================================
|
|
if (in_array($phase, ['full', 'factory'])) {
|
|
logMsg("--- PHASE 4: SEED FACTORY ---");
|
|
|
|
// Check factory accounts IMAP
|
|
$accounts = $pdo->query("SELECT * FROM admin.seed_factory_accounts WHERE status IN ('created','verified','subscribing') LIMIT 20")->fetchAll(PDO::FETCH_ASSOC);
|
|
$verified = 0; $failedAccounts = 0;
|
|
|
|
foreach ($accounts as $a) {
|
|
$imapCheck = shell_exec("python3 -c \"
|
|
import imaplib
|
|
try:
|
|
m = imaplib.IMAP4_SSL('{$a['imap_host']}', {$a['imap_port']})
|
|
m.login('{$a['email']}', '{$a['imap_password']}')
|
|
m.logout()
|
|
print('OK')
|
|
except Exception as e:
|
|
print(f'FAIL:{e}')
|
|
\" 2>&1");
|
|
|
|
if (trim($imapCheck) === 'OK') {
|
|
if ($a['status'] === 'created') {
|
|
$pdo->exec("UPDATE admin.seed_factory_accounts SET status='verified', last_check_at=NOW(), check_error=NULL WHERE id={$a['id']}");
|
|
$verified++;
|
|
}
|
|
} else {
|
|
$pdo->prepare("UPDATE admin.seed_factory_accounts SET last_check_at=NOW(), check_error=? WHERE id=?")->execute([trim($imapCheck), $a['id']]);
|
|
$failedAccounts++;
|
|
}
|
|
}
|
|
|
|
// Promote ready seeds (verified with 3+ newsletters)
|
|
$ready = $pdo->query("SELECT * FROM admin.seed_factory_accounts WHERE status='active' AND newsletters_count >= 3")->fetchAll(PDO::FETCH_ASSOC);
|
|
$promoted = 0;
|
|
foreach ($ready as $a) {
|
|
$stmt = $pdo->prepare("INSERT INTO admin.brain_seeds (email, password, isp, imap_host, imap_port, is_active, check_status) VALUES (?,?,?,?,?,true,'valid') ON CONFLICT DO NOTHING");
|
|
$stmt->execute([$a['email'], $a['imap_password'] ?: $a['password'], $a['isp'], $a['imap_host'], $a['imap_port']]);
|
|
if ($stmt->rowCount()) { $pdo->exec("UPDATE admin.seed_factory_accounts SET status='promoted' WHERE id={$a['id']}"); $promoted++; }
|
|
}
|
|
|
|
logMsg("PHASE 4 DONE: verified=$verified failed=$failedAccounts promoted=$promoted");
|
|
}
|
|
|
|
// ============================================================
|
|
// PHASE 5: FILTER INTELLIGENCE - Learn from results
|
|
// ============================================================
|
|
if (in_array($phase, ['full', 'filter'])) {
|
|
logMsg("--- PHASE 5: FILTER LEARNING ---");
|
|
|
|
// Aggregate results by ISP + method
|
|
$stats = $pdo->query("
|
|
SELECT bsc.isp_target, bsc.send_method, bsc.x_mailer, bsc.content_type,
|
|
COUNT(CASE WHEN tr.inbox_status='inbox' THEN 1 END) as inbox_ct,
|
|
COUNT(CASE WHEN tr.inbox_status='spam' THEN 1 END) as spam_ct,
|
|
COUNT(tr.id) as total
|
|
FROM admin.brain_send_configs bsc
|
|
JOIN admin.brain_test_results tr ON tr.config_id = bsc.id AND tr.inbox_status IS NOT NULL
|
|
GROUP BY bsc.isp_target, bsc.send_method, bsc.x_mailer, bsc.content_type
|
|
HAVING COUNT(tr.id) >= 2
|
|
")->fetchAll(PDO::FETCH_ASSOC);
|
|
|
|
$learned = 0;
|
|
foreach ($stats as $s) {
|
|
$rate = $s['total'] > 0 ? round($s['inbox_ct'] / $s['total'] * 100, 1) : 0;
|
|
$pattern = json_encode(['method'=>$s['send_method'], 'x_mailer'=>$s['x_mailer'], 'content_type'=>$s['content_type']]);
|
|
|
|
$pdo->prepare("INSERT INTO admin.filter_rules (isp, filter_name, rule_type, pattern, action, confidence, last_verified, is_active)
|
|
VALUES (?, 'brain_learned', 'send_config', ?, ?, ?, NOW(), true)
|
|
ON CONFLICT (isp, rule_type, pattern) DO UPDATE SET confidence=EXCLUDED.confidence, last_verified=NOW()")
|
|
->execute([$s['isp_target'], $pattern, $rate >= 70 ? 'allow' : 'block', $rate / 100]);
|
|
$learned++;
|
|
}
|
|
|
|
// Update ISP profiles success_rate
|
|
$pdo->exec("
|
|
UPDATE admin.brain_isp_profiles SET success_rate = sub.rate, last_updated = NOW()
|
|
FROM (
|
|
SELECT bsc.isp_target,
|
|
ROUND(AVG(CASE WHEN tr.inbox_status='inbox' THEN 100 ELSE 0 END)::numeric, 2) as rate
|
|
FROM admin.brain_send_configs bsc
|
|
JOIN admin.brain_test_results tr ON tr.config_id = bsc.id AND tr.inbox_status IN ('inbox','spam')
|
|
GROUP BY bsc.isp_target
|
|
) sub
|
|
WHERE UPPER(admin.brain_isp_profiles.isp_name) = UPPER(sub.isp_target)
|
|
");
|
|
|
|
logMsg("PHASE 5 DONE: learned=$learned filter rules");
|
|
}
|
|
|
|
logMsg("=== BRAIN PIPELINE COMPLETE ===\n");
|
|
fclose($LOG);
|
|
|