diff --git a/crontab-backup-pre-factory.txt b/crontab-backup-pre-factory.txt new file mode 100644 index 00000000..48a54539 --- /dev/null +++ b/crontab-backup-pre-factory.txt @@ -0,0 +1,59 @@ + +# --- Brain Optimization --- +# --- Brain Send --- +# --- Brain Tracking --- +# --- E2E Pipeline --- +# --- Ethica (B2B Medical) --- +# --- Matviews Refresh --- +# --- Monitoring & Maintenance --- +# --- Original Guardian/Sentinel (from snapshot) --- +# --- Warmup --- +# === WEVADS ADX Production Crons === +# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/guardian-scan.php?action=full_scan > /var/log/guardian-scan.log 2>&1 +# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/sentinel-brain.php?action=scan&fix=1 >> /var/log/sentinel.log 2>&1 +# Migrated from CCX33 - All additions, no ADX changes +# PG TCP Watchdog - restart if TCP down +# STANDBY-SEND ethica-send: 0 14 * * 1-5 php /opt/wevads/scripts/ethica/ethica-send.php auto 200 >> /opt/wevads/logs/ethica-send.log 2>&1 +# STANDBY-SEND ethica-send: 0 9 * * 1-5 php /opt/wevads/scripts/ethica/ethica-send.php auto 200 >> /opt/wevads/logs/ethica-send.log 2>&1 +# STANDBY-SEND ethica-sms: 0 10 * * 1-5 php /opt/wevads/scripts/ethica/ethica-sms-send.php auto 500 >> /opt/wevads/logs/ethica-sms.log 2>&1 +# STANDBY-SEND ethica-sms: 0 15 * * 1-5 php /opt/wevads/scripts/ethica/ethica-sms-send.php auto 500 >> /opt/wevads/logs/ethica-sms.log 2>&1 +# STANDBY-SEND tracking-seeds-opens: 30 */2 * * * php /opt/wevads/public/api/brain-tracking-seeds.php check_opens >> /var/log/wevads/brain-graph.log 2>&1 +# STANDBY-SEND tracking-seeds-send: 0 */2 * * * php /opt/wevads/public/api/brain-tracking-seeds.php send_tracked >> /var/log/wevads/brain-graph.log 2>&1 +# STANDBY-SEND warmup-advance: 0 0 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=advance_day' >> /opt/wevads/logs/warmup-advance.log 2>&1 +# STANDBY-SEND warmup-execute: */15 6-22 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=execute_warmup&batch=50' >> /opt/wevads/logs/warmup-execute.log 2>&1 +# STANDBY-SEND warmup-reset: 0 0 * * * sudo -u postgres psql adx_system -c "UPDATE admin.warmup_accounts SET sent_today=0 WHERE status='warming'" >> /opt/wevads/logs/warmup-reset.log 2>&1 +*/1 * * * * crontab -l > /opt/wevads/crontab-sync.txt 2>/dev/null +*/10 * * * * /usr/local/bin/weval-watchdog +*/15 * * * * curl -s http://localhost:5821/api/bounce-processor.php >> /var/log/wevads/bounce-cron.log 2>&1 +*/15 * * * * php /opt/wevads/public/api/brain-productive.php send >> /var/log/wevads/brain-productive.log 2>&1 +*/15 * * * * sudo -u postgres psql adx_system -c "UPDATE admin.ia_provider_accounts SET status='active' WHERE status='rate_limited' AND cooldown_until /dev/null 2>&1 +*/15 7-22 * * * /usr/bin/bash /opt/wevads/scripts/send_batch.sh 200 >> /opt/wevads/logs/send_cron_output.log 2>&1 +*/30 * * * * /opt/wevads/scripts/mind-autonomous.sh >> /opt/wevads/logs/mind-cron.log 2>&1 +*/30 * * * * /opt/wevads/scripts/sentinel-autorepair.sh --analyze-only +*/30 * * * * curl -s "http://127.0.0.1:5821/api/conversions-collector.php?action=pull" >> /opt/wevads/logs/conversions-pull.log 2>&1 +*/30 * * * * php /opt/wevads/public/api/brain-graph-api.php send >> /var/log/wevads/brain-graph.log 2>&1 +*/30 * * * * php /opt/wevads/scripts/brain-pipeline.php full >> /var/log/wevads/brain-pipeline.log 2>&1 +*/30 * * * * php /opt/wevads/scripts/ethica/ethica-validator.php 500 >> /opt/wevads/logs/ethica-validator.log 2>&1 +*/5 * * * * /opt/wevads/scripts/security-sentinel.sh >> /opt/wevads/logs/security-sentinel-cron.log 2>&1 +*/5 * * * * pg_isready -h 127.0.0.1 -p 5432 > /dev/null 2>&1 || sudo systemctl restart postgresql >> /opt/wevads/logs/pg-watchdog.log 2>&1 +*/5 * * * * php /opt/wevads/public/api/brain-pmta-send.php send >> /var/log/wevads/brain-pmta.log 2>&1 +0 * * * * PGPASSWORD=admin123 psql -h /var/run/postgresql -U admin adx_system -c "REFRESH MATERIALIZED VIEW admin.v_affiliate_funnel; REFRESH MATERIALIZED VIEW admin.v_channel_summary;" > /dev/null 2>&1 +0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-hamid-check.php' >> /opt/wevads/logs/hamid-affiliate.log 2>&1 +0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-monitor.php?action=compare&from=2026-02-10&to=2026-02-11' >> /opt/wevads/logs/affiliate-monitor.log 2>&1 +0 * * * * python3 /opt/wevads/scripts/creative-performance-engine.py all 20 >> /opt/wevads/logs/creative-perf.log 2>&1 +0 */2 * * * php /opt/wevads/public/api/kb-sync-cron.php >> /var/log/wevads/claude_kb_sync.log 2>&1 +0 */4 * * * /opt/wevads/scripts/brain-optimizer-cron.sh +0 */4 * * * php /opt/wevads/scripts/scrape-to-sendcontacts.php >> /opt/wevads/logs/scrape-feeder.log 2>&1 +0 */6 * * * curl -s http://127.0.0.1:5890/api/vault-guard.php?action=cron >> /opt/wevads/logs/vault-guard-cron.log 2>&1 +0 */6 * * * php /opt/wevads/public/api/brain-graph-api.php discover >> /var/log/wevads/brain-graph.log 2>&1 +0 */6 * * * php /opt/wevads/scripts/brain-creative-engine.php full_cycle >> /opt/wevads/logs/brain-creative-engine.log 2>&1 +0 */6 * * * php /opt/wevads/scripts/ethica/ethica-mega-scraper.php all all google >> /opt/wevads/logs/ethica-scraper-continuous.log 2>&1 +0 0 * * * PGPASSWORD=admin123 psql -U admin -d adx_system -c "UPDATE ethica.sms_providers SET sent_today=0" >> /opt/wevads/logs/ethica-sms-reset.log 2>&1 +0 0 * * * curl -s http://127.0.0.1:5821/api/ia-provider-factory.php?action=rotate > /dev/null 2>&1 +0 0 * * * php -r "\$d=new PDO('pgsql:host=localhost;dbname=adx_system','admin','admin123');\$d->exec('UPDATE ethica.senders SET sent_today=0');" >> /opt/wevads/logs/ethica-reset.log 2>&1 +0 3 1,15 * * php /opt/wevads/scripts/ethica/ethica-scraper.php all all >> /opt/wevads/logs/ethica-scraper.log 2>&1 +0 6 * * * php /opt/wevads/scripts/brain-pipeline.php factory >> /var/log/wevads/brain-pipeline.log 2>&1 +15,45 * * * * php /opt/wevads/public/api/brain-graph-api.php check >> /var/log/wevads/brain-graph.log 2>&1 +15,45 * * * * php /opt/wevads/scripts/brain-pipeline.php check >> /var/log/wevads/brain-pipeline.log 2>&1 +30 */6 * * * php /opt/wevads/scripts/ethica/ethica-mega-scraper.php all all directories >> /opt/wevads/logs/ethica-scraper-continuous.log 2>&1 +30 0 */3 * * /opt/wevads/scripts/offer-refresh-cron.sh diff --git a/scripts/dark-scraper-cron.php b/scripts/dark-scraper-cron.php new file mode 100755 index 00000000..480b2e23 --- /dev/null +++ b/scripts/dark-scraper-cron.php @@ -0,0 +1,105 @@ + 0) { $log(" SKIP {$t['name']} (already running)"); continue; } + + pg_query($db, "INSERT INTO dark_scraper_jobs (source_url, source_type, emails_found, phones_found, status, results, created_at) VALUES ('$url', '$cat', 0, 0, 'running', '[]', NOW())"); + $created++; + $log(" JOB created: {$t['name']} → $url"); +} +$log("$created new jobs created"); + +// 3. Process running jobs (scrape URLs) +$jobs = []; +$r = pg_query($db, "SELECT id, source_url, source_type FROM dark_scraper_jobs WHERE status='running' ORDER BY created_at DESC LIMIT 10"); +while($row = pg_fetch_assoc($r)) $jobs[] = $row; +$log(count($jobs).' running jobs to process'); + +$totalEmails = 0; +foreach($jobs as $j) { + $ch = curl_init($j['source_url']); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_TIMEOUT => 20, + CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + CURLOPT_SSL_VERIFYPEER => false, + ]); + $html = curl_exec($ch); + $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if($code != 200 || empty($html)) { + pg_query($db, "UPDATE dark_scraper_jobs SET status='failed' WHERE id={$j['id']}"); + $log(" FAIL job#{$j['id']} HTTP $code"); + continue; + } + + // Extract emails + preg_match_all('/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/', $html, $matches); + $emails = array_unique(array_filter($matches[0], function($e) { + return filter_var($e, FILTER_VALIDATE_EMAIL) && !preg_match('/(example|test|noreply|no-reply|wixpress|sentry)/', $e); + })); + + // Extract phones + preg_match_all('/\+?[\d\s\-\(\)]{10,}/', $html, $pMatches); + $phones = count(array_unique($pMatches[0])); + + $emailCount = count($emails); + pg_query($db, "UPDATE dark_scraper_jobs SET status='completed', emails_found=$emailCount, phones_found=$phones WHERE id={$j['id']}"); + + // Inject into scrapping_results + send_contacts + foreach($emails as $email) { + $email = strtolower(trim($email)); + $esc = pg_escape_string($db, $email); + $domain = substr($email, strpos($email, '@') + 1); + $esc_url = pg_escape_string($db, $j['source_url']); + + pg_query($db, "INSERT INTO scrapping_results (email, source_url, confidence_score, is_verified, extracted_at) VALUES ('$esc', '$esc_url', 70, true, NOW()) ON CONFLICT DO NOTHING"); + + // Detect ISP + $isp = 'OTHER'; + $map = ['gmail.com'=>'gmail','hotmail.com'=>'hotmail','outlook.com'=>'hotmail','live.com'=>'hotmail','yahoo.com'=>'yahoo','aol.com'=>'aol','gmx.de'=>'gmx','gmx.net'=>'gmx','t-online.de'=>'tonline','web.de'=>'webde','orange.fr'=>'orange','free.fr'=>'free','sfr.fr'=>'sfr','laposte.net'=>'laposte','protonmail.com'=>'proton','proton.me'=>'proton']; + if(isset($map[$domain])) $isp = $map[$domain]; + + pg_query($db, "INSERT INTO send_contacts (email, isp, domain, status, source) VALUES ('$esc', '$isp', '$domain', 'active', 'dark_scraper') ON CONFLICT (email) DO NOTHING"); + } + + $totalEmails += $emailCount; + $log(" OK job#{$j['id']} → $emailCount emails, $phones phones"); +} + +$log("TOTAL: $totalEmails emails scraped"); + +// Update scrapping_targets last_scraped +foreach($targets as $t) { + pg_query($db, "UPDATE scrapping_targets SET last_scraped=NOW() WHERE id={$t['id']}"); +} + +// Stats +$r = pg_query($db, "SELECT status, COUNT(*) FROM dark_scraper_jobs GROUP BY status ORDER BY status"); +$log("=== DARK SCRAPER JOBS ==="); +while($row = pg_fetch_assoc($r)) $log(" {$row['status']}: {$row['count']}"); + +$total = pg_fetch_result(pg_query($db, "SELECT COUNT(*) FROM scrapping_results"), 0, 0); +$log("Total scrapping_results: $total"); diff --git a/scripts/ghost-scout-cron.php b/scripts/ghost-scout-cron.php new file mode 100755 index 00000000..2eaada17 --- /dev/null +++ b/scripts/ghost-scout-cron.php @@ -0,0 +1,31 @@ + 0 ? implode(', ', array_slice($mx, 0, 5)) : 'No MX'; + + pg_query($db, "UPDATE dark_scout_targets SET relays_found=$relays, payloads='".pg_escape_string($db, json_encode(['mx'=>$mx]))."' WHERE id={$t['id']}"); + + $title = pg_escape_string($db, "MX Scan: $domain → $relays relays"); + $data = pg_escape_string($db, json_encode(['domain'=>$domain,'mx'=>$mx,'relays'=>$relays])); + pg_query($db, "INSERT INTO ghost_scout_reports (report_type, title, data, priority, is_read, created_at) VALUES ('mx_scan', '$title', '$data', 1, false, NOW())"); + $reportsCreated++; + $log(" $domain → $relays MX ($mxList)"); + } +} + +$reports = pg_fetch_result(pg_query($db, "SELECT COUNT(*) FROM ghost_scout_reports"), 0, 0); +$log("Reports: $reports (new: $reportsCreated)");