FACTORY-ACTIVATE 02-03-2026: 6 data providers, 20 ADX-Import lists, auto-scraper+dark-scraper+ghost-scout crons, 16 active crons, 28+10+10 scraping targets
This commit is contained in:
59
crontab-backup-pre-factory.txt
Normal file
59
crontab-backup-pre-factory.txt
Normal file
@@ -0,0 +1,59 @@
|
||||
|
||||
# --- Brain Optimization ---
|
||||
# --- Brain Send ---
|
||||
# --- Brain Tracking ---
|
||||
# --- E2E Pipeline ---
|
||||
# --- Ethica (B2B Medical) ---
|
||||
# --- Matviews Refresh ---
|
||||
# --- Monitoring & Maintenance ---
|
||||
# --- Original Guardian/Sentinel (from snapshot) ---
|
||||
# --- Warmup ---
|
||||
# === WEVADS ADX Production Crons ===
|
||||
# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/guardian-scan.php?action=full_scan > /var/log/guardian-scan.log 2>&1
|
||||
# KILLED-REGRESSION: 0 22 * * * curl -s http://localhost:5890/api/sentinel-brain.php?action=scan&fix=1 >> /var/log/sentinel.log 2>&1
|
||||
# Migrated from CCX33 - All additions, no ADX changes
|
||||
# PG TCP Watchdog - restart if TCP down
|
||||
# STANDBY-SEND ethica-send: 0 14 * * 1-5 php /opt/wevads/scripts/ethica/ethica-send.php auto 200 >> /opt/wevads/logs/ethica-send.log 2>&1
|
||||
# STANDBY-SEND ethica-send: 0 9 * * 1-5 php /opt/wevads/scripts/ethica/ethica-send.php auto 200 >> /opt/wevads/logs/ethica-send.log 2>&1
|
||||
# STANDBY-SEND ethica-sms: 0 10 * * 1-5 php /opt/wevads/scripts/ethica/ethica-sms-send.php auto 500 >> /opt/wevads/logs/ethica-sms.log 2>&1
|
||||
# STANDBY-SEND ethica-sms: 0 15 * * 1-5 php /opt/wevads/scripts/ethica/ethica-sms-send.php auto 500 >> /opt/wevads/logs/ethica-sms.log 2>&1
|
||||
# STANDBY-SEND tracking-seeds-opens: 30 */2 * * * php /opt/wevads/public/api/brain-tracking-seeds.php check_opens >> /var/log/wevads/brain-graph.log 2>&1
|
||||
# STANDBY-SEND tracking-seeds-send: 0 */2 * * * php /opt/wevads/public/api/brain-tracking-seeds.php send_tracked >> /var/log/wevads/brain-graph.log 2>&1
|
||||
# STANDBY-SEND warmup-advance: 0 0 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=advance_day' >> /opt/wevads/logs/warmup-advance.log 2>&1
|
||||
# STANDBY-SEND warmup-execute: */15 6-22 * * * curl -s 'http://127.0.0.1:5890/api/warmup-engine.php?action=execute_warmup&batch=50' >> /opt/wevads/logs/warmup-execute.log 2>&1
|
||||
# STANDBY-SEND warmup-reset: 0 0 * * * sudo -u postgres psql adx_system -c "UPDATE admin.warmup_accounts SET sent_today=0 WHERE status='warming'" >> /opt/wevads/logs/warmup-reset.log 2>&1
|
||||
*/1 * * * * crontab -l > /opt/wevads/crontab-sync.txt 2>/dev/null
|
||||
*/10 * * * * /usr/local/bin/weval-watchdog
|
||||
*/15 * * * * curl -s http://localhost:5821/api/bounce-processor.php >> /var/log/wevads/bounce-cron.log 2>&1
|
||||
*/15 * * * * php /opt/wevads/public/api/brain-productive.php send >> /var/log/wevads/brain-productive.log 2>&1
|
||||
*/15 * * * * sudo -u postgres psql adx_system -c "UPDATE admin.ia_provider_accounts SET status='active' WHERE status='rate_limited' AND cooldown_until<NOW()" > /dev/null 2>&1
|
||||
*/15 7-22 * * * /usr/bin/bash /opt/wevads/scripts/send_batch.sh 200 >> /opt/wevads/logs/send_cron_output.log 2>&1
|
||||
*/30 * * * * /opt/wevads/scripts/mind-autonomous.sh >> /opt/wevads/logs/mind-cron.log 2>&1
|
||||
*/30 * * * * /opt/wevads/scripts/sentinel-autorepair.sh --analyze-only
|
||||
*/30 * * * * curl -s "http://127.0.0.1:5821/api/conversions-collector.php?action=pull" >> /opt/wevads/logs/conversions-pull.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/public/api/brain-graph-api.php send >> /var/log/wevads/brain-graph.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/scripts/brain-pipeline.php full >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
*/30 * * * * php /opt/wevads/scripts/ethica/ethica-validator.php 500 >> /opt/wevads/logs/ethica-validator.log 2>&1
|
||||
*/5 * * * * /opt/wevads/scripts/security-sentinel.sh >> /opt/wevads/logs/security-sentinel-cron.log 2>&1
|
||||
*/5 * * * * pg_isready -h 127.0.0.1 -p 5432 > /dev/null 2>&1 || sudo systemctl restart postgresql >> /opt/wevads/logs/pg-watchdog.log 2>&1
|
||||
*/5 * * * * php /opt/wevads/public/api/brain-pmta-send.php send >> /var/log/wevads/brain-pmta.log 2>&1
|
||||
0 * * * * PGPASSWORD=admin123 psql -h /var/run/postgresql -U admin adx_system -c "REFRESH MATERIALIZED VIEW admin.v_affiliate_funnel; REFRESH MATERIALIZED VIEW admin.v_channel_summary;" > /dev/null 2>&1
|
||||
0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-hamid-check.php' >> /opt/wevads/logs/hamid-affiliate.log 2>&1
|
||||
0 * * * * curl -s 'http://127.0.0.1:5890/api/affiliate-monitor.php?action=compare&from=2026-02-10&to=2026-02-11' >> /opt/wevads/logs/affiliate-monitor.log 2>&1
|
||||
0 * * * * python3 /opt/wevads/scripts/creative-performance-engine.py all 20 >> /opt/wevads/logs/creative-perf.log 2>&1
|
||||
0 */2 * * * php /opt/wevads/public/api/kb-sync-cron.php >> /var/log/wevads/claude_kb_sync.log 2>&1
|
||||
0 */4 * * * /opt/wevads/scripts/brain-optimizer-cron.sh
|
||||
0 */4 * * * php /opt/wevads/scripts/scrape-to-sendcontacts.php >> /opt/wevads/logs/scrape-feeder.log 2>&1
|
||||
0 */6 * * * curl -s http://127.0.0.1:5890/api/vault-guard.php?action=cron >> /opt/wevads/logs/vault-guard-cron.log 2>&1
|
||||
0 */6 * * * php /opt/wevads/public/api/brain-graph-api.php discover >> /var/log/wevads/brain-graph.log 2>&1
|
||||
0 */6 * * * php /opt/wevads/scripts/brain-creative-engine.php full_cycle >> /opt/wevads/logs/brain-creative-engine.log 2>&1
|
||||
0 */6 * * * php /opt/wevads/scripts/ethica/ethica-mega-scraper.php all all google >> /opt/wevads/logs/ethica-scraper-continuous.log 2>&1
|
||||
0 0 * * * PGPASSWORD=admin123 psql -U admin -d adx_system -c "UPDATE ethica.sms_providers SET sent_today=0" >> /opt/wevads/logs/ethica-sms-reset.log 2>&1
|
||||
0 0 * * * curl -s http://127.0.0.1:5821/api/ia-provider-factory.php?action=rotate > /dev/null 2>&1
|
||||
0 0 * * * php -r "\$d=new PDO('pgsql:host=localhost;dbname=adx_system','admin','admin123');\$d->exec('UPDATE ethica.senders SET sent_today=0');" >> /opt/wevads/logs/ethica-reset.log 2>&1
|
||||
0 3 1,15 * * php /opt/wevads/scripts/ethica/ethica-scraper.php all all >> /opt/wevads/logs/ethica-scraper.log 2>&1
|
||||
0 6 * * * php /opt/wevads/scripts/brain-pipeline.php factory >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
15,45 * * * * php /opt/wevads/public/api/brain-graph-api.php check >> /var/log/wevads/brain-graph.log 2>&1
|
||||
15,45 * * * * php /opt/wevads/scripts/brain-pipeline.php check >> /var/log/wevads/brain-pipeline.log 2>&1
|
||||
30 */6 * * * php /opt/wevads/scripts/ethica/ethica-mega-scraper.php all all directories >> /opt/wevads/logs/ethica-scraper-continuous.log 2>&1
|
||||
30 0 */3 * * /opt/wevads/scripts/offer-refresh-cron.sh
|
||||
105
scripts/dark-scraper-cron.php
Executable file
105
scripts/dark-scraper-cron.php
Executable file
@@ -0,0 +1,105 @@
|
||||
<?php
|
||||
/**
|
||||
* DARK SCRAPER CRON — Triggers dark-scraper API to process pending jobs
|
||||
* and creates new jobs from scrapping_targets
|
||||
*/
|
||||
$db = pg_connect('host=localhost dbname=adx_system user=admin password=admin123');
|
||||
pg_query($db, 'SET search_path TO admin,public');
|
||||
$log = function($m) { echo '['.date('Y-m-d H:i:s').'] '.$m."\n"; };
|
||||
|
||||
// 1. Get scrapping_targets
|
||||
$targets = [];
|
||||
$r = pg_query($db, "SELECT id, name, url_pattern, category, daily_limit FROM scrapping_targets ORDER BY priority ASC");
|
||||
while($row = pg_fetch_assoc($r)) $targets[] = $row;
|
||||
$log(count($targets).' scrapping targets loaded');
|
||||
|
||||
// 2. Create dark_scraper_jobs from targets
|
||||
$created = 0;
|
||||
foreach($targets as $t) {
|
||||
$url = pg_escape_string($db, $t['url_pattern']);
|
||||
$cat = pg_escape_string($db, $t['category']);
|
||||
|
||||
// Check not already running
|
||||
$existing = pg_fetch_result(pg_query($db, "SELECT COUNT(*) FROM dark_scraper_jobs WHERE source_url='$url' AND status='running'"), 0, 0);
|
||||
if($existing > 0) { $log(" SKIP {$t['name']} (already running)"); continue; }
|
||||
|
||||
pg_query($db, "INSERT INTO dark_scraper_jobs (source_url, source_type, emails_found, phones_found, status, results, created_at) VALUES ('$url', '$cat', 0, 0, 'running', '[]', NOW())");
|
||||
$created++;
|
||||
$log(" JOB created: {$t['name']} → $url");
|
||||
}
|
||||
$log("$created new jobs created");
|
||||
|
||||
// 3. Process running jobs (scrape URLs)
|
||||
$jobs = [];
|
||||
$r = pg_query($db, "SELECT id, source_url, source_type FROM dark_scraper_jobs WHERE status='running' ORDER BY created_at DESC LIMIT 10");
|
||||
while($row = pg_fetch_assoc($r)) $jobs[] = $row;
|
||||
$log(count($jobs).' running jobs to process');
|
||||
|
||||
$totalEmails = 0;
|
||||
foreach($jobs as $j) {
|
||||
$ch = curl_init($j['source_url']);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
CURLOPT_TIMEOUT => 20,
|
||||
CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
CURLOPT_SSL_VERIFYPEER => false,
|
||||
]);
|
||||
$html = curl_exec($ch);
|
||||
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if($code != 200 || empty($html)) {
|
||||
pg_query($db, "UPDATE dark_scraper_jobs SET status='failed' WHERE id={$j['id']}");
|
||||
$log(" FAIL job#{$j['id']} HTTP $code");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract emails
|
||||
preg_match_all('/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/', $html, $matches);
|
||||
$emails = array_unique(array_filter($matches[0], function($e) {
|
||||
return filter_var($e, FILTER_VALIDATE_EMAIL) && !preg_match('/(example|test|noreply|no-reply|wixpress|sentry)/', $e);
|
||||
}));
|
||||
|
||||
// Extract phones
|
||||
preg_match_all('/\+?[\d\s\-\(\)]{10,}/', $html, $pMatches);
|
||||
$phones = count(array_unique($pMatches[0]));
|
||||
|
||||
$emailCount = count($emails);
|
||||
pg_query($db, "UPDATE dark_scraper_jobs SET status='completed', emails_found=$emailCount, phones_found=$phones WHERE id={$j['id']}");
|
||||
|
||||
// Inject into scrapping_results + send_contacts
|
||||
foreach($emails as $email) {
|
||||
$email = strtolower(trim($email));
|
||||
$esc = pg_escape_string($db, $email);
|
||||
$domain = substr($email, strpos($email, '@') + 1);
|
||||
$esc_url = pg_escape_string($db, $j['source_url']);
|
||||
|
||||
pg_query($db, "INSERT INTO scrapping_results (email, source_url, confidence_score, is_verified, extracted_at) VALUES ('$esc', '$esc_url', 70, true, NOW()) ON CONFLICT DO NOTHING");
|
||||
|
||||
// Detect ISP
|
||||
$isp = 'OTHER';
|
||||
$map = ['gmail.com'=>'gmail','hotmail.com'=>'hotmail','outlook.com'=>'hotmail','live.com'=>'hotmail','yahoo.com'=>'yahoo','aol.com'=>'aol','gmx.de'=>'gmx','gmx.net'=>'gmx','t-online.de'=>'tonline','web.de'=>'webde','orange.fr'=>'orange','free.fr'=>'free','sfr.fr'=>'sfr','laposte.net'=>'laposte','protonmail.com'=>'proton','proton.me'=>'proton'];
|
||||
if(isset($map[$domain])) $isp = $map[$domain];
|
||||
|
||||
pg_query($db, "INSERT INTO send_contacts (email, isp, domain, status, source) VALUES ('$esc', '$isp', '$domain', 'active', 'dark_scraper') ON CONFLICT (email) DO NOTHING");
|
||||
}
|
||||
|
||||
$totalEmails += $emailCount;
|
||||
$log(" OK job#{$j['id']} → $emailCount emails, $phones phones");
|
||||
}
|
||||
|
||||
$log("TOTAL: $totalEmails emails scraped");
|
||||
|
||||
// Update scrapping_targets last_scraped
|
||||
foreach($targets as $t) {
|
||||
pg_query($db, "UPDATE scrapping_targets SET last_scraped=NOW() WHERE id={$t['id']}");
|
||||
}
|
||||
|
||||
// Stats
|
||||
$r = pg_query($db, "SELECT status, COUNT(*) FROM dark_scraper_jobs GROUP BY status ORDER BY status");
|
||||
$log("=== DARK SCRAPER JOBS ===");
|
||||
while($row = pg_fetch_assoc($r)) $log(" {$row['status']}: {$row['count']}");
|
||||
|
||||
$total = pg_fetch_result(pg_query($db, "SELECT COUNT(*) FROM scrapping_results"), 0, 0);
|
||||
$log("Total scrapping_results: $total");
|
||||
31
scripts/ghost-scout-cron.php
Executable file
31
scripts/ghost-scout-cron.php
Executable file
@@ -0,0 +1,31 @@
|
||||
<?php
|
||||
$db = pg_connect('host=localhost dbname=adx_system user=admin password=admin123');
|
||||
pg_query($db, 'SET search_path TO admin,public');
|
||||
$log = function($m) { echo '['.date('Y-m-d H:i:s').'] '.$m."\n"; };
|
||||
|
||||
$targets = [];
|
||||
$r = pg_query($db, "SELECT id, target, scan_type, status FROM dark_scout_targets WHERE status='active' ORDER BY id");
|
||||
while($row = pg_fetch_assoc($r)) $targets[] = $row;
|
||||
$log(count($targets).' scout targets');
|
||||
|
||||
$reportsCreated = 0;
|
||||
foreach($targets as $t) {
|
||||
if($t['scan_type'] === 'relay_scan') {
|
||||
$domain = $t['target'];
|
||||
$mx = [];
|
||||
getmxrr($domain, $mx);
|
||||
$relays = count($mx);
|
||||
$mxList = $relays > 0 ? implode(', ', array_slice($mx, 0, 5)) : 'No MX';
|
||||
|
||||
pg_query($db, "UPDATE dark_scout_targets SET relays_found=$relays, payloads='".pg_escape_string($db, json_encode(['mx'=>$mx]))."' WHERE id={$t['id']}");
|
||||
|
||||
$title = pg_escape_string($db, "MX Scan: $domain → $relays relays");
|
||||
$data = pg_escape_string($db, json_encode(['domain'=>$domain,'mx'=>$mx,'relays'=>$relays]));
|
||||
pg_query($db, "INSERT INTO ghost_scout_reports (report_type, title, data, priority, is_read, created_at) VALUES ('mx_scan', '$title', '$data', 1, false, NOW())");
|
||||
$reportsCreated++;
|
||||
$log(" $domain → $relays MX ($mxList)");
|
||||
}
|
||||
}
|
||||
|
||||
$reports = pg_fetch_result(pg_query($db, "SELECT COUNT(*) FROM ghost_scout_reports"), 0, 0);
|
||||
$log("Reports: $reports (new: $reportsCreated)");
|
||||
Reference in New Issue
Block a user