497 lines
16 KiB
Python
Executable File
497 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
WARMUP MANAGER
|
|
Gère le warmup intelligent des IPs/méthodes:
|
|
- Détecte automatiquement le temps de warmup nécessaire
|
|
- Ajuste le volume progressivement
|
|
- Calcule le score composite (inbox * volume)
|
|
- Optimise pour maximiser inbox ET volume
|
|
"""
|
|
|
|
import psycopg2
|
|
import json
|
|
import math
|
|
from datetime import datetime, timedelta
|
|
from collections import defaultdict
|
|
|
|
DB_CONFIG = {
|
|
'host': 'localhost',
|
|
'database': 'adx_system',
|
|
'user': 'admin',
|
|
'password': 'admin123'
|
|
}
|
|
|
|
# Stages de warmup
|
|
STAGES = {
|
|
1: {'name': 'cold', 'color': '🔵', 'multiplier': 0.5},
|
|
2: {'name': 'warming', 'color': '🟡', 'multiplier': 0.8},
|
|
3: {'name': 'warm', 'color': '🟢', 'multiplier': 1.0},
|
|
4: {'name': 'hot', 'color': '🔥', 'multiplier': 1.5},
|
|
5: {'name': 'burned', 'color': '💀', 'multiplier': 0},
|
|
}
|
|
|
|
def get_db():
|
|
return psycopg2.connect(**DB_CONFIG)
|
|
|
|
def get_warmup_rules(method):
|
|
"""Get warmup rules for method"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("""
|
|
SELECT initial_daily_limit, max_daily_limit, warmup_days_needed,
|
|
daily_increase_percent, daily_decrease_percent,
|
|
min_inbox_rate_to_increase, max_spam_rate_before_decrease,
|
|
stage_2_after_sent, stage_3_after_sent, stage_4_after_sent
|
|
FROM admin.warmup_rules
|
|
WHERE method_name = %s
|
|
""", (method,))
|
|
|
|
row = cur.fetchone()
|
|
conn.close()
|
|
|
|
if row:
|
|
return {
|
|
'initial_limit': row[0],
|
|
'max_limit': row[1],
|
|
'warmup_days': row[2],
|
|
'increase_pct': row[3],
|
|
'decrease_pct': row[4],
|
|
'min_inbox_to_increase': row[5],
|
|
'max_spam_to_decrease': row[6],
|
|
'stage_2_sent': row[7],
|
|
'stage_3_sent': row[8],
|
|
'stage_4_sent': row[9]
|
|
}
|
|
|
|
# Default rules
|
|
return {
|
|
'initial_limit': 50,
|
|
'max_limit': 10000,
|
|
'warmup_days': 7,
|
|
'increase_pct': 30,
|
|
'decrease_pct': 50,
|
|
'min_inbox_to_increase': 70,
|
|
'max_spam_to_decrease': 20,
|
|
'stage_2_sent': 100,
|
|
'stage_3_sent': 500,
|
|
'stage_4_sent': 2000
|
|
}
|
|
|
|
def calculate_volume_score(inbox_rate, daily_volume):
|
|
"""
|
|
Calculate composite score: inbox_rate * log10(volume)
|
|
High inbox + High volume = High score
|
|
|
|
Examples:
|
|
- 90% inbox, 100/day = 90 * 2 = 180
|
|
- 80% inbox, 1000/day = 80 * 3 = 240
|
|
- 70% inbox, 10000/day = 70 * 4 = 280
|
|
- 50% inbox, 50000/day = 50 * 4.7 = 235
|
|
"""
|
|
if daily_volume <= 0:
|
|
return 0
|
|
if inbox_rate <= 0:
|
|
return 0
|
|
|
|
return inbox_rate * math.log10(max(1, daily_volume))
|
|
|
|
def get_or_create_warmup(ip, method, domain=None, combo_id=None):
|
|
"""Get or create warmup tracking record"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("""
|
|
SELECT id, current_daily_limit, warmup_stage, total_sent,
|
|
current_inbox_rate, status, warmup_started_at
|
|
FROM admin.warmup_tracking
|
|
WHERE ip_address = %s AND sending_method = %s
|
|
AND COALESCE(domain, '') = COALESCE(%s, '')
|
|
""", (ip, method, domain))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row:
|
|
conn.close()
|
|
return {
|
|
'id': row[0],
|
|
'daily_limit': row[1],
|
|
'stage': row[2],
|
|
'total_sent': row[3],
|
|
'inbox_rate': row[4],
|
|
'status': row[5],
|
|
'started_at': row[6],
|
|
'exists': True
|
|
}
|
|
|
|
# Create new
|
|
rules = get_warmup_rules(method)
|
|
|
|
cur.execute("""
|
|
INSERT INTO admin.warmup_tracking
|
|
(ip_address, sending_method, domain, combo_id, current_daily_limit, warmup_stage)
|
|
VALUES (%s, %s, %s, %s, %s, 1)
|
|
RETURNING id
|
|
""", (ip, method, domain, combo_id, rules['initial_limit']))
|
|
|
|
warmup_id = cur.fetchone()[0]
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {
|
|
'id': warmup_id,
|
|
'daily_limit': rules['initial_limit'],
|
|
'stage': 1,
|
|
'total_sent': 0,
|
|
'inbox_rate': None,
|
|
'status': 'cold',
|
|
'started_at': datetime.now(),
|
|
'exists': False
|
|
}
|
|
|
|
def record_send_batch(warmup_id, sent, inbox, spam, bounced):
|
|
"""Record a batch of sends"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
# Update totals
|
|
cur.execute("""
|
|
UPDATE admin.warmup_tracking SET
|
|
total_sent = total_sent + %s,
|
|
total_inbox = total_inbox + %s,
|
|
total_spam = total_spam + %s,
|
|
total_bounced = total_bounced + %s,
|
|
last_send_at = NOW()
|
|
WHERE id = %s
|
|
RETURNING total_sent, total_inbox, total_spam
|
|
""", (sent, inbox, spam, bounced, warmup_id))
|
|
|
|
totals = cur.fetchone()
|
|
|
|
# Calculate rates
|
|
total_sent = totals[0]
|
|
inbox_rate = (totals[1] / total_sent * 100) if total_sent > 0 else 0
|
|
spam_rate = (totals[2] / total_sent * 100) if total_sent > 0 else 0
|
|
|
|
cur.execute("""
|
|
UPDATE admin.warmup_tracking SET
|
|
current_inbox_rate = %s,
|
|
avg_inbox_rate = %s
|
|
WHERE id = %s
|
|
""", (inbox_rate, inbox_rate, warmup_id))
|
|
|
|
# Record hourly
|
|
hour = datetime.now().replace(minute=0, second=0, microsecond=0)
|
|
cur.execute("""
|
|
INSERT INTO admin.volume_history
|
|
(warmup_id, hour_timestamp, emails_sent, emails_inbox, emails_spam, emails_bounced, inbox_rate)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
|
ON CONFLICT (warmup_id, hour_timestamp) DO UPDATE SET
|
|
emails_sent = admin.volume_history.emails_sent + EXCLUDED.emails_sent,
|
|
emails_inbox = admin.volume_history.emails_inbox + EXCLUDED.emails_inbox,
|
|
emails_spam = admin.volume_history.emails_spam + EXCLUDED.emails_spam
|
|
""", (warmup_id, hour, sent, inbox, spam, bounced, inbox_rate))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {'total_sent': total_sent, 'inbox_rate': inbox_rate, 'spam_rate': spam_rate}
|
|
|
|
def evaluate_and_adjust(warmup_id):
|
|
"""Evaluate warmup progress and adjust limits"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
# Get current state
|
|
cur.execute("""
|
|
SELECT w.*, r.*
|
|
FROM admin.warmup_tracking w
|
|
JOIN admin.warmup_rules r ON w.sending_method = r.method_name
|
|
WHERE w.id = %s
|
|
""", (warmup_id,))
|
|
|
|
row = cur.fetchone()
|
|
if not row:
|
|
conn.close()
|
|
return None
|
|
|
|
# Parse data (adjust indices based on actual columns)
|
|
cur.execute("""
|
|
SELECT
|
|
current_daily_limit, warmup_stage, total_sent, total_inbox, total_spam,
|
|
current_inbox_rate, status, warmup_started_at, sending_method
|
|
FROM admin.warmup_tracking WHERE id = %s
|
|
""", (warmup_id,))
|
|
|
|
w = cur.fetchone()
|
|
current_limit, stage, total_sent, total_inbox, total_spam, inbox_rate, status, started, method = w
|
|
|
|
rules = get_warmup_rules(method)
|
|
|
|
# Calculate spam rate
|
|
spam_rate = (total_spam / total_sent * 100) if total_sent > 0 else 0
|
|
inbox_rate = inbox_rate or 0
|
|
|
|
action = None
|
|
new_limit = current_limit
|
|
new_stage = stage
|
|
new_status = status
|
|
|
|
# Check for problems first
|
|
if spam_rate > rules['max_spam_to_decrease']:
|
|
# DECREASE - too much spam
|
|
new_limit = max(rules['initial_limit'], int(current_limit * (1 - rules['decrease_pct'] / 100)))
|
|
action = f"DECREASE (spam {spam_rate:.1f}%)"
|
|
new_status = 'throttled'
|
|
|
|
elif inbox_rate >= rules['min_inbox_to_increase'] and status != 'throttled':
|
|
# Can INCREASE
|
|
new_limit = min(rules['max_limit'], int(current_limit * (1 + rules['increase_pct'] / 100)))
|
|
action = f"INCREASE (inbox {inbox_rate:.1f}%)"
|
|
|
|
# Update stage based on total sent
|
|
if total_sent >= rules['stage_4_sent']:
|
|
new_stage = 4
|
|
new_status = 'hot'
|
|
elif total_sent >= rules['stage_3_sent']:
|
|
new_stage = 3
|
|
new_status = 'warm'
|
|
elif total_sent >= rules['stage_2_sent']:
|
|
new_stage = 2
|
|
new_status = 'warming'
|
|
|
|
# Check if warmed
|
|
warmup_hours = (datetime.now() - started).total_seconds() / 3600 if started else 0
|
|
is_warmed = new_stage >= 3 and inbox_rate >= rules['min_inbox_to_increase']
|
|
|
|
# Calculate volume score
|
|
daily_volume = current_limit # Approximation
|
|
volume_score = calculate_volume_score(inbox_rate, daily_volume)
|
|
|
|
# Update
|
|
cur.execute("""
|
|
UPDATE admin.warmup_tracking SET
|
|
current_daily_limit = %s,
|
|
recommended_daily_limit = %s,
|
|
warmup_stage = %s,
|
|
status = %s,
|
|
is_warmed = %s,
|
|
warmup_duration_hours = %s,
|
|
volume_score = %s,
|
|
last_check_at = NOW()
|
|
WHERE id = %s
|
|
""", (new_limit, new_limit, new_stage, new_status, is_warmed, warmup_hours, volume_score, warmup_id))
|
|
|
|
if new_stage != stage:
|
|
cur.execute("""
|
|
UPDATE admin.warmup_tracking SET stage_changed_at = NOW() WHERE id = %s
|
|
""", (warmup_id,))
|
|
|
|
if is_warmed and not w[7]: # warmup_ended_at was NULL
|
|
cur.execute("""
|
|
UPDATE admin.warmup_tracking SET warmup_ended_at = NOW() WHERE id = %s
|
|
""", (warmup_id,))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {
|
|
'action': action,
|
|
'old_limit': current_limit,
|
|
'new_limit': new_limit,
|
|
'stage': new_stage,
|
|
'status': new_status,
|
|
'inbox_rate': inbox_rate,
|
|
'spam_rate': spam_rate,
|
|
'volume_score': volume_score,
|
|
'is_warmed': is_warmed,
|
|
'warmup_hours': warmup_hours
|
|
}
|
|
|
|
def get_send_allowance(ip, method, domain=None):
|
|
"""Get how many emails can be sent right now"""
|
|
warmup = get_or_create_warmup(ip, method, domain)
|
|
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
# Check today's sends
|
|
cur.execute("""
|
|
SELECT COALESCE(SUM(emails_sent), 0)
|
|
FROM admin.volume_history
|
|
WHERE warmup_id = %s
|
|
AND hour_timestamp >= CURRENT_DATE
|
|
""", (warmup['id'],))
|
|
|
|
sent_today = cur.fetchone()[0]
|
|
conn.close()
|
|
|
|
remaining = max(0, warmup['daily_limit'] - sent_today)
|
|
|
|
return {
|
|
'warmup_id': warmup['id'],
|
|
'daily_limit': warmup['daily_limit'],
|
|
'sent_today': sent_today,
|
|
'remaining': remaining,
|
|
'stage': warmup['stage'],
|
|
'status': warmup['status'],
|
|
'can_send': remaining > 0 and warmup['status'] not in ['burned', 'throttled']
|
|
}
|
|
|
|
def show_warmup_status():
|
|
"""Display all warmup status"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
print("=" * 90)
|
|
print("🔥 WARMUP STATUS")
|
|
print("=" * 90)
|
|
|
|
cur.execute("""
|
|
SELECT
|
|
ip_address, sending_method, domain,
|
|
warmup_stage, status, current_daily_limit,
|
|
total_sent, current_inbox_rate, volume_score,
|
|
warmup_duration_hours, is_warmed
|
|
FROM admin.warmup_tracking
|
|
ORDER BY volume_score DESC NULLS LAST
|
|
""")
|
|
|
|
print(f"\n{'IP':<16} {'Method':<12} {'Stage':<8} {'Limit':>8} {'Sent':>8} {'Inbox%':>7} {'Score':>8} {'Warmed'}")
|
|
print("-" * 90)
|
|
|
|
for row in cur.fetchall():
|
|
ip, method, domain, stage, status, limit, sent, inbox, score, hours, warmed = row
|
|
|
|
stage_info = STAGES.get(stage, STAGES[1])
|
|
stage_str = f"{stage_info['color']} {stage_info['name']}"
|
|
warmed_str = "✅" if warmed else f"{hours or 0:.0f}h"
|
|
|
|
print(f"{str(ip):<16} {method:<12} {stage_str:<8} {limit or 0:>8} {sent or 0:>8} {inbox or 0:>6.1f}% {score or 0:>8.1f} {warmed_str}")
|
|
|
|
# Method rankings
|
|
print("\n" + "=" * 90)
|
|
print("📊 METHOD RANKINGS (Volume Quality Score)")
|
|
print("=" * 90)
|
|
|
|
cur.execute("""
|
|
SELECT
|
|
sending_method,
|
|
COUNT(*) as ips,
|
|
AVG(current_daily_limit) as avg_limit,
|
|
SUM(total_sent) as total_sent,
|
|
AVG(current_inbox_rate) as avg_inbox,
|
|
AVG(volume_score) as avg_score,
|
|
COUNT(CASE WHEN is_warmed THEN 1 END) as warmed_count
|
|
FROM admin.warmup_tracking
|
|
GROUP BY sending_method
|
|
ORDER BY avg_score DESC NULLS LAST
|
|
""")
|
|
|
|
print(f"\n{'Method':<15} {'IPs':>5} {'Avg Limit':>10} {'Total Sent':>12} {'Inbox%':>8} {'Score':>8} {'Warmed'}")
|
|
print("-" * 70)
|
|
|
|
for row in cur.fetchall():
|
|
method, ips, avg_limit, total, inbox, score, warmed = row
|
|
print(f"{method:<15} {ips:>5} {avg_limit or 0:>10.0f} {total or 0:>12,} {inbox or 0:>7.1f}% {score or 0:>8.1f} {warmed}/{ips}")
|
|
|
|
conn.close()
|
|
|
|
def update_method_performance():
|
|
"""Update method performance summary"""
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("""
|
|
INSERT INTO admin.method_performance
|
|
(method_name, target_isp, total_sent, max_daily_achieved, avg_daily_volume,
|
|
avg_inbox_rate, volume_quality_score)
|
|
SELECT
|
|
sending_method,
|
|
'all',
|
|
SUM(total_sent),
|
|
MAX(current_daily_limit),
|
|
AVG(current_daily_limit),
|
|
AVG(current_inbox_rate),
|
|
AVG(volume_score)
|
|
FROM admin.warmup_tracking
|
|
WHERE total_sent > 0
|
|
GROUP BY sending_method
|
|
ON CONFLICT (method_name, target_isp) DO UPDATE SET
|
|
total_sent = EXCLUDED.total_sent,
|
|
max_daily_achieved = EXCLUDED.max_daily_achieved,
|
|
avg_daily_volume = EXCLUDED.avg_daily_volume,
|
|
avg_inbox_rate = EXCLUDED.avg_inbox_rate,
|
|
volume_quality_score = EXCLUDED.volume_quality_score,
|
|
last_updated = NOW()
|
|
""")
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def main():
|
|
import sys
|
|
|
|
if len(sys.argv) < 2:
|
|
show_warmup_status()
|
|
return
|
|
|
|
cmd = sys.argv[1]
|
|
|
|
if cmd == 'status':
|
|
show_warmup_status()
|
|
|
|
elif cmd == 'check':
|
|
# Check and adjust all warmups
|
|
conn = get_db()
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT id FROM admin.warmup_tracking")
|
|
|
|
for (warmup_id,) in cur.fetchall():
|
|
result = evaluate_and_adjust(warmup_id)
|
|
if result and result['action']:
|
|
print(f"Warmup #{warmup_id}: {result['action']} → {result['new_limit']}/day")
|
|
|
|
conn.close()
|
|
update_method_performance()
|
|
|
|
elif cmd == 'allowance':
|
|
if len(sys.argv) < 4:
|
|
print("Usage: warmup-manager.py allowance <ip> <method> [domain]")
|
|
return
|
|
ip = sys.argv[2]
|
|
method = sys.argv[3]
|
|
domain = sys.argv[4] if len(sys.argv) > 4 else None
|
|
|
|
result = get_send_allowance(ip, method, domain)
|
|
print(f"\n📊 Send Allowance for {ip} ({method}):")
|
|
print(f" Daily Limit: {result['daily_limit']}")
|
|
print(f" Sent Today: {result['sent_today']}")
|
|
print(f" Remaining: {result['remaining']}")
|
|
print(f" Stage: {STAGES[result['stage']]['color']} {result['status']}")
|
|
print(f" Can Send: {'✅ Yes' if result['can_send'] else '❌ No'}")
|
|
|
|
elif cmd == 'record':
|
|
if len(sys.argv) < 7:
|
|
print("Usage: warmup-manager.py record <ip> <method> <sent> <inbox> <spam>")
|
|
return
|
|
ip = sys.argv[2]
|
|
method = sys.argv[3]
|
|
sent, inbox, spam = int(sys.argv[4]), int(sys.argv[5]), int(sys.argv[6])
|
|
|
|
warmup = get_or_create_warmup(ip, method)
|
|
result = record_send_batch(warmup['id'], sent, inbox, spam, 0)
|
|
print(f"Recorded: {sent} sent, inbox rate: {result['inbox_rate']:.1f}%")
|
|
|
|
# Evaluate
|
|
eval_result = evaluate_and_adjust(warmup['id'])
|
|
if eval_result['action']:
|
|
print(f"Action: {eval_result['action']}")
|
|
|
|
else:
|
|
print(f"Unknown command: {cmd}")
|
|
print("Commands: status, check, allowance, record")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|