147 lines
4.7 KiB
Bash
Executable File
147 lines
4.7 KiB
Bash
Executable File
#!/bin/bash
|
||
# Script d'automatisation Scrapping Factory Duassi
|
||
|
||
LOG_FILE="/opt/wevads/logs/duassi_automation.log"
|
||
CONFIG_FILE="/opt/wevads/configs/duassi_config.json"
|
||
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] DUASSI: $1" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
run_scrap_session() {
|
||
log "🕷️ Session Scrapping Duassi démarrée"
|
||
|
||
# Lancer le scrap via l'API
|
||
response=$(curl -s "http://127.0.0.1:5821/deliverads/scrapping-factory-enhanced.php?action=run_duassi")
|
||
|
||
if echo "$response" | grep -q '"success":true'; then
|
||
contacts_found=$(echo "$response" | grep -o '"contacts_found":[0-9]*' | cut -d: -f2)
|
||
log "✅ $contacts_found contacts high-value trouvés"
|
||
|
||
# Extraire et sauvegarder les contacts
|
||
echo "$response" | jq '.scrapping.contacts' > "/opt/wevads/data/duassi_contacts_$(date +%Y%m%d_%H%M%S).json"
|
||
|
||
# Alimenter SendData
|
||
feed_senddata "$response"
|
||
|
||
else
|
||
log "❌ Échec du scraping"
|
||
fi
|
||
}
|
||
|
||
feed_senddata() {
|
||
local response="$1"
|
||
|
||
log "📤 Alimentation SendData..."
|
||
|
||
# Extraire les contacts du JSON
|
||
contacts=$(echo "$response" | jq -c '.scrapping.contacts')
|
||
|
||
# Envoyer à SendData via API
|
||
feed_result=$(curl -s -X POST "http://127.0.0.1:5821/deliverads/scrapping-factory-enhanced.php?action=feed_adx" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$contacts")
|
||
|
||
sent=$(echo "$feed_result" | grep -o '"sent_to_senddata":[0-9]*' | cut -d: -f2)
|
||
|
||
if [ -n "$sent" ] && [ "$sent" -gt 0 ]; then
|
||
log "✅ $sent contacts alimentés dans SendData"
|
||
else
|
||
log "⚠️ Aucun contact alimenté dans SendData"
|
||
fi
|
||
}
|
||
|
||
check_low_data_offers() {
|
||
log "🔍 Vérification offres avec peu de données..."
|
||
|
||
# Récupérer les offres avec moins de 30% de données
|
||
low_data_offers=$(PGPASSWORD=admin123 psql -h localhost -U admin -d adx_system -t -c "
|
||
SELECT offer_name, data_coverage
|
||
FROM adx_offers
|
||
WHERE data_coverage < 30
|
||
ORDER BY data_coverage ASC
|
||
LIMIT 5;" 2>/dev/null)
|
||
|
||
if [ -n "$low_data_offers" ]; then
|
||
log "🎯 Offres à cibler:"
|
||
echo "$low_data_offers" | while read -r line; do
|
||
log " $line"
|
||
done
|
||
else
|
||
log "ℹ️ Toutes les offres ont suffisamment de données"
|
||
fi
|
||
}
|
||
|
||
generate_targeting_list() {
|
||
local offer_type="$1"
|
||
|
||
case "$offer_type" in
|
||
"crypto")
|
||
log "🎯 Génération liste ciblage Crypto..."
|
||
# Sources spécifiques crypto
|
||
sources=("coindesk.com" "cointelegraph.com" "binance.com" "coinmarketcap.com")
|
||
;;
|
||
"real_estate")
|
||
log "🎯 Génération liste ciblage Real Estate..."
|
||
sources=("bloomberg.com/real-estate" "forbes.com/real-estate" "therealdeal.com")
|
||
;;
|
||
"tech_vc")
|
||
log "🎯 Génération liste ciblage Tech VC..."
|
||
sources=("techcrunch.com" "venturebeat.com" "crunchbase.com" "angel.co")
|
||
;;
|
||
*)
|
||
log "🎯 Génération liste ciblage général..."
|
||
sources=("linkedin.com" "forbes.com" "bloomberg.com")
|
||
;;
|
||
esac
|
||
|
||
# Sauvegarder la liste de ciblage
|
||
printf '%s\n' "${sources[@]}" > "/opt/wevads/data/targeting_${offer_type}_$(date +%Y%m%d).txt"
|
||
log "✅ Liste de ciblage générée: targeting_${offer_type}_$(date +%Y%m%d).txt"
|
||
}
|
||
|
||
main() {
|
||
log "🚀 Démarrage automatisation Duassi"
|
||
|
||
# 1. Vérifier les offres avec peu de données
|
||
check_low_data_offers
|
||
|
||
# 2. Lancer une session de scraping
|
||
run_scrap_session
|
||
|
||
# 3. Générer des listes de ciblage spécifiques
|
||
generate_targeting_list "crypto"
|
||
generate_targeting_list "real_estate"
|
||
generate_targeting_list "tech_vc"
|
||
|
||
# 4. Synthèse
|
||
total_contacts=$(find /opt/wevads/data -name "duassi_contacts_*.json" -exec cat {} \; | jq '. | length' 2>/dev/null | paste -sd+ | bc)
|
||
total_contacts=${total_contacts:-0}
|
||
|
||
log "📊 SYNTHÈSE DUASSI:"
|
||
log " Contacts high-value collectés: $total_contacts"
|
||
log " Listes de ciblage générées: $(find /opt/wevads/data -name "targeting_*.txt" | wc -l)"
|
||
log " Prochaine exécution: $(date -d '+1 hour' '+%H:%M')"
|
||
|
||
log "🏁 Automatisation Duassi terminée"
|
||
}
|
||
|
||
# Exécution
|
||
case "$1" in
|
||
"run")
|
||
main
|
||
;;
|
||
"check")
|
||
check_low_data_offers
|
||
;;
|
||
"target")
|
||
generate_targeting_list "$2"
|
||
;;
|
||
*)
|
||
echo "Usage: $0 {run|check|target}"
|
||
echo " run - Exécution complète"
|
||
echo " check - Vérifier offres peu de données"
|
||
echo " target <type> - Générer liste ciblage"
|
||
;;
|
||
esac
|