192 lines
8.8 KiB
Python
Executable File
192 lines
8.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""WEVIA Snapshot Archiver v2 - Direct rescue→GitHub (no SCP to S204 = no OOM)"""
|
|
import requests, json, subprocess, time, sys, os
|
|
|
|
HZ_TOKEN = "xUcbvWMjkMgetuTU0llazUgB85jc7aQBLMhQ79NZ1Yf7j2TRF598DfNxoVrMnVOj"
|
|
GH_TOKEN = "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"
|
|
GH_REPO = "Yacineutt/weval-archive"
|
|
HZ_HEADERS = {"Authorization": f"Bearer {HZ_TOKEN}", "Content-Type": "application/json"}
|
|
GH_HEADERS = {"Authorization": f"token {GH_TOKEN}", "Content-Type": "application/json"}
|
|
LOG = "/tmp/wevia-snapshot-archiver.log"
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%H:%M:%S')}] {msg}"
|
|
print(line, flush=True)
|
|
with open(LOG, "a") as f: f.write(line + "\n")
|
|
|
|
def hz_get(path):
|
|
return requests.get(f"https://api.hetzner.cloud/v1/{path}", headers=HZ_HEADERS, timeout=30).json()
|
|
|
|
def hz_post(path, data=None):
|
|
return requests.post(f"https://api.hetzner.cloud/v1/{path}", headers=HZ_HEADERS, json=data or {}, timeout=30).json()
|
|
|
|
def hz_delete(path):
|
|
return requests.delete(f"https://api.hetzner.cloud/v1/{path}", headers=HZ_HEADERS, timeout=30).json()
|
|
|
|
def ssh_rescue(ip, pw, cmd, timeout=300):
|
|
r = subprocess.run(['sshpass','-p',pw,'ssh','-o','StrictHostKeyChecking=no','-o','UserKnownHostsFile=/dev/null','-o','ConnectTimeout=15',f'root@{ip}',cmd],
|
|
capture_output=True, text=True, timeout=timeout)
|
|
out = r.stdout.strip()
|
|
if not out and r.stderr:
|
|
out = "STDERR:" + r.stderr[:200]
|
|
return out
|
|
|
|
def list_snapshots():
|
|
d = hz_get("images?type=snapshot&per_page=50")
|
|
snaps = []
|
|
for i in d.get("images",[]):
|
|
snaps.append({"id":i["id"],"name":i.get("description","?"),"size":i.get("image_size",0),
|
|
"disk":i.get("disk_size",0),"created":i["created"],"from":i.get("created_from",{}).get("name","?")})
|
|
return snaps
|
|
|
|
def archive_snapshot(snap_id, tag_name, release_name):
|
|
log(f"=== ARCHIVING SNAPSHOT {snap_id} -> {tag_name} ===")
|
|
|
|
# 1. Create temp server
|
|
log("Creating temp server...")
|
|
d = hz_post("servers", {"name":f"temp-{tag_name}-{int(time.time())%10000}","server_type":"ccx23","image":snap_id,
|
|
"location":"hel1","start_after_create":True,"networks":[12033255]})
|
|
if "server" not in d:
|
|
log(f"ERROR creating server: {d}")
|
|
return False
|
|
srv_id = d["server"]["id"]
|
|
pub_ip = d["server"]["public_net"]["ipv4"]["ip"]
|
|
log(f"Server {srv_id} created, IP={pub_ip}")
|
|
|
|
# 2. Wait for creation (50 x 20s = 17 min max)
|
|
for i in range(100):
|
|
time.sleep(30)
|
|
acts = hz_get(f"servers/{srv_id}/actions?sort=started:desc&per_page=3").get("actions",[])
|
|
if all(a["status"]=="success" for a in acts):
|
|
log(f"Server ready after {(i+1)*20}s")
|
|
break
|
|
if i % 5 == 4:
|
|
create_pct = next((a["progress"] for a in acts if a["command"]=="create_server"), "?")
|
|
log(f"Still creating... {create_pct}%")
|
|
else:
|
|
log("TIMEOUT waiting for server")
|
|
hz_delete(f"servers/{srv_id}")
|
|
return False
|
|
|
|
# 3. Rescue mode: poweroff -> enable_rescue -> poweron
|
|
log("Powering off...")
|
|
hz_post(f"servers/{srv_id}/actions/poweroff")
|
|
time.sleep(20)
|
|
|
|
log("Enabling rescue mode (no ssh_keys = password generated)...")
|
|
rescue = hz_post(f"servers/{srv_id}/actions/enable_rescue", {"type":"linux64"})
|
|
rescue_pw = rescue.get("root_password","")
|
|
if not rescue_pw:
|
|
rescue_pw = rescue.get("action",{}).get("root_password","")
|
|
log(f"Rescue PW: {'SET' if rescue_pw else 'EMPTY!'} (len={len(rescue_pw)})")
|
|
|
|
if not rescue_pw:
|
|
log("ERROR: No rescue password generated!")
|
|
hz_delete(f"servers/{srv_id}")
|
|
return False
|
|
|
|
time.sleep(5)
|
|
hz_post(f"servers/{srv_id}/actions/poweron")
|
|
log("Waiting 90s for rescue boot...")
|
|
time.sleep(150)
|
|
|
|
# 4. SSH to rescue
|
|
log("Connecting to rescue...")
|
|
test = ""
|
|
for attempt in range(8):
|
|
try:
|
|
test = ssh_rescue(pub_ip, rescue_pw, "echo CONNECTED && lsblk -f", timeout=30)
|
|
if "CONNECTED" in test:
|
|
log(f"SSH OK (attempt {attempt+1})")
|
|
break
|
|
except Exception as e:
|
|
log(f"SSH attempt {attempt+1}/8: {e}")
|
|
time.sleep(20)
|
|
|
|
if "CONNECTED" not in str(test):
|
|
log(f"SSH FAILED after 8 attempts: {test}")
|
|
hz_delete(f"servers/{srv_id}")
|
|
return False
|
|
|
|
# 5. Mount disk
|
|
log("Mounting disk...")
|
|
ssh_rescue(pub_ip, rescue_pw, "mkdir -p /mnt/data && mount /dev/sda1 /mnt/data 2>&1 || mount /dev/sda2 /mnt/data 2>&1 || mount /dev/sda3 /mnt/data 2>&1")
|
|
|
|
# Scan content
|
|
log("Scanning...")
|
|
content = ssh_rescue(pub_ip, rescue_pw, "du -sh /mnt/data/opt/* /mnt/data/var/www/* 2>/dev/null | sort -rh | head -20")
|
|
log(f"Content:\n{content}")
|
|
|
|
# 6. Create GitHub release FIRST (lightweight, from S204)
|
|
log("Creating GitHub release...")
|
|
rel = requests.post(f"https://api.github.com/repos/{GH_REPO}/releases", headers=GH_HEADERS, json={
|
|
"tag_name":tag_name, "name":release_name,
|
|
"body":f"Hetzner snapshot {snap_id}\nContent:\n{content}"
|
|
}, timeout=30).json()
|
|
rel_id = rel.get("id")
|
|
upload_url = rel.get("upload_url","").replace("{?name,label}","")
|
|
|
|
if not rel_id:
|
|
log(f"ERROR: GitHub release failed: {rel}")
|
|
hz_delete(f"servers/{srv_id}")
|
|
return False
|
|
log(f"Release created: id={rel_id}")
|
|
|
|
# 7. Create archives ON RESCUE + upload DIRECTLY to GitHub (S204 not involved = no OOM)
|
|
archives = [
|
|
("opt-all", "cd /mnt/data && tar czf /mnt/data/_tmp_opt-all.tar.gz --exclude='node_modules' --exclude='.git' --exclude='storage' --exclude='vendor' --exclude='cache' opt/ 2>&1; ls -lh /mnt/data/_tmp_opt-all.tar.gz"),
|
|
("www-all", "cd /mnt/data && tar czf /mnt/data/_tmp_www-all.tar.gz var/www/ 2>&1; ls -lh /mnt/data/_tmp_www-all.tar.gz"),
|
|
("configs", "cd /mnt/data && tar czf /mnt/data/_tmp_configs.tar.gz var/spool/cron/ etc/nginx/ etc/ssh/sshd_config root/.bashrc root/.bash_history root/firewall* root/*.py root/*.sh 2>&1; ls -lh /mnt/data/_tmp_configs.tar.gz"),
|
|
("postgresql", "cd /mnt/data && tar czf /mnt/data/_tmp_postgresql.tar.gz var/lib/postgresql/ 2>&1; ls -lh /mnt/data/_tmp_postgresql.tar.gz"),
|
|
]
|
|
|
|
for name, cmd in archives:
|
|
log(f"Creating {name}...")
|
|
r = ssh_rescue(pub_ip, rescue_pw, cmd, timeout=1800)
|
|
log(f"Archive: {r}")
|
|
|
|
# Upload DIRECTLY from rescue→GitHub via curl (0 RAM on S204!)
|
|
# Check file size — split if >500MB (curl OOM on rescue)
|
|
fsize_r = ssh_rescue(pub_ip, rescue_pw, f"stat -c%s /mnt/data/_tmp_{name}.tar.gz 2>/dev/null || echo 0")
|
|
fsize_mb = int(fsize_r.strip() or '0') // 1024 // 1024
|
|
log(f"Uploading {name} ({fsize_mb}MB) rescue->GitHub...")
|
|
if fsize_mb > 500:
|
|
ssh_rescue(pub_ip, rescue_pw, f"cd /mnt/data && split -b 400m _tmp_{name}.tar.gz _tmp_{name}_part_", timeout=1200)
|
|
parts_raw = ssh_rescue(pub_ip, rescue_pw, f"ls -1 /mnt/data/_tmp_{name}_part_* 2>/dev/null")
|
|
parts = [p.strip() for p in parts_raw.split(chr(10)) if p.strip()]
|
|
for pi, part in enumerate(parts):
|
|
pname = f"{tag_name}-{name}-part{pi+1}.gz"
|
|
up_cmd = f"curl -s -X POST -H 'Authorization: token {GH_TOKEN}' -H 'Content-Type: application/gzip' --data-binary @{part} '{upload_url}?name={pname}' -o /dev/null -w '%{{http_code}}'"
|
|
res = ssh_rescue(pub_ip, rescue_pw, up_cmd, timeout=900)
|
|
log(f" {pname} HTTP: {res}")
|
|
ssh_rescue(pub_ip, rescue_pw, f"rm -f {part}")
|
|
else:
|
|
upload_cmd = f"curl -s -X POST -H 'Authorization: token {GH_TOKEN}' -H 'Content-Type: application/gzip' --data-binary @/mnt/data/_tmp_{name}.tar.gz '{upload_url}?name={tag_name}-{name}.tar.gz' -o /dev/null -w '%{{http_code}}'"
|
|
res = ssh_rescue(pub_ip, rescue_pw, upload_cmd, timeout=900)
|
|
log(f"{name} HTTP: {res}")
|
|
|
|
|
|
# 8. Cleanup temp server
|
|
log(f"Deleting temp server {srv_id}...")
|
|
hz_delete(f"servers/{srv_id}")
|
|
log(f"=== SNAPSHOT {snap_id} ARCHIVED ===")
|
|
return True
|
|
|
|
if __name__ == "__main__":
|
|
action = sys.argv[1] if len(sys.argv) > 1 else "list"
|
|
|
|
if action == "list":
|
|
for s in list_snapshots():
|
|
print(f"ID:{s['id']} {s['name'][:50]:50s} {s['size']:.0f}GB {s['created'][:10]} from:{s['from']}")
|
|
|
|
elif action == "archive":
|
|
snap_id = int(sys.argv[2]) if len(sys.argv) > 2 else 0
|
|
tag = sys.argv[3] if len(sys.argv) > 3 else f"snap-{snap_id}"
|
|
name = sys.argv[4] if len(sys.argv) > 4 else f"Snapshot {snap_id}"
|
|
if snap_id:
|
|
result = archive_snapshot(snap_id, tag, name)
|
|
sys.exit(0 if result else 1)
|
|
|
|
elif action == "status":
|
|
print(open(LOG).read() if os.path.exists(LOG) else "No log yet")
|