43 lines
1.9 KiB
Bash
Executable File
43 lines
1.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Run weekly: rebuild dataset, version it, push to HF if HF_TOKEN, log result
|
|
LOG=/var/log/weval/finetune-weekly.log
|
|
sudo touch $LOG; sudo chmod 666 $LOG
|
|
echo "[$(date -Iseconds)] === WEEKLY FINETUNE START ===" >> $LOG
|
|
source /etc/weval/secrets.env 2>/dev/null
|
|
|
|
# 1. Rebuild dataset
|
|
RESULT=$(/opt/weval-ops/top-ia/finetune_prep.sh 2>&1)
|
|
SAMPLES=$(echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('samples',0))" 2>/dev/null)
|
|
echo "[$(date -Iseconds)] Dataset built: $SAMPLES samples" >> $LOG
|
|
|
|
# 2. Version snapshot
|
|
TS=$(date +%Y%m%d-%H%M%S)
|
|
SNAPSHOT=/opt/weval-ops/top-ia/datasets/weval-finetune-$TS.jsonl
|
|
sudo mkdir -p /opt/weval-ops/top-ia/datasets 2>/dev/null
|
|
sudo cp /opt/weval-ops/top-ia/finetune-dataset.jsonl $SNAPSHOT 2>/dev/null
|
|
echo "[$(date -Iseconds)] Snapshot: $SNAPSHOT" >> $LOG
|
|
|
|
# 3. Optional: upload to HuggingFace dataset repo
|
|
HF_OK="skipped"
|
|
if [ -n "$HF_TOKEN" ] && [ "$SAMPLES" -gt 5 ]; then
|
|
# Check if huggingface-cli available
|
|
if command -v huggingface-cli >/dev/null 2>&1; then
|
|
HF_OK=$(huggingface-cli upload yace222/weval-finetune-dataset $SNAPSHOT --token=$HF_TOKEN 2>&1 | tail -1 | head -c 100)
|
|
else
|
|
# Fallback: use HF API directly
|
|
REPO="yace222/weval-finetune-dataset"
|
|
HF_OK=$(curl -sS --max-time 20 -X POST \
|
|
"https://huggingface.co/api/repos/create" \
|
|
-H "Authorization: Bearer $HF_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"name\":\"weval-finetune-dataset\",\"type\":\"dataset\",\"private\":true}" 2>&1 | head -c 200)
|
|
fi
|
|
fi
|
|
echo "[$(date -Iseconds)] HF: $HF_OK" >> $LOG
|
|
|
|
# 4. Audit log
|
|
/opt/weval-ops/top-ia/audit_log.sh "cron" "finetune_weekly" "weval-finetune" "{\"samples\":$SAMPLES,\"snapshot\":\"$SNAPSHOT\"}" >/dev/null 2>&1
|
|
|
|
echo "[$(date -Iseconds)] === DONE samples=$SAMPLES ===" >> $LOG
|
|
echo "{\"samples\":$SAMPLES,\"snapshot\":\"$SNAPSHOT\",\"hf\":\"$HF_OK\",\"log\":\"$LOG\"}"
|