#!/usr/bin/env bash
# ==============================================================================
# AI_Strategy Observer — waits for ExpertEnsemble, then starts MoE
# ==============================================================================
# Запуск (PID auto-detect): nohup bash scripts/observer.sh > logs/observer.log 2>&1 &
# Запуск (with PID):        nohup bash scripts/observer.sh 1105477 > logs/observer.log 2>&1 &
# ==============================================================================

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
cd "$SCRIPT_DIR"

LOG_FILE="logs/observer.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

# ── 1. Определяем PID экспертов ──────────────────────────────────────────────
if [ -n "${1:-}" ]; then
    EXPERT_PID="$1"
    log "Using provided PID: $EXPERT_PID"
else
    # pgrep -f может ловить самого себя, используем ps с grep
    EXPERT_PID=$(ps aux | grep "[t]rain_experts_all" | awk '{print $2}' | head -1 || echo "")
    if [ -n "$EXPERT_PID" ]; then
        log "Auto-detected PID: $EXPERT_PID"
    else
        EXPERT_COUNT=$(find models/saved -name "*_experts.joblib" 2>/dev/null | wc -l)
        log "No ExpertEnsemble process. Models: $EXPERT_COUNT/42"
        if [ "$EXPERT_COUNT" -ge 42 ]; then
            log "All 42 models exist."
        else
            log "Only $EXPERT_COUNT/42. Exiting."
            exit 1
        fi
    fi
fi

# ── 2. Мониторинг ────────────────────────────────────────────────────────────
if [ -n "${EXPERT_PID:-}" ]; then
    log "Monitoring ExpertEnsemble (PID $EXPERT_PID) every 120s..."
    while ps -p "$EXPERT_PID" > /dev/null 2>&1; do
        LAST_LOG=$(tail -5 "$SCRIPT_DIR/logs/train_experts_full.log" 2>/dev/null | grep "\[[0-9]*/42\]" | tail -1)
        if [ -n "$LAST_LOG" ]; then
            log "Progress: $LAST_LOG"
        fi
        sleep 120
    done
    log "ExpertEnsemble FINISHED (PID $EXPERT_PID exited)"
fi

EXPERT_COUNT=$(find "$SCRIPT_DIR/models/saved" -name "*_experts.joblib" 2>/dev/null | wc -l)
log "ExpertEnsemble models saved: $EXPERT_COUNT / 42"

# ── 3. MoE: все 14 тикеров ───────────────────────────────────────────────────
log ""
log "============================================"
log "  STARTING MoE FOR ALL 14 TICKERS"
log "============================================"

MOE_TICKERS=("ASTR" "GAZP" "LKOH" "MOEX" "MTSS" "NSVZ" "NVTK" "PHOR" "PLZL" "ROSN" "SBER" "SNGSP" "VTBR" "X5")
TOTAL=${#MOE_TICKERS[@]}
COUNT=0
ERRORS=0

for ticker in "${MOE_TICKERS[@]}"; do
    COUNT=$((COUNT + 1))
    ticker_lower=$(echo "$ticker" | tr '[:upper:]' '[:lower:]')
    SAVE_PATH="models/saved/${ticker_lower}_moe_v12.joblib"

    log "[$COUNT/$TOTAL] Training MoE for $ticker..."
    START_TS=$(date +%s)

    if python3 main.py --ticker "$ticker" --moe --save "$SAVE_PATH" >> "$LOG_FILE" 2>&1; then
        END_TS=$(date +%s)
        DUR=$((END_TS - START_TS))
        log "[$COUNT/$TOTAL] ✓ $ticker done in ${DUR}s"
    else
        ERRORS=$((ERRORS + 1))
        log "[$COUNT/$TOTAL] ✗ $ticker FAILED"
    fi

    # GPU memory cooldown
    sleep 5
done

# ── 4. Итоги ──────────────────────────────────────────────────────────────────
log ""
log "============================================"
log "  ALL TRAINING COMPLETE"
log "  Date: $(date '+%Y-%m-%d %H:%M')"
log "  MoE errors: $ERRORS / $TOTAL"
log "============================================"
log ""
log "Final model counts:"
log "  Directional:    $(find models/saved -name '*_directional.joblib' 2>/dev/null | wc -l) / 42"
log "  ExpertEnsemble: $(find models/saved -name '*_experts.joblib' 2>/dev/null | wc -l) / 42"
log "  MoE:           $(find models/saved -name '*_moe_v12.joblib' 2>/dev/null | wc -l) / $TOTAL"