"""Unified feature engineering for AI models. Single source of truth for indicator calculations and feature extraction. Used by both ai/dataset.py (training) and ai/inference.py (prediction). Architecture: - ~40 FEATURE_COLS × 3 summary stats + 7 OHLCV ratios + 6 scalars = 133 global features - 24 bars × 7 per-bar features = 168 sequence features - Total: ~301 features """ from __future__ import annotations from typing import List, Optional, Tuple import numpy as np import pandas as pd CONTEXT_WINDOW = 24 # ============================================================ # FEATURE_COLS — all indicators computed per window (39 total) # Each gets 3 summary stats in extract_feature_vector: # (last - mean) / std — z-score like # (last - first) / |mean| — change ratio # std / |mean| — coefficient of variation # ============================================================ FEATURE_COLS = [ # --- Momentum (10) --- "RSI_14", "rsi_centered", "MACD", "macd_norm", "macd_hist_sign", "Stoch_K", "stoch_kd_diff", "adx", "adx_directional_ratio", "williams_r", # --- Volatility (5) --- "ATR_14", "parkinson_vol", "garman_klass_vol", "bb_width_norm", "vol_percentile_252", # --- Trend/MA (6) --- "dist_to_sma_5", "dist_to_sma_24", "dist_to_sma_50", "zscore_24", "dist_to_high_24", "dist_to_low_24", # --- Interaction (5) --- "trend_over_vol", "momentum_regime", "vol_adj_return_1d", "trend_volume_confirm", "rsi_trend_diverg", # --- Price/Volume patterns (6, unchanged) --- "OBV", "body_pct", "wick_ratio", "vol_ratio", "pin_bar", "engulfing", # --- Other (2) --- "momentum_5", "spread_zscore", # --- Temporal (4) --- "dow_sin", "dow_cos", "is_month_end", "is_quarter_end", ] # Count: 10 + 5 + 6 + 5 + 6 + 2 + 4 = 38 assert len(FEATURE_COLS) == 38, f"FEATURE_COLS has {len(FEATURE_COLS)}, expected 38" def true_range(df: pd.DataFrame) -> pd.Series: high_low = df["High"] - df["Low"] high_close = abs(df["High"] - df["Close"].shift(1)) low_close = abs(df["Low"] - df["Close"].shift(1)) return pd.concat([high_low, high_close, low_close], axis=1).max(axis=1) def calculate_rsi(series: pd.Series, window: int) -> pd.Series: delta = series.diff() gain = delta.where(delta > 0, 0).rolling(window).mean() loss = (-delta.where(delta < 0, 0)).rolling(window).mean() rs = gain / (loss + 1e-10) return 100 - (100 / (1 + rs)) def calculate_atr(df: pd.DataFrame, window: int) -> pd.Series: tr = true_range(df) return tr.rolling(window).mean() def calculate_bollinger( series: pd.Series, window: int = 20, std_dev: float = 2.0 ) -> Tuple[pd.Series, pd.Series, pd.Series]: sma = series.rolling(window).mean() std = series.rolling(window).std() upper = sma + (std * std_dev) lower = sma - (std * std_dev) return upper, sma, lower def calculate_macd( series: pd.Series, fast: int = 12, slow: int = 26, signal: int = 9 ) -> Tuple[pd.Series, pd.Series]: ema_fast = series.ewm(span=fast, adjust=False).mean() ema_slow = series.ewm(span=slow, adjust=False).mean() macd = ema_fast - ema_slow macd_signal = macd.ewm(span=signal, adjust=False).mean() return macd, macd_signal def calculate_stochastic( df: pd.DataFrame, k_period: int = 14, d_period: int = 3 ) -> Tuple[pd.Series, pd.Series]: low_min = df["Low"].rolling(k_period).min() high_max = df["High"].rolling(k_period).max() k = 100 * (df["Close"] - low_min) / (high_max - low_min + 1e-10) d = k.rolling(d_period).mean() return k, d def calculate_obv(df: pd.DataFrame) -> pd.Series: return (np.sign(df["Close"].diff()) * df["Volume"]).fillna(0).cumsum() def parkinson_volatility(df: pd.DataFrame, window: int = 20) -> pd.Series: """Parkinson (1980) volatility estimator using High-Low range. ~5x more efficient than close-to-close. Assumes drift ≈ 0. Formula: σ = sqrt(1/(4*ln(2)) * mean(ln(High/Low)^2)) """ hl_ratio = np.log(df["High"] / df["Low"]) variance = (hl_ratio ** 2).rolling(window).mean() / (4 * np.log(2)) return np.sqrt(variance) def garman_klass_volatility(df: pd.DataFrame, window: int = 20) -> pd.Series: """Garman-Klass (1980) volatility estimator using OHLC. ~8x more efficient than close-to-close. Best general-purpose estimator. Formula: σ = sqrt(0.5*mean(ln(H/L)^2) - (2*ln(2)-1)*mean(ln(C/O)^2)) """ hl = np.log(df["High"] / df["Low"]) ** 2 co = np.log(df["Close"] / df["Open"]) ** 2 variance = (0.5 * hl - (2 * np.log(2) - 1) * co).rolling(window).mean() return np.sqrt(variance) def williams_r(df: pd.DataFrame, window: int = 14) -> pd.Series: """Williams %R. Similar to Stochastic but inverted. Range: [-100, 0]. Lower = more oversold. """ high_max = df["High"].rolling(window).max() low_min = df["Low"].rolling(window).min() return -100 * (high_max - df["Close"]) / (high_max - low_min + 1e-10) def percentile_rank(series: pd.Series, window: int = 252) -> pd.Series: """Rolling percentile rank: where current value sits in its history. Returns [0, 1]. Useful for regime detection. """ def _pct_rank(x): if len(x) < 2: return 0.5 return (x.values[-1] > x.values).mean() return series.rolling(window).apply(_pct_rank, raw=False) def add_technicals(df: pd.DataFrame) -> pd.DataFrame: """Add ALL technical indicators used by FEATURE_COLS and helpers. Single source of truth for feature computation. Every new feature must be computed HERE, not duplicated elsewhere. """ if df.empty: return df df = df.copy() df["returns"] = df["Close"].pct_change() df["log_returns"] = np.log(df["Close"] / df["Close"].shift(1)) # ---- Basic MAs for internal use ---- df["SMA_5"] = df["Close"].rolling(5).mean() df["SMA_24"] = df["Close"].rolling(24).mean() df["SMA_50"] = df["Close"].rolling(50).mean() # ---- 1. Momentum indicators ---- df["RSI_14"] = calculate_rsi(df["Close"], 14) df["rsi_centered"] = df["RSI_14"] - 50 # [-50, +50] macd_line, macd_signal = calculate_macd(df["Close"]) df["MACD"] = macd_line df["MACD_signal"] = macd_signal df["macd_norm"] = df["MACD"] / (df["Close"] + 1e-10) # scale-free df["macd_hist"] = df["MACD"] - df["MACD_signal"] df["macd_hist_sign"] = np.sign(df["macd_hist"]) # {-1, 0, +1} df["Stoch_K"], df["Stoch_D"] = calculate_stochastic(df) df["stoch_kd_diff"] = df["Stoch_K"] - df["Stoch_D"] # [-100, +100] # ADX components df["tr"] = true_range(df) plus_dm = df["High"].diff() minus_dm = -df["Low"].diff() plus_dm = plus_dm.where((plus_dm > 0) & (plus_dm > minus_dm), 0) minus_dm = minus_dm.where((minus_dm > 0) & (minus_dm > plus_dm), 0) atr14 = df["tr"].rolling(14).mean() df["di_plus"] = 100 * plus_dm.rolling(14).mean() / (atr14 + 1e-10) df["di_minus"] = 100 * minus_dm.rolling(14).mean() / (atr14 + 1e-10) df["adx"] = 100 * abs(df["di_plus"] - df["di_minus"]) / (df["di_plus"] + df["di_minus"] + 1e-10) df["adx_directional_ratio"] = (df["di_plus"] - df["di_minus"]) / (df["di_plus"] + df["di_minus"] + 1e-10) df["williams_r"] = williams_r(df, 14) # ---- 2. Volatility indicators ---- df["ATR_14"] = calculate_atr(df, 14) df["parkinson_vol"] = parkinson_volatility(df, 20) df["garman_klass_vol"] = garman_klass_volatility(df, 20) bb_upper, bb_mid, bb_lower = calculate_bollinger(df["Close"]) df["BB_upper"] = bb_upper df["BB_middle"] = bb_mid df["BB_lower"] = bb_lower df["bb_width_norm"] = (df["BB_upper"] - df["BB_lower"]) / (df["BB_middle"] + 1e-10) df["vol_percentile_252"] = percentile_rank(df["ATR_14"], 252) # ---- 3. Trend / MA indicators ---- df["dist_to_sma_5"] = df["Close"] / (df["SMA_5"] + 1e-10) - 1 df["dist_to_sma_24"] = df["Close"] / (df["SMA_24"] + 1e-10) - 1 df["dist_to_sma_50"] = df["Close"] / (df["SMA_50"] + 1e-10) - 1 df["zscore_24"] = (df["Close"] - df["SMA_24"]) / (df["Close"].rolling(24).std() + 1e-10) df["dist_to_high_24"] = df["Close"] / (df["High"].rolling(24).max() + 1e-10) - 1 # ≤ 0 df["dist_to_low_24"] = df["Close"] / (df["Low"].rolling(24).min() + 1e-10) - 1 # ≥ 0 # ---- 5. Price/Volume patterns ---- df["OBV"] = calculate_obv(df) df["body"] = df["Close"] - df["Open"] df["body_pct"] = abs(df["body"]) / (df["High"] - df["Low"] + 1e-10) df["upper_wick"] = df["High"] - df[["Open", "Close"]].max(axis=1) df["lower_wick"] = df[["Open", "Close"]].min(axis=1) - df["Low"] df["wick_ratio"] = (df["upper_wick"] + df["lower_wick"]) / (df["High"] - df["Low"] + 1e-10) df["vol_ratio"] = df["Volume"] / (df["Volume"].rolling(20).mean() + 1e-10) df["momentum_5"] = df["Close"] / (df["Close"].shift(5) + 1e-10) - 1 df["pin_bar"] = ((df["lower_wick"] > 2 * abs(df["body"]) + 1e-10) | (df["upper_wick"] > 2 * abs(df["body"]) + 1e-10)).astype(float) df["engulfing"] = ((abs(df["body"]) > abs(df["body"].shift(1)) + 1e-10) & (np.sign(df["body"]) != np.sign(df["body"].shift(1)))).astype(float) spread = df["High"] - df["Low"] df["spread_zscore"] = (spread - spread.rolling(20).mean()) / (spread.rolling(20).std() + 1e-10) # ---- 6. Temporal features ---- if "timestamp" in df.columns: dt_idx = pd.to_datetime(df["timestamp"], unit="s") elif "Date" in df.columns: dt_idx = pd.to_datetime(df["Date"]) else: dt_idx = df.index # Handle both Series (.dt accessor) and DatetimeIndex (direct access) if isinstance(dt_idx, pd.Series): dow = dt_idx.dt.dayofweek month_end = dt_idx.dt.is_month_end.astype(float) quarter_end = dt_idx.dt.is_quarter_end.astype(float) else: dow = dt_idx.dayofweek month_end = dt_idx.is_month_end.astype(float) quarter_end = dt_idx.is_quarter_end.astype(float) df["dow_sin"] = np.sin(2 * np.pi * dow / 5) df["dow_cos"] = np.cos(2 * np.pi * dow / 5) df["is_month_end"] = month_end df["is_quarter_end"] = quarter_end # ---- 7. Interaction features (MUST be last — depends on all above) ---- hist_vol_20 = df["returns"].rolling(20).std() df["trend_over_vol"] = df["dist_to_sma_24"] / (hist_vol_20 + 1e-10) df["momentum_regime"] = df["dist_to_sma_5"] - df["dist_to_sma_50"] df["vol_adj_return_1d"] = df["returns"] / (hist_vol_20 + 1e-10) df["trend_volume_confirm"] = df["dist_to_sma_24"] * df["vol_ratio"] df["rsi_trend_diverg"] = (df["RSI_14"] - 50) * df["dist_to_sma_24"] df = df.ffill().bfill().infer_objects(copy=False) return df def normalize(arr: np.ndarray) -> List[float]: """Z-score standardize array within window. Replaced min-max normalization to fix: - RuntimeWarning: overflow in scalar subtract (min_val near -inf) - Loss of absolute level info (RSI=70 and RSI=30 both map to [0,1]) """ arr = np.array(arr, dtype=np.float64) mean = float(np.mean(arr)) std = float(np.std(arr)) if std < 1e-10: return [0.0] * len(arr) return ((arr - mean) / std).tolist() def extract_feature_vector( window_df: pd.DataFrame, side: str = "LONG", atr_val: float = 0.0, ticker_id: int = 0, n_tickers: int = 1, entry_time: Optional[int] = None, context_window: int = CONTEXT_WINDOW, ) -> np.ndarray: """Unified feature extraction — SINGLE SOURCE OF TRUTH for train AND inference. 🚫 ЭТА ФУНКЦИЯ — ЕДИНСТВЕННОЕ МЕСТО ГЕНЕРАЦИИ ФИЧЕЙ. Никаких дублирующих реализаций в dataset_v2.py или inference_v2.py. Любое изменение здесь автоматом применяется и в обучении, и в инференсе. Feature vector structure (~301 components) — DUAL PATH layout: PATH A: Per-bar sequence (168 = context_window × 7) — для LSTM/Transformer [bar_0_features, bar_1_features, ..., bar_23_features] each bar: [Open_z, High_z, Low_z, Close_z, Volume_z, returns_z, log_returns_z] PATH B: Global features (133 = 7 summary + 114 FEATURE_COLS + 12 scalars) — для всех моделей - OHLCV last/mean ratios (5) - returns/log_returns last value (2) - FEATURE_COLS summary stats (38 × 3 = 114) - atr_val/100, side, time(3), ticker_id, session flag (6+) MLP: использует все ~301 фич как плоский вектор LSTM/Transformer: разделяет на sequence (168) + global (~133) Args: window_df: Context window of OHLCV data (WITH technicals pre-computed) side: Trading direction atr_val: Current ATR value ticker_id: Numeric ticker identifier n_tickers: Total number of tickers (for normalization) entry_time: Entry timestamp (int, Unix epoch) context_window: Context window size Returns: Feature vector as float32 numpy array """ df = window_df # technicals should already be pre-computed # ---- Step 1: Compute per-bar z-scores ---- seq_cols = ["Open", "High", "Low", "Close", "Volume"] ret_cols = ["returns", "log_returns"] all_seq_cols = seq_cols + ret_cols seq_arrays = {} for col in all_seq_cols: if col in df.columns: if col in ret_cols: vals = df[col].fillna(0).values else: vals = df[col].values seq_arrays[col] = normalize(vals) # ---- Step 2: Build per-bar sequence (interleaved by bar) ---- per_bar_seq = [] for bar_idx in range(context_window): for col in all_seq_cols: if col in seq_arrays: per_bar_seq.append(seq_arrays[col][bar_idx]) # ---- Step 3: Summary stats (OHLCV last/mean, returns/log_returns last) ---- summary = [] for col in seq_cols: if col in df.columns: vals = df[col].values summary.append(vals[-1] / (vals.mean() + 1e-10)) for col in ret_cols: if col in df.columns: vals = df[col].fillna(0).values summary.append(vals[-1]) # ---- Step 4: FEATURE_COLS summary stats ---- for col in FEATURE_COLS: if col in df.columns: vals = df[col].fillna(0).values mean = float(np.mean(vals)) std = float(np.std(vals)) last = float(vals[-1]) first = float(vals[0]) summary.append((last - mean) / (std + 1e-10)) summary.append((last - first) / (np.abs(mean) + 1e-10)) summary.append(std / (np.abs(mean) + 1e-10)) # ---- Step 5: Scalars ---- scalars = [atr_val / 100.0] scalars.append(1.0 if side == "LONG" else 0.0) try: if entry_time is not None: dt = pd.Timestamp(entry_time, unit="s") scalars.append(dt.hour / 23.0) scalars.append(dt.weekday() / 6.0) scalars.append(1.0 if 10 <= dt.hour < 16 else 0.0) else: scalars.extend([0.5, 0.5, 0.0]) except Exception: scalars.extend([0.5, 0.5, 0.0]) scalars.append(ticker_id / max(n_tickers, 1)) # ---- Combine: per-bar sequence + summary + scalars ---- features = per_bar_seq + summary + scalars return np.array(features, dtype=np.float32)