evotraders/backend/services/storage.py

# -*- coding: utf-8 -*-
"""
Storage Service - Data persistence and dashboard file management
Handles reading/writing dashboard JSON files and portfolio state
"""
# pylint: disable=R0904
import json
import logging
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional

from backend.data.market_store import MarketStore
from .runtime_db import RuntimeDb

logger = logging.getLogger(__name__)


class StorageService:
    """
    Storage service for data persistence

    Responsibilities:
    1. Export dashboard JSON files (compatibility layer)
        (summary, holdings, stats, trades, leaderboard)
    2. Load/save internal state (_internal_state.json)
    3. Load/save server state (server_state.json) with feed history
    4. Manage portfolio state persistence
    5. Support loading from saved state to resume execution

    Architecture Notes:
    - runs/<run_id>/ is the authoritative runtime state root
    - team_dashboard/*.json is a NON-AUTHORITATIVE export/compatibility layer
      for external consumers (frontend, reports, etc.)
    - Authoritative runtime reads should prefer:
      1. In-memory state (runtime manager)
      2. state/server_state.json
      3. state/runtime.db
      4. market_research.db
    - Compatibility exports can be disabled via ENABLE_DASHBOARD_COMPAT_EXPORTS=false
    """

    def __init__(
        self,
        dashboard_dir: Path,
        initial_cash: float = 100000.0,
        config_name: str = "runtime",
        enable_compat_exports: Optional[bool] = None,
    ):
        """
        Initialize storage service

        Args:
            dashboard_dir: Directory for dashboard files
            initial_cash: Initial cash amount
            config_name: Logical runtime config/run label for state directory context
            enable_compat_exports: Whether to keep writing team_dashboard/*.json
        """
        self.dashboard_dir = Path(dashboard_dir)
        self.dashboard_dir.mkdir(parents=True, exist_ok=True)
        self.initial_cash = initial_cash
        self.config_name = config_name
        self.enable_compat_exports = (
            self._resolve_compat_exports_default()
            if enable_compat_exports is None
            else bool(enable_compat_exports)
        )

        # Dashboard export file paths
        self.files = {
            "summary": self.dashboard_dir / "summary.json",
            "holdings": self.dashboard_dir / "holdings.json",
            "stats": self.dashboard_dir / "stats.json",
            "trades": self.dashboard_dir / "trades.json",
            "leaderboard": self.dashboard_dir / "leaderboard.json",
        }

        # Internal state file
        self.internal_state_file = self.dashboard_dir / "_internal_state.json"

        # Server state directory and file
        self.state_dir = self.dashboard_dir.parent / "state"
        self.state_dir.mkdir(parents=True, exist_ok=True)
        self.server_state_file = self.state_dir / "server_state.json"
        self.runtime_db = RuntimeDb(self.state_dir / "runtime.db")
        self.market_store = MarketStore()

        # Feed history (for agent messages)
        self.max_feed_history = 200

        # File modification time cache (for change detection)
        self.file_mtimes: Dict[str, float] = {}

        # Live returns tracking (for current trading session)
        self._session_start_equity: Optional[float] = None
        self._session_start_baseline: Optional[float] = None
        self._session_start_baseline_vw: Optional[float] = None
        self._session_start_momentum: Optional[float] = None
        self._live_return_history: List[Dict[str, Any]] = []

        logger.info(f"Storage service initialized: {self.dashboard_dir}")

    @staticmethod
    def _resolve_compat_exports_default() -> bool:
        """Default compatibility export policy, overridable via env."""
        raw = str(os.getenv("ENABLE_DASHBOARD_COMPAT_EXPORTS", "true")).strip().lower()
        return raw not in {"0", "false", "no", "off"}

    def load_export_file(self, file_type: str) -> Optional[Any]:
        """Load dashboard export JSON file."""
        file_path = self.files.get(file_type)
        if not file_path or not file_path.exists():
            return None

        try:
            with open(file_path, "r", encoding="utf-8") as f:
                return json.load(f)
        except Exception as e:
            logger.error(f"Failed to load {file_type}.json: {e}")
            return None

    def load_file(self, file_type: str) -> Optional[Any]:
        """Backward-compatible alias for export-layer JSON reads."""
        return self.load_export_file(file_type)

    def save_export_file(self, file_type: str, data: Any):
        """Save one compatibility dashboard export JSON file."""
        if not self.enable_compat_exports:
            return
        file_path = self.files.get(file_type)
        if not file_path:
            logger.error(f"Unknown file type: {file_type}")
            return

        try:
            with open(file_path, "w", encoding="utf-8") as f:
                json.dump(data, f, indent=2, ensure_ascii=False)
            if file_type == "leaderboard" and isinstance(data, list):
                self.runtime_db.replace_signals_for_leaderboard(data)
            elif file_type == "trades" and isinstance(data, list):
                for trade in data:
                    self.runtime_db.upsert_trade(trade)
        except Exception as e:
            logger.error(f"Failed to save {file_type}.json: {e}")

    def save_file(self, file_type: str, data: Any):
        """Backward-compatible alias for export-layer JSON writes."""
        self.save_export_file(file_type, data)

    def save_dashboard_exports(self, exports: Dict[str, Any]) -> None:
        """Persist compatibility dashboard exports from a normalized snapshot."""
        if not self.enable_compat_exports:
            return
        for file_type in ("summary", "holdings", "stats", "trades", "leaderboard"):
            if file_type in exports:
                self.save_export_file(file_type, exports[file_type])

    def read_persisted_server_state(self) -> Dict[str, Any]:
        """Read server_state.json without logging or DB side effects."""
        if not self.server_state_file.exists():
            return {}
        try:
            with open(self.server_state_file, "r", encoding="utf-8") as f:
                payload = json.load(f)
            return payload if isinstance(payload, dict) else {}
        except Exception as exc:
            logger.warning("Failed to read persisted server state: %s", exc)
            return {}

    def load_runtime_leaderboard(self, state: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
        """Prefer runtime state for leaderboard reads, fall back to export JSON."""
        runtime_state = state or self.read_persisted_server_state()
        leaderboard = runtime_state.get("leaderboard")
        if isinstance(leaderboard, list) and leaderboard:
            return leaderboard
        return self.load_export_file("leaderboard") or []

    def persist_runtime_leaderboard(
        self,
        leaderboard: List[Dict[str, Any]],
        state: Optional[Dict[str, Any]] = None,
    ) -> None:
        """Persist leaderboard to runtime state first, keeping JSON export for compatibility."""
        self.save_export_file("leaderboard", leaderboard)
        runtime_state = state or self.read_persisted_server_state()
        if not runtime_state:
            runtime_state = self.load_server_state()
        runtime_state["leaderboard"] = leaderboard
        self.save_server_state(runtime_state)

    def build_dashboard_snapshot_from_state(
        self,
        state: Optional[Dict[str, Any]] = None,
    ) -> Dict[str, Any]:
        """Build dashboard view data from runtime state instead of JSON exports."""
        runtime_state = state or self.load_server_state()
        persisted_state = self.read_persisted_server_state() if state is not None else {}
        portfolio = dict(
            runtime_state.get("portfolio")
            or persisted_state.get("portfolio")
            or {},
        )
        holdings = list(
            runtime_state.get("holdings")
            or persisted_state.get("holdings")
            or [],
        )
        stats = (
            runtime_state.get("stats")
            or persisted_state.get("stats")
            or self._get_default_stats()
        )
        trades = list(
            runtime_state.get("trades")
            or persisted_state.get("trades")
            or [],
        )
        leaderboard = list(
            runtime_state.get("leaderboard")
            or persisted_state.get("leaderboard")
            or [],
        )

        summary = {
            "totalAssetValue": portfolio.get("total_value", self.initial_cash),
            "totalReturn": portfolio.get("pnl_percent", 0.0),
            "cashPosition": portfolio.get("cash", self.initial_cash),
            "tickerWeights": stats.get("tickerWeights", {}),
            "totalTrades": len(trades),
            "pnlPct": portfolio.get("pnl_percent", 0.0),
            "balance": portfolio.get("total_value", self.initial_cash),
            "equity": portfolio.get("equity", []),
            "baseline": portfolio.get("baseline", []),
            "baseline_vw": portfolio.get("baseline_vw", []),
            "momentum": portfolio.get("momentum", []),
            "equity_return": portfolio.get("equity_return", []),
            "baseline_return": portfolio.get("baseline_return", []),
            "baseline_vw_return": portfolio.get("baseline_vw_return", []),
            "momentum_return": portfolio.get("momentum_return", []),
        }

        return {
            "summary": summary,
            "holdings": holdings,
            "stats": stats,
            "trades": trades,
            "leaderboard": leaderboard,
        }

    def check_file_updates(self) -> Dict[str, bool]:
        """
        Check which dashboard files have been updated since last check

        Returns:
            Dictionary mapping file_type to whether it was updated
        """
        updated = {}

        for file_type, file_path in self.files.items():
            if not file_path.exists():
                updated[file_type] = False
                continue

            try:
                current_mtime = file_path.stat().st_mtime
                last_mtime = self.file_mtimes.get(file_type, 0)

                if current_mtime > last_mtime:
                    updated[file_type] = True
                    self.file_mtimes[file_type] = current_mtime
                else:
                    updated[file_type] = False
            except Exception as e:
                logger.error(f"Failed to check file update ({file_type}): {e}")
                updated[file_type] = False

        return updated

    def load_internal_state(self) -> Dict[str, Any]:
        """
        Load internal state from file

        Returns:
            Internal state dictionary with default values
        """
        default_state = {
            "baseline_state": {"initialized": False, "initial_allocation": {}},
            "baseline_vw_state": {
                "initialized": False,
                "initial_allocation": {},
            },
            "momentum_state": {
                "positions": {},
                "cash": 0.0,
                "initialized": False,
            },
            "equity_history": [],
            "baseline_history": [],
            "baseline_vw_history": [],
            "momentum_history": [],
            "price_history": {},
            "portfolio_state": {
                "cash": self.initial_cash,
                "positions": {},
                "margin_used": 0.0,
            },
            "all_trades": [],
            "daily_position_history": {},
        }

        if not self.internal_state_file.exists():
            return default_state

        try:
            with open(self.internal_state_file, "r", encoding="utf-8") as f:
                data = json.load(f)

            # Merge with defaults
            for key, value in default_state.items():
                data.setdefault(key, value)

            logger.info("Loaded internal state from file")
            return data

        except Exception as e:
            logger.warning(
                f"Failed to load internal state, using defaults: {e}",
            )
            return default_state

    def save_internal_state(self, state: Dict[str, Any]):
        """
        Save internal state to file

        Args:
            state: Internal state dictionary
        """
        if not state:
            return

        try:
            with open(self.internal_state_file, "w", encoding="utf-8") as f:
                json.dump(state, f, indent=2, ensure_ascii=False)
            self._sync_price_history_to_db(state.get("price_history", {}))
        except Exception as e:
            logger.error(f"Failed to save internal state: {e}")

    def load_portfolio_state(self) -> Dict[str, Any]:
        """
        Load portfolio state from internal state

        Returns:
            Portfolio state dictionary: {cash, positions, margin_used}
        """
        internal_state = self.load_internal_state()
        portfolio_state = internal_state.get("portfolio_state", {})

        return {
            "cash": portfolio_state.get("cash", self.initial_cash),
            "positions": portfolio_state.get("positions", {}),
            "margin_used": portfolio_state.get("margin_used", 0.0),
            "margin_requirement": 0.25,  # Default 25% margin requirement
        }

    @staticmethod
    def _portfolio_is_pristine(portfolio_state: Dict[str, Any]) -> bool:
        """Return whether the persisted portfolio can be safely rebased."""
        positions = portfolio_state.get("positions", {})
        has_positions = any(
            position.get("long", 0) or position.get("short", 0)
            for position in positions.values()
        )
        margin_used = float(portfolio_state.get("margin_used", 0.0) or 0.0)
        return not has_positions and margin_used == 0.0

    def can_apply_initial_cash(self) -> bool:
        """Only allow initial cash changes before the run has traded."""
        state = self.load_internal_state()
        if not self._portfolio_is_pristine(state.get("portfolio_state", {})):
            return False
        if state.get("all_trades"):
            return False
        return len(state.get("equity_history", [])) <= 1

    def apply_initial_cash(self, initial_cash: float) -> bool:
        """Rebase storage state to a new initial cash when the run is pristine."""
        if not self.can_apply_initial_cash():
            return False

        self.initial_cash = float(initial_cash)
        if self.internal_state_file.exists():
            self.internal_state_file.unlink()

        self.initialize_empty_dashboard()
        state = self.load_server_state()
        self.update_server_state_from_dashboard(state)
        self.save_server_state(state)
        return True

    def save_portfolio_state(self, portfolio: Dict[str, Any]):
        """
        Save portfolio state to internal state

        Args:
            portfolio: Portfolio dictionary
        """
        internal_state = self.load_internal_state()
        internal_state["portfolio_state"] = {
            "cash": portfolio.get("cash", 0.0),
            "positions": portfolio.get("positions", {}),
            "margin_used": portfolio.get("margin_used", 0.0),
        }
        self.save_internal_state(internal_state)

    def initialize_empty_dashboard(self):
        """Initialize compatibility dashboard exports with default values."""
        self.save_dashboard_exports(
            {
                "summary": {
                    "totalAssetValue": self.initial_cash,
                    "totalReturn": 0.0,
                    "cashPosition": self.initial_cash,
                    "tickerWeights": {},
                    "totalTrades": 0,
                    "pnlPct": 0.0,
                    "balance": self.initial_cash,
                    "equity": [],
                    "baseline": [],
                    "baseline_vw": [],
                    "momentum": [],
                },
                "holdings": [],
                "stats": {
                    "totalAssetValue": self.initial_cash,
                    "totalReturn": 0.0,
                    "cashPosition": self.initial_cash,
                    "tickerWeights": {},
                    "totalTrades": 0,
                    "winRate": 0.0,
                    "bullBear": {
                        "bull": {"n": 0, "win": 0},
                        "bear": {"n": 0, "win": 0},
                    },
                },
                "trades": [],
            },
        )
        # Leaderboard with model info
        self.generate_leaderboard()

        logger.info("Initialized empty dashboard")

    def generate_leaderboard(self):
        """Generate leaderboard with agent model info from environment"""
        from ..config.constants import AGENT_CONFIG
        from ..llm.models import get_agent_model_info

        leaderboard = []
        ranking_entries = []
        team_entries = []

        for agent_id, config in AGENT_CONFIG.items():
            model_name, model_provider = get_agent_model_info(agent_id)

            entry = {
                "agentId": agent_id,
                "name": config["name"],
                "role": config["role"],
                "avatar": config["avatar"],
                "rank": None if config["is_team_role"] else 0,
                "winRate": None,
                "bull": {"n": 0, "win": 0, "unknown": 0},
                "bear": {"n": 0, "win": 0, "unknown": 0},
                "logs": [],
                "signals": [],
                "modelName": model_name,
                "modelProvider": model_provider,
            }

            if config["is_team_role"]:
                team_entries.append(entry)
            else:
                ranking_entries.append(entry)

        leaderboard = team_entries + ranking_entries
        self.persist_runtime_leaderboard(leaderboard)
        logger.info("Leaderboard generated with model info")

    def update_leaderboard_model_info(self):
        """
        Update model info in existing leaderboard (preserves performance data)
        """
        from ..config.constants import AGENT_CONFIG
        from ..llm.models import get_agent_model_info

        existing = self.load_runtime_leaderboard()

        if not existing:
            self.generate_leaderboard()
            return

        for entry in existing:
            agent_id = entry.get("agentId")
            if agent_id and agent_id in AGENT_CONFIG:
                model_name, model_provider = get_agent_model_info(agent_id)
                entry["modelName"] = model_name
                entry["modelProvider"] = model_provider

        self.persist_runtime_leaderboard(existing)
        logger.info("Leaderboard model info updated")

    def get_current_timestamp_ms(self, date: str = None) -> int:
        """
        Get timestamp in milliseconds from date string or current time

        Args:
            date: Optional date string (YYYY-MM-DD) for backtest compatibility.
                  Uses market close time (16:00) for the timestamp.

        Returns:
            Timestamp in milliseconds
        """
        if date:
            # Parse date and use market close time (16:00) for backtest
            dt = datetime.strptime(f"{date} 16:00:00", "%Y-%m-%d %H:%M:%S")
            return int(dt.timestamp() * 1000)
        return int(datetime.now().timestamp() * 1000)

    def calculate_portfolio_value(
        self,
        portfolio: Dict[str, Any],
        prices: Dict[str, float],
    ) -> float:
        """
        Calculate total portfolio value (net asset value)

        Args:
            portfolio: Portfolio state with cash and positions
            prices: Current prices for each ticker

        Returns:
            Total portfolio value
        """
        cash = portfolio.get("cash", 0.0)
        margin_used = portfolio.get("margin_used", 0.0)
        total = cash + margin_used

        positions = portfolio.get("positions", {})
        for ticker, position in positions.items():
            price = prices.get(ticker, 0)
            if price > 0:
                long_qty = position.get("long", 0)
                short_qty = position.get("short", 0)
                total += long_qty * price
                total -= short_qty * price

        return total

    def update_dashboard_after_cycle(
        self,
        portfolio: Dict[str, Any],
        prices: Dict[str, float],
        date: str,
        executed_trades: List[Dict[str, Any]] = None,
        baseline_values: Optional[Dict[str, float]] = None,
    ):
        """
        Update all dashboard files after a trading cycle

        Args:
            portfolio: Current portfolio state
            prices: Current prices for each ticker
            date: Trading date (YYYY-MM-DD)
            executed_trades: List of executed trades
            [{ticker, action, quantity, price}]
            baseline_values: Optional baseline portfolio values from settlement
                {equal_weight, market_cap_weighted, momentum}
        """
        # Use provided date for timestamp (backtest compatible)
        timestamp_ms = self.get_current_timestamp_ms(date)

        net_value = self.calculate_portfolio_value(portfolio, prices)

        state = self.load_internal_state()

        # Initialize all histories
        for key in [
            "equity_history",
            "baseline_history",
            "baseline_vw_history",
            "momentum_history",
        ]:
            if key not in state:
                state[key] = []

        # Add initial points if empty (all histories should start together)
        if len(state["equity_history"]) == 0:
            initial_point = {
                "t": timestamp_ms - 86400000,
                "v": round(self.initial_cash, 2),
            }
            state["equity_history"].append(initial_point)
            state["baseline_history"].append(initial_point.copy())
            state["baseline_vw_history"].append(initial_point.copy())
            state["momentum_history"].append(initial_point.copy())

        # Add current data points - all histories updated together
        state["equity_history"].append(
            {
                "t": timestamp_ms,
                "v": round(net_value, 2),
            },
        )

        # If baseline_values provided, use them;
        # otherwise forward-fill from last value
        if baseline_values:
            state["baseline_history"].append(
                {
                    "t": timestamp_ms,
                    "v": round(
                        baseline_values.get("equal_weight", self.initial_cash),
                        2,
                    ),
                },
            )
            state["baseline_vw_history"].append(
                {
                    "t": timestamp_ms,
                    "v": round(
                        baseline_values.get(
                            "market_cap_weighted",
                            self.initial_cash,
                        ),
                        2,
                    ),
                },
            )
            state["momentum_history"].append(
                {
                    "t": timestamp_ms,
                    "v": round(
                        baseline_values.get("momentum", self.initial_cash),
                        2,
                    ),
                },
            )
        else:
            # Forward-fill: use last known value
            last_baseline = (
                state["baseline_history"][-1]["v"]
                if state["baseline_history"]
                else self.initial_cash
            )
            last_baseline_vw = (
                state["baseline_vw_history"][-1]["v"]
                if state["baseline_vw_history"]
                else self.initial_cash
            )
            last_momentum = (
                state["momentum_history"][-1]["v"]
                if state["momentum_history"]
                else self.initial_cash
            )

            state["baseline_history"].append(
                {"t": timestamp_ms, "v": last_baseline},
            )
            state["baseline_vw_history"].append(
                {"t": timestamp_ms, "v": last_baseline_vw},
            )
            state["momentum_history"].append(
                {"t": timestamp_ms, "v": last_momentum},
            )

        state["portfolio_state"] = {
            "cash": portfolio.get("cash", 0.0),
            "positions": portfolio.get("positions", {}),
            "margin_used": portfolio.get("margin_used", 0.0),
        }

        # Update trades with structured data
        if executed_trades:
            if "all_trades" not in state:
                state["all_trades"] = []

            for i, trade in enumerate(executed_trades):
                action = trade.get("action", "hold")
                side = (
                    "LONG"
                    if action == "long"
                    else "SHORT"
                    if action == "short"
                    else "HOLD"
                )

                trade_id = (
                    f"t_{date.replace('-', '')}_{trade.get('ticker', '')}_{i}"
                )
                state["all_trades"].append(
                    {
                        "id": trade_id,
                        "ts": timestamp_ms,
                        "trading_date": date,
                        "side": side,
                        "ticker": trade.get("ticker", ""),
                        "qty": trade.get("quantity", 0),
                        "price": round(trade.get("price", 0), 2),
                    },
                )

        state["last_update_date"] = date

        self.save_internal_state(state)
        self.export_dashboard_compatibility_files(
            state,
            net_value=net_value,
            prices=prices,
        )

        logger.info(f"Dashboard updated: net_value=${net_value:,.2f}")

    def _build_summary_export(
        self,
        state: Dict[str, Any],
        net_value: float,
        prices: Dict[str, float],
    ) -> Dict[str, Any]:
        """Build compatibility summary export payload."""
        portfolio_state = state.get("portfolio_state", {})
        cash = portfolio_state.get("cash", self.initial_cash)

        # Calculate ticker weights
        positions = portfolio_state.get("positions", {})
        ticker_weights = {}

        for ticker, position in positions.items():
            price = prices.get(ticker, 0)
            if price > 0 and net_value > 0:
                long_qty = position.get("long", 0)
                short_qty = position.get("short", 0)
                position_value = (long_qty - short_qty) * price
                ticker_weights[ticker] = round(position_value / net_value, 4)

        # Calculate return
        total_return = (
            (net_value - self.initial_cash) / self.initial_cash
        ) * 100

        return {
            "totalAssetValue": round(net_value, 2),
            "totalReturn": round(total_return, 2),
            "cashPosition": round(cash, 2),
            "tickerWeights": ticker_weights,
            "totalTrades": len(state.get("all_trades", [])),
            "pnlPct": round(total_return, 2),
            "balance": round(net_value, 2),
            "equity": state.get("equity_history", []),
            "baseline": state.get("baseline_history", []),
            "baseline_vw": state.get("baseline_vw_history", []),
            "momentum": state.get("momentum_history", []),
        }

    def _build_holdings_export(
        self,
        state: Dict[str, Any],
        prices: Dict[str, float],
    ) -> List[Dict[str, Any]]:
        """Build compatibility holdings export payload."""
        portfolio_state = state.get("portfolio_state", {})
        positions = portfolio_state.get("positions", {})
        cash = portfolio_state.get("cash", self.initial_cash)
        margin_used = portfolio_state.get("margin_used", 0.0)

        # Calculate total value
        total_value = cash + margin_used
        for ticker, position in positions.items():
            price = prices.get(ticker, 0)
            if price > 0:
                long_qty = position.get("long", 0)
                short_qty = position.get("short", 0)
                total_value += (long_qty - short_qty) * price

        holdings = []

        # Add stock positions
        for ticker, position in positions.items():
            price = prices.get(ticker, 0)
            long_qty = position.get("long", 0)
            short_qty = position.get("short", 0)
            net_qty = long_qty - short_qty

            if net_qty != 0 and price > 0:
                market_value = net_qty * price
                weight = (
                    abs(market_value) / total_value if total_value > 0 else 0
                )

                holdings.append(
                    {
                        "ticker": ticker,
                        "quantity": net_qty,
                        "currentPrice": round(price, 2),
                        "marketValue": round(market_value, 2),
                        "weight": round(weight, 4),
                    },
                )

        # Add cash
        cash_weight = cash / total_value if total_value > 0 else 0
        holdings.append(
            {
                "ticker": "CASH",
                "quantity": 1,
                "currentPrice": round(cash, 2),
                "marketValue": round(cash, 2),
                "weight": round(cash_weight, 4),
            },
        )

        # Sort by weight
        holdings.sort(key=lambda x: abs(x["weight"]), reverse=True)
        return holdings

    def _build_stats_export(self, state: Dict[str, Any], net_value: float) -> Dict[str, Any]:
        """Build compatibility stats export payload."""
        portfolio_state = state.get("portfolio_state", {})
        cash = portfolio_state.get("cash", self.initial_cash)
        total_return = (
            (net_value - self.initial_cash) / self.initial_cash
        ) * 100

        return {
            "totalAssetValue": round(net_value, 2),
            "totalReturn": round(total_return, 2),
            "cashPosition": round(cash, 2),
            "tickerWeights": {},
            "totalTrades": len(state.get("all_trades", [])),
            "winRate": 0.0,
            "bullBear": {
                "bull": {"n": 0, "win": 0},
                "bear": {"n": 0, "win": 0},
            },
        }

    def _build_trades_export(self, state: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Build compatibility trades export payload."""
        all_trades = state.get("all_trades", [])

        sorted_trades = sorted(
            all_trades,
            key=lambda x: x.get("ts", 0),
            reverse=True,
        )

        trades = []
        for trade in sorted_trades[:100]:
            trades.append(
                {
                    "id": trade.get("id"),
                    "timestamp": trade.get("ts"),
                    "trading_date": trade.get("trading_date"),
                    "side": trade.get("side", ""),
                    "ticker": trade.get("ticker", ""),
                    "qty": trade.get("qty", 0),
                    "price": trade.get("price", 0),
                },
            )

        return trades

    def export_dashboard_compatibility_files(
        self,
        state: Dict[str, Any],
        *,
        net_value: float,
        prices: Dict[str, float],
    ) -> None:
        """Write compatibility dashboard exports from current runtime state."""
        self.save_dashboard_exports(
            {
                "summary": self._build_summary_export(state, net_value, prices),
                "holdings": self._build_holdings_export(state, prices),
                "stats": self._build_stats_export(state, net_value),
                "trades": self._build_trades_export(state),
            },
        )

    # Server State Management Methods

    def load_server_state(self) -> Dict[str, Any]:
        """
        Load server state from file

        Returns:
            Server state dictionary with feed_history and other data
        """
        default_state = {
            "status": "initializing",
            "current_date": None,
            "portfolio": {
                "total_value": self.initial_cash,
                "cash": self.initial_cash,
                "pnl_percent": 0.0,
                "equity": [],
                "baseline": [],
                "baseline_vw": [],
                "momentum": [],
                "strategies": [],
            },
            "holdings": [],
            "trades": [],
            "stats": self._get_default_stats(),
            "leaderboard": [],
            "realtime_prices": {},
            "system_started": datetime.now().isoformat(),
            "feed_history": [],
            "last_day_history": [],
            "trading_days_total": 0,
            "trading_days_completed": 0,
            "price_history": {},
        }

        if not self.server_state_file.exists():
            return default_state

        with open(self.server_state_file, "r", encoding="utf-8") as f:
            saved_state = json.load(f)

        # Merge with defaults to ensure all fields exist
        for key, value in default_state.items():
            saved_state.setdefault(key, value)

        logger.info(f"Server state loaded from: {self.server_state_file}")
        logger.info(
            f"Feed history: {len(saved_state.get('feed_history', []))} messages",  # noqa: E501
        )
        logger.info(
            f"Holdings: {len(saved_state.get('holdings', []))} items",
        )
        logger.info(f"Trades: {len(saved_state.get('trades', []))} records")

        for event in saved_state.get("feed_history", []):
            self.runtime_db.insert_event(event)
        for trade in saved_state.get("trades", []):
            self.runtime_db.upsert_trade(trade)

        return saved_state

    def save_server_state(self, state: Dict[str, Any]):
        """
        Save server state to file

        Args:
            state: Server state dictionary
        """
        state_to_save = {
            **state,
            "last_saved": datetime.now().isoformat(),
        }

        # Limit feed_history size
        if "feed_history" in state_to_save:
            state_to_save["feed_history"] = state_to_save["feed_history"][
                : self.max_feed_history
            ]

        # Limit trades
        if "trades" in state_to_save:
            state_to_save["trades"] = state_to_save["trades"][:100]

        with open(self.server_state_file, "w", encoding="utf-8") as f:
            json.dump(
                state_to_save,
                f,
                ensure_ascii=False,
                indent=2,
                default=str,
            )

        logger.debug(f"Server state saved to: {self.server_state_file}")

    def add_feed_message(
        self,
        state: Dict[str, Any],
        event: Dict[str, Any],
    ) -> bool:
        """
        Add a message to feed history

        Args:
            state: Server state dictionary to update
            event: Event dictionary with type, content, etc.

        Returns:
            True if message was added, False if filtered out
        """
        event_type = event.get("type", "")

        # Types to save in feed history
        save_types = {
            "system",
            "agent_message",
            "day_start",
            "day_complete",
            "day_error",
            "team_summary",
            "conference_start",
            "conference_message",
            "conference_end",
            "memory",
        }

        if event_type not in save_types:
            return False

        # Ensure timestamp exists
        if "timestamp" not in event:
            event["timestamp"] = datetime.now().isoformat()

        # Store event directly (flat structure, no metadata wrapper)
        feed_msg = dict(event)

        # Insert at beginning (newest first)
        if "feed_history" not in state:
            state["feed_history"] = []

        state["feed_history"].insert(0, feed_msg)
        self.runtime_db.insert_event(feed_msg)

        # Trim to max size
        if len(state["feed_history"]) > self.max_feed_history:
            state["feed_history"] = state["feed_history"][
                : self.max_feed_history
            ]

        return True

    def record_price_point(
        self,
        *,
        ticker: str,
        timestamp: str,
        price: float,
        open_price: Optional[float] = None,
        ret: Optional[float] = None,
        source: Optional[str] = None,
        meta: Optional[Dict[str, Any]] = None,
    ):
        """Persist a runtime price point for later query-oriented reads."""
        if not ticker or not timestamp:
            return
        try:
            self.runtime_db.insert_price_point(
                ticker=ticker,
                timestamp=timestamp,
                price=price,
                open_price=open_price,
                ret=ret,
                source=source,
                meta=meta,
            )
        except Exception as exc:
            logger.warning("Failed to record price point for %s: %s", ticker, exc)

    def _sync_price_history_to_db(self, price_history: Dict[str, Any]):
        """Backfill structured price points from serialized internal state."""
        if not isinstance(price_history, dict):
            return
        for ticker, points in price_history.items():
            if not ticker or not isinstance(points, list):
                continue
            for point in points:
                if isinstance(point, (list, tuple)) and len(point) >= 2:
                    timestamp, price = point[0], point[1]
                    try:
                        self.record_price_point(
                            ticker=str(ticker),
                            timestamp=str(timestamp),
                            price=float(price),
                        )
                    except (TypeError, ValueError):
                        continue
                elif isinstance(point, dict):
                    timestamp = point.get("timestamp") or point.get("label") or point.get("date")
                    price = point.get("price") or point.get("close") or point.get("value")
                    if not timestamp or price is None:
                        continue
                    try:
                        self.record_price_point(
                            ticker=str(ticker),
                            timestamp=str(timestamp),
                            price=float(price),
                            open_price=point.get("open"),
                            ret=point.get("ret"),
                            source=point.get("source"),
                            meta=point,
                        )
                    except (TypeError, ValueError):
                        continue

    def _get_default_stats(self) -> Dict[str, Any]:
        """Get default stats structure"""
        return {
            "totalAssetValue": 0.0,
            "totalReturn": -100.0,
            "cashPosition": 0.0,
            "tickerWeights": {},
            "totalTrades": 0,
            "winRate": 0.0,
            "bullBear": {
                "bull": {"n": 0, "win": 0},
                "bear": {"n": 0, "win": 0},
            },
        }

    def update_server_state_from_dashboard(self, state: Dict[str, Any]):
        """
        Update server state with current dashboard data

        Args:
            state: Server state dictionary to update
        """
        dashboard_snapshot = self.build_dashboard_snapshot_from_state(state)
        summary = dashboard_snapshot.get("summary") or {}
        holdings = dashboard_snapshot.get("holdings") or []
        stats = dashboard_snapshot.get("stats") or self._get_default_stats()
        trades = dashboard_snapshot.get("trades") or []
        leaderboard = dashboard_snapshot.get("leaderboard") or []
        internal_state = self.load_internal_state()

        # Update state
        state["portfolio"] = {
            "total_value": summary.get("totalAssetValue", self.initial_cash),
            "cash": summary.get("cashPosition", self.initial_cash),
            "pnl_percent": summary.get("pnlPct", 0.0),
            "equity": summary.get("equity", []),
            "baseline": summary.get("baseline", []),
            "baseline_vw": summary.get("baseline_vw", []),
            "momentum": summary.get("momentum", []),
            "strategies": [],
            # Live returns (will be populated when session is active)
            "equity_return": [],
            "baseline_return": [],
            "baseline_vw_return": [],
            "momentum_return": [],
        }
        state["holdings"] = holdings
        state["stats"] = stats
        state["trades"] = trades
        state["leaderboard"] = leaderboard
        state["price_history"] = internal_state.get("price_history", {})
        self.runtime_db.replace_signals_for_leaderboard(leaderboard)
        self._sync_price_history_to_db(state["price_history"])

    # ========== Live Returns Tracking ==========

    def start_live_session(self):
        """
        Start tracking live returns for current trading session.
        Captures current values as session start baseline.
        """
        state = self.load_internal_state()

        # Capture current values as session start
        equity_history = state.get("equity_history", [])
        baseline_history = state.get("baseline_history", [])
        baseline_vw_history = state.get("baseline_vw_history", [])
        momentum_history = state.get("momentum_history", [])

        self._session_start_equity = (
            equity_history[-1]["v"]
            if equity_history
            else self.initial_cash
        )
        self._session_start_baseline = (
            baseline_history[-1]["v"]
            if baseline_history
            else self.initial_cash
        )
        self._session_start_baseline_vw = (
            baseline_vw_history[-1]["v"]
            if baseline_vw_history
            else self.initial_cash
        )
        self._session_start_momentum = (
            momentum_history[-1]["v"]
            if momentum_history
            else self.initial_cash
        )

        # Clear live return history
        self._live_return_history = []

        # Add starting point at 0%
        timestamp = int(datetime.now().timestamp() * 1000)
        self._live_return_history.append(
            {
                "t": timestamp,
                "equity": 0.0,
                "baseline": 0.0,
                "baseline_vw": 0.0,
                "momentum": 0.0,
            },
        )

        logger.info(
            "Live session started: "
            f"equity=${self._session_start_equity:,.2f}, "
            f"baseline=${self._session_start_baseline:,.2f}",
        )

    def end_live_session(self):
        """End live returns tracking session"""
        self._session_start_equity = None
        self._session_start_baseline = None
        self._session_start_baseline_vw = None
        self._session_start_momentum = None
        self._live_return_history = []
        logger.info("Live session ended")

    def update_live_returns(
        self,
        current_equity: Optional[float] = None,
        current_baseline: Optional[float] = None,
        current_baseline_vw: Optional[float] = None,
        current_momentum: Optional[float] = None,
    ) -> Optional[Dict[str, Any]]:
        """
        Update live returns with current values.

        Args:
            current_equity: Current portfolio value
            current_baseline: Current EW baseline value
            current_baseline_vw: Current VW baseline value
            current_momentum: Current momentum strategy value

        Returns:
            Dict with live return data point or None if session not active
        """
        if (
            self._session_start_equity is None
            or self._session_start_baseline is None
            or self._session_start_baseline_vw is None
            or self._session_start_momentum is None
        ):
            return None

        timestamp = int(datetime.now().timestamp() * 1000)
        point = {"t": timestamp}

        # Calculate returns (only if we have valid values)
        if current_equity is not None and self._session_start_equity > 0:
            ret = (
                (current_equity - self._session_start_equity)
                / self._session_start_equity
            ) * 100
            point["equity"] = round(ret, 4)

        if current_baseline is not None and self._session_start_baseline > 0:
            ret = (
                (current_baseline - self._session_start_baseline)
                / self._session_start_baseline
            ) * 100
            point["baseline"] = round(ret, 4)

        if (
            current_baseline_vw is not None
            and self._session_start_baseline_vw > 0
        ):
            ret = (
                (current_baseline_vw - self._session_start_baseline_vw)
                / self._session_start_baseline_vw
            ) * 100
            point["baseline_vw"] = round(ret, 4)

        if current_momentum is not None and self._session_start_momentum > 0:
            ret = (
                (current_momentum - self._session_start_momentum)
                / self._session_start_momentum
            ) * 100
            point["momentum"] = round(ret, 4)

        # Only add point if we have at least one return value
        if any(k != "t" for k in point):
            self._live_return_history.append(point)
            # Limit history size
            if len(self._live_return_history) > 500:
                self._live_return_history = self._live_return_history[-500:]
            return point

        return None

    def get_live_returns(self) -> Dict[str, List[Dict[str, Any]]]:
        """
        Get live return curves for the current session.

        Returns:
            Dict with separate arrays for each strategy:
            - equity_return: [{t, v}, ...]
            - baseline_return: [{t, v}, ...]
            - baseline_vw_return: [{t, v}, ...]
            - momentum_return: [{t, v}, ...]
        """
        if not self._live_return_history:
            return {
                "equity_return": [],
                "baseline_return": [],
                "baseline_vw_return": [],
                "momentum_return": [],
            }

        # Convert combined history to separate arrays
        equity_return = []
        baseline_return = []
        baseline_vw_return = []
        momentum_return = []

        for point in self._live_return_history:
            t = point["t"]
            if "equity" in point:
                equity_return.append({"t": t, "v": point["equity"]})
            if "baseline" in point:
                baseline_return.append({"t": t, "v": point["baseline"]})
            if "baseline_vw" in point:
                baseline_vw_return.append({"t": t, "v": point["baseline_vw"]})
            if "momentum" in point:
                momentum_return.append({"t": t, "v": point["momentum"]})

        return {
            "equity_return": equity_return,
            "baseline_return": baseline_return,
            "baseline_vw_return": baseline_vw_return,
            "momentum_return": momentum_return,
        }

    @property
    def is_live_session_active(self) -> bool:
        """Check if live session is active"""
        return self._session_start_equity is not None