Add explain analysis workflow and UI

backend/cli.py

@@ -12,7 +12,7 @@ import os
 import shutil
 import subprocess
 import sys
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Optional
 from zoneinfo import ZoneInfo
@@ -21,18 +21,27 @@ import typer
 from rich.console import Console
 from rich.panel import Panel
 from rich.prompt import Confirm
+from rich.table import Table
+from dotenv import load_dotenv
 
 from backend.agents.prompt_loader import PromptLoader
 from backend.agents.workspace_manager import WorkspaceManager
+from backend.data.market_ingest import ingest_symbols
+from backend.data.market_store import MarketStore
+from backend.enrich.llm_enricher import get_explain_model_info, llm_enrichment_enabled
+from backend.enrich.news_enricher import enrich_symbols
 
 app = typer.Typer(
     name="evotraders",
     help="EvoTraders: A self-evolving multi-agent trading system",
     add_completion=False,
 )
+ingest_app = typer.Typer(help="Ingest Polygon market data into the research warehouse.")
+app.add_typer(ingest_app, name="ingest")
 
 console = Console()
 _prompt_loader = PromptLoader()
+load_dotenv()
 
 
 def get_project_root() -> Path:
@@ -204,6 +213,189 @@ def initialize_workspace(config_name: str) -> Path:
     return workspace_manager.get_run_dir(config_name)
 
 
+def _resolve_symbols(raw_tickers: Optional[str], config_name: Optional[str] = None) -> list[str]:
+    """Resolve symbols from explicit input or runtime bootstrap config."""
+    if raw_tickers and raw_tickers.strip():
+        return [
+            item.strip().upper()
+            for item in raw_tickers.split(",")
+            if item.strip()
+        ]
+
+    workspace_manager = WorkspaceManager(project_root=get_project_root())
+    bootstrap_path = workspace_manager.get_run_dir(config_name or "default") / "BOOTSTRAP.md"
+    if bootstrap_path.exists():
+        content = bootstrap_path.read_text(encoding="utf-8")
+        for line in content.splitlines():
+            if line.strip().startswith("tickers:"):
+                raw = line.split(":", 1)[1]
+                return [
+                    item.strip().upper()
+                    for item in raw.split(",")
+                    if item.strip()
+                ]
+    return []
+
+
+def _filter_problematic_report_rows(rows: list[dict]) -> list[dict]:
+    """Keep tickers with incomplete coverage or without any LLM-enriched rows."""
+    return [
+        row
+        for row in rows
+        if float(row.get("coverage_pct") or 0.0) < 100.0
+        or int(row.get("llm_count") or 0) == 0
+    ]
+
+
+def auto_update_market_store(
+    config_name: str,
+    *,
+    end_date: Optional[str] = None,
+) -> None:
+    """Refresh the long-lived Polygon market store for the active watchlist."""
+    api_key = os.getenv("POLYGON_API_KEY", "").strip()
+    if not api_key:
+        console.print(
+            "[dim]Skipping Polygon market store update: POLYGON_API_KEY not set[/dim]",
+        )
+        return
+
+    symbols = _resolve_symbols(None, config_name)
+    if not symbols:
+        console.print(
+            f"[dim]Skipping Polygon market store update: no tickers found for config '{config_name}'[/dim]",
+        )
+        return
+
+    target_end = end_date or datetime.now().date().isoformat()
+    console.print(
+        f"[cyan]Updating Polygon market store for {', '.join(symbols)} -> {target_end}[/cyan]",
+    )
+
+    try:
+        results = ingest_symbols(
+            symbols,
+            mode="incremental",
+            end_date=target_end,
+        )
+    except Exception as exc:
+        console.print(
+            f"[yellow]Polygon market store update failed, continuing startup: {exc}[/yellow]",
+        )
+        return
+
+    for result in results:
+        console.print(
+            "[green]"
+            f"{result['symbol']}"
+            "[/green] "
+            f"prices={result['prices']} news={result['news']} aligned={result['aligned']}"
+        )
+
+
+def auto_prepare_backtest_market_store(
+    config_name: str,
+    *,
+    start_date: str,
+    end_date: str,
+) -> None:
+    """Ensure the market store has the requested backtest window for the active watchlist."""
+    api_key = os.getenv("POLYGON_API_KEY", "").strip()
+    if not api_key:
+        console.print(
+            "[dim]Skipping Polygon backtest preload: POLYGON_API_KEY not set[/dim]",
+        )
+        return
+
+    symbols = _resolve_symbols(None, config_name)
+    if not symbols:
+        console.print(
+            f"[dim]Skipping Polygon backtest preload: no tickers found for config '{config_name}'[/dim]",
+        )
+        return
+
+    console.print(
+        f"[cyan]Preparing Polygon market store for backtest {start_date} -> {end_date} "
+        f"({', '.join(symbols)})[/cyan]",
+    )
+
+    try:
+        results = ingest_symbols(
+            symbols,
+            mode="full",
+            start_date=start_date,
+            end_date=end_date,
+        )
+    except Exception as exc:
+        console.print(
+            f"[yellow]Polygon backtest preload failed, continuing startup: {exc}[/yellow]",
+        )
+        return
+
+    for result in results:
+        console.print(
+            "[green]"
+            f"{result['symbol']}"
+            "[/green] "
+            f"prices={result['prices']} news={result['news']} aligned={result['aligned']}"
+        )
+
+
+def auto_enrich_market_store(
+    config_name: str,
+    *,
+    end_date: Optional[str] = None,
+    lookback_days: int = 120,
+    force: bool = False,
+) -> None:
+    """Refresh explain-oriented enriched news for the active watchlist."""
+    symbols = _resolve_symbols(None, config_name)
+    if not symbols:
+        console.print(
+            f"[dim]Skipping explain enrich: no tickers found for config '{config_name}'[/dim]",
+        )
+        return
+
+    target_end = end_date or datetime.now().date().isoformat()
+    try:
+        end_dt = datetime.strptime(target_end, "%Y-%m-%d")
+    except ValueError:
+        console.print(
+            f"[yellow]Skipping explain enrich: invalid end date {target_end}[/yellow]",
+        )
+        return
+
+    start_date = (end_dt - timedelta(days=max(1, lookback_days))).date().isoformat()
+    console.print(
+        f"[cyan]Refreshing explain enrich for {', '.join(symbols)} -> {target_end}[/cyan]",
+    )
+    store = MarketStore()
+    try:
+        results = enrich_symbols(
+            store,
+            symbols,
+            start_date=start_date,
+            end_date=target_end,
+            limit=300,
+            skip_existing=not force,
+        )
+    except Exception as exc:
+        console.print(
+            f"[yellow]Explain enrich failed, continuing startup: {exc}[/yellow]",
+        )
+        return
+
+    for result in results:
+        console.print(
+            "[green]"
+            f"{result['symbol']}"
+            "[/green] "
+            f"news={result['news_count']} queued={result['queued_count']} analyzed={result['analyzed']} "
+            f"skipped={result['skipped_existing_count']} deduped={result['deduped_count']} "
+            f"llm={result['llm_count']} local={result['local_count']}"
+        )
+
+
 @app.command("init-workspace")
 def init_workspace(
     config_name: str = typer.Option(
@@ -223,6 +415,213 @@ def init_workspace(
     )
 
 
+@ingest_app.command("full")
+def ingest_full(
+    tickers: Optional[str] = typer.Option(
+        None,
+        "--tickers",
+        "-t",
+        help="Comma-separated tickers to ingest",
+    ),
+    start: Optional[str] = typer.Option(
+        None,
+        "--start",
+        help="Start date for full ingestion (YYYY-MM-DD)",
+    ),
+    end: Optional[str] = typer.Option(
+        None,
+        "--end",
+        help="End date for ingestion (YYYY-MM-DD)",
+    ),
+    config_name: str = typer.Option(
+        "default",
+        "--config-name",
+        "-c",
+        help="Fallback config to read tickers from BOOTSTRAP.md",
+    ),
+):
+    """Run full Polygon ingestion for the specified symbols."""
+    symbols = _resolve_symbols(tickers, config_name)
+    if not symbols:
+        console.print("[red]No tickers provided and none found in BOOTSTRAP.md[/red]")
+        raise typer.Exit(1)
+
+    console.print(f"[cyan]Starting full Polygon ingest for {', '.join(symbols)}[/cyan]")
+    results = ingest_symbols(symbols, mode="full", start_date=start, end_date=end)
+    for result in results:
+        console.print(
+            f"[green]{result['symbol']}[/green] prices={result['prices']} news={result['news']} aligned={result['aligned']}"
+        )
+
+
+@ingest_app.command("update")
+def ingest_update(
+    tickers: Optional[str] = typer.Option(
+        None,
+        "--tickers",
+        "-t",
+        help="Comma-separated tickers to update",
+    ),
+    end: Optional[str] = typer.Option(
+        None,
+        "--end",
+        help="Optional end date override (YYYY-MM-DD)",
+    ),
+    config_name: str = typer.Option(
+        "default",
+        "--config-name",
+        "-c",
+        help="Fallback config to read tickers from BOOTSTRAP.md",
+    ),
+):
+    """Run incremental Polygon ingestion using stored watermarks."""
+    symbols = _resolve_symbols(tickers, config_name)
+    if not symbols:
+        console.print("[red]No tickers provided and none found in BOOTSTRAP.md[/red]")
+        raise typer.Exit(1)
+
+    console.print(f"[cyan]Starting incremental Polygon ingest for {', '.join(symbols)}[/cyan]")
+    results = ingest_symbols(symbols, mode="incremental", end_date=end)
+    for result in results:
+        console.print(
+            f"[green]{result['symbol']}[/green] prices={result['prices']} news={result['news']} aligned={result['aligned']}"
+        )
+
+
+@ingest_app.command("enrich")
+def ingest_enrich(
+    tickers: Optional[str] = typer.Option(
+        None,
+        "--tickers",
+        "-t",
+        help="Comma-separated tickers to enrich",
+    ),
+    start: Optional[str] = typer.Option(
+        None,
+        "--start",
+        help="Optional start date for enrichment window (YYYY-MM-DD)",
+    ),
+    end: Optional[str] = typer.Option(
+        None,
+        "--end",
+        help="Optional end date for enrichment window (YYYY-MM-DD)",
+    ),
+    limit: int = typer.Option(
+        300,
+        "--limit",
+        help="Maximum raw news rows per ticker to analyze",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Re-analyze already enriched news instead of only missing rows",
+    ),
+    config_name: str = typer.Option(
+        "default",
+        "--config-name",
+        "-c",
+        help="Fallback config to read tickers from BOOTSTRAP.md",
+    ),
+):
+    """Run explain-oriented news enrichment for symbols already in the market store."""
+    symbols = _resolve_symbols(tickers, config_name)
+    if not symbols:
+        console.print("[red]No tickers provided and none found in BOOTSTRAP.md[/red]")
+        raise typer.Exit(1)
+
+    console.print(f"[cyan]Starting explain enrich for {', '.join(symbols)}[/cyan]")
+    store = MarketStore()
+    results = enrich_symbols(
+        store,
+        symbols,
+        start_date=start,
+        end_date=end,
+        limit=max(10, limit),
+        skip_existing=not force,
+    )
+    for result in results:
+        console.print(
+            f"[green]{result['symbol']}[/green] "
+            f"news={result['news_count']} queued={result['queued_count']} analyzed={result['analyzed']} "
+            f"skipped={result['skipped_existing_count']} deduped={result['deduped_count']} "
+            f"llm={result['llm_count']} local={result['local_count']}"
+        )
+
+
+@ingest_app.command("report")
+def ingest_report(
+    tickers: Optional[str] = typer.Option(
+        None,
+        "--tickers",
+        "-t",
+        help="Optional comma-separated tickers to report",
+    ),
+    start: Optional[str] = typer.Option(
+        None,
+        "--start",
+        help="Optional start date for report window (YYYY-MM-DD)",
+    ),
+    end: Optional[str] = typer.Option(
+        None,
+        "--end",
+        help="Optional end date for report window (YYYY-MM-DD)",
+    ),
+    config_name: str = typer.Option(
+        "default",
+        "--config-name",
+        "-c",
+        help="Fallback config to read tickers from BOOTSTRAP.md",
+    ),
+    only_problematic: bool = typer.Option(
+        False,
+        "--only-problematic",
+        help="Only show tickers with incomplete coverage or no LLM-enriched news",
+    ),
+):
+    """Show explain enrichment coverage and freshness per ticker."""
+    symbols = _resolve_symbols(tickers, config_name)
+    store = MarketStore()
+    report_rows = store.get_enrich_report(
+        symbols=symbols or None,
+        start_date=start,
+        end_date=end,
+    )
+    if only_problematic:
+        report_rows = _filter_problematic_report_rows(report_rows)
+    if not report_rows:
+        if only_problematic:
+            console.print("[green]No problematic enrich report rows found for the requested scope[/green]")
+        else:
+            console.print("[yellow]No enrich report rows found for the requested scope[/yellow]")
+        raise typer.Exit(0)
+
+    model_info = get_explain_model_info()
+    model_label = model_info["label"] if llm_enrichment_enabled() else "disabled"
+    table = Table(title="Explain Enrichment Report")
+    table.add_column("Ticker", style="cyan")
+    table.add_column("Raw News", justify="right")
+    table.add_column("Analyzed", justify="right")
+    table.add_column("Coverage", justify="right")
+    table.add_column("LLM", justify="right")
+    table.add_column("Local", justify="right")
+    table.add_column("Latest Trade Date")
+    table.add_column("Latest Analysis")
+    table.caption = f"Explain LLM: {model_label}"
+
+    for row in report_rows:
+        table.add_row(
+            row["symbol"],
+            str(row["raw_news_count"]),
+            str(row["analyzed_news_count"]),
+            f'{row["coverage_pct"]:.1f}%',
+            str(row["llm_count"]),
+            str(row["local_count"]),
+            str(row["latest_trade_date"] or "-"),
+            str(row["latest_analysis_at"] or "-"),
+        )
+    console.print(table)
+
+
 @app.command()
 def backtest(
     start: Optional[str] = typer.Option(
@@ -331,6 +730,16 @@ def backtest(
 
     # Run data updater
     run_data_updater(project_root)
+    auto_prepare_backtest_market_store(
+        config_name,
+        start_date=start,
+        end_date=end,
+    )
+    auto_enrich_market_store(
+        config_name,
+        end_date=end,
+        force=False,
+    )
 
     # Build command using backend.main
     cmd = [
@@ -514,6 +923,15 @@ def live(
     # Data update (if not mock mode)
     if not mock:
         run_data_updater(project_root)
+        auto_update_market_store(
+            config_name,
+            end_date=nyse_now.date().isoformat(),
+        )
+        auto_enrich_market_store(
+            config_name,
+            end_date=nyse_now.date().isoformat(),
+            force=False,
+        )
     else:
         console.print(
             "\n[dim]Mock mode enabled - skipping data update[/dim]\n",