Initial commit of integrated agent system

backend/domains/news.py

@@ -0,0 +1,320 @@
+# -*- coding: utf-8 -*-
+"""News/explain domain helpers shared by app surfaces and gateway fallbacks."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from backend.data.market_store import MarketStore
+from backend.data.market_ingest import update_ticker_incremental
+from backend.enrich.news_enricher import enrich_news_for_symbol
+from backend.explain.range_explainer import build_range_explanation
+from backend.explain.similarity_service import find_similar_days
+from backend.explain.story_service import get_or_create_stock_story
+
+
+def news_rows_need_enrichment(rows: list[dict[str, Any]]) -> bool:
+    """Return whether news rows are missing explain-oriented analysis fields."""
+    if not rows:
+        return True
+    return all(
+        not row.get("sentiment")
+        and not row.get("relevance")
+        and not row.get("key_discussion")
+        for row in rows
+    )
+
+
+def ensure_news_fresh(
+    store: MarketStore,
+    *,
+    ticker: str,
+    target_date: str | None = None,
+    refresh_if_stale: bool = True,
+) -> dict[str, Any]:
+    """Refresh raw news incrementally when stored watermarks are stale."""
+    normalized_target = str(target_date or "").strip()[:10]
+    if not normalized_target:
+        return {
+            "ticker": ticker,
+            "target_date": None,
+            "last_news_fetch": None,
+            "refreshed": False,
+        }
+
+    watermarks = store.get_ticker_watermarks(ticker)
+    last_news_fetch = str(watermarks.get("last_news_fetch") or "").strip()[:10]
+    refreshed = False
+    if refresh_if_stale and (not last_news_fetch or last_news_fetch < normalized_target):
+        update_ticker_incremental(
+            ticker,
+            end_date=normalized_target,
+            store=store,
+        )
+        refreshed = True
+        watermarks = store.get_ticker_watermarks(ticker)
+        last_news_fetch = str(watermarks.get("last_news_fetch") or "").strip()[:10]
+
+    return {
+        "ticker": ticker,
+        "target_date": normalized_target,
+        "last_news_fetch": last_news_fetch or None,
+        "refreshed": refreshed,
+    }
+
+
+def get_enriched_news(
+    store: MarketStore,
+    *,
+    ticker: str,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    limit: int = 100,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=end_date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    rows = store.get_news_items_enriched(
+        ticker,
+        start_date=start_date,
+        end_date=end_date,
+        limit=limit,
+    )
+    if news_rows_need_enrichment(rows):
+        enrich_news_for_symbol(
+            store,
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+        )
+        rows = store.get_news_items_enriched(
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+        )
+    return {"ticker": ticker, "news": rows, "freshness": freshness}
+
+
+def get_news_for_date(
+    store: MarketStore,
+    *,
+    ticker: str,
+    date: str,
+    limit: int = 20,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    rows = store.get_news_items_enriched(
+        ticker,
+        trade_date=date,
+        limit=limit,
+    )
+    if news_rows_need_enrichment(rows):
+        enrich_news_for_symbol(
+            store,
+            ticker,
+            start_date=date,
+            end_date=date,
+            limit=limit,
+        )
+        rows = store.get_news_items_enriched(
+            ticker,
+            trade_date=date,
+            limit=limit,
+        )
+    return {"ticker": ticker, "date": date, "news": rows, "freshness": freshness}
+
+
+def get_news_timeline(
+    store: MarketStore,
+    *,
+    ticker: str,
+    start_date: str,
+    end_date: str,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=end_date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    timeline = store.get_news_timeline_enriched(
+        ticker,
+        start_date=start_date,
+        end_date=end_date,
+    )
+    if not timeline:
+        enrich_news_for_symbol(
+            store,
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=200,
+        )
+        timeline = store.get_news_timeline_enriched(
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+        )
+    return {
+        "ticker": ticker,
+        "timeline": timeline,
+        "start_date": start_date,
+        "end_date": end_date,
+        "freshness": freshness,
+    }
+
+
+def get_news_categories(
+    store: MarketStore,
+    *,
+    ticker: str,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    limit: int = 200,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=end_date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    rows = store.get_news_items_enriched(
+        ticker,
+        start_date=start_date,
+        end_date=end_date,
+        limit=limit,
+    )
+    if news_rows_need_enrichment(rows):
+        enrich_news_for_symbol(
+            store,
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+        )
+    categories = store.get_news_categories_enriched(
+        ticker,
+        start_date=start_date,
+        end_date=end_date,
+        limit=limit,
+    )
+    return {"ticker": ticker, "categories": categories, "freshness": freshness}
+
+
+def get_similar_days_payload(
+    store: MarketStore,
+    *,
+    ticker: str,
+    date: str,
+    n_similar: int = 5,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    result = find_similar_days(
+        store,
+        symbol=ticker,
+        target_date=date,
+        top_k=n_similar,
+    )
+    result["freshness"] = freshness
+    return result
+
+
+def get_story_payload(
+    store: MarketStore,
+    *,
+    ticker: str,
+    as_of_date: str,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=as_of_date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    enrich_news_for_symbol(
+        store,
+        ticker,
+        end_date=as_of_date,
+        limit=80,
+    )
+    result = get_or_create_stock_story(
+        store,
+        symbol=ticker,
+        as_of_date=as_of_date,
+    )
+    result["freshness"] = freshness
+    return result
+
+
+def get_range_explain_payload(
+    store: MarketStore,
+    *,
+    ticker: str,
+    start_date: str,
+    end_date: str,
+    article_ids: list[str] | None = None,
+    limit: int = 100,
+    refresh_if_stale: bool = False,
+) -> dict[str, Any]:
+    freshness = ensure_news_fresh(
+        store,
+        ticker=ticker,
+        target_date=end_date,
+        refresh_if_stale=refresh_if_stale,
+    )
+    news_rows = []
+    if article_ids:
+        news_rows = store.get_news_by_ids_enriched(ticker, article_ids)
+    if not news_rows:
+        news_rows = store.get_news_items_enriched(
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+        )
+    if news_rows_need_enrichment(news_rows):
+        enrich_news_for_symbol(
+            store,
+            ticker,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+        )
+        news_rows = (
+            store.get_news_by_ids_enriched(ticker, article_ids)
+            if article_ids
+            else store.get_news_items_enriched(
+                ticker,
+                start_date=start_date,
+                end_date=end_date,
+                limit=limit,
+            )
+        )
+    result = build_range_explanation(
+        ticker=ticker,
+        start_date=start_date,
+        end_date=end_date,
+        news_rows=news_rows,
+    )
+    return {"ticker": ticker, "result": result, "freshness": freshness}