473 lines
16 KiB
Python
473 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Run-scoped SQLite storage for query-oriented runtime history."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, Optional
|
|
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id TEXT PRIMARY KEY,
|
|
event_type TEXT NOT NULL,
|
|
timestamp TEXT,
|
|
agent_id TEXT,
|
|
agent_name TEXT,
|
|
ticker TEXT,
|
|
title TEXT,
|
|
content TEXT,
|
|
payload_json TEXT NOT NULL,
|
|
run_date TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_type_time ON events(event_type, timestamp DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_events_ticker_time ON events(ticker, timestamp DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS trades (
|
|
id TEXT PRIMARY KEY,
|
|
ticker TEXT NOT NULL,
|
|
side TEXT,
|
|
qty REAL,
|
|
price REAL,
|
|
timestamp TEXT,
|
|
trading_date TEXT,
|
|
agent_id TEXT,
|
|
meta_json TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_trades_ticker_time ON trades(ticker, timestamp DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS signals (
|
|
id TEXT PRIMARY KEY,
|
|
ticker TEXT NOT NULL,
|
|
agent_id TEXT,
|
|
agent_name TEXT,
|
|
role TEXT,
|
|
signal TEXT,
|
|
confidence REAL,
|
|
reasoning_json TEXT,
|
|
real_return REAL,
|
|
is_correct TEXT,
|
|
trade_date TEXT,
|
|
created_at TEXT,
|
|
meta_json TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_signals_ticker_date ON signals(ticker, trade_date DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_signals_agent_date ON signals(agent_id, trade_date DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS price_points (
|
|
id TEXT PRIMARY KEY,
|
|
ticker TEXT NOT NULL,
|
|
timestamp TEXT NOT NULL,
|
|
price REAL NOT NULL,
|
|
open_price REAL,
|
|
ret REAL,
|
|
source TEXT,
|
|
meta_json TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_price_points_ticker_time ON price_points(ticker, timestamp DESC);
|
|
"""
|
|
|
|
|
|
def _json_dumps(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
|
|
|
|
|
|
def _hash_key(*parts: Any) -> str:
|
|
raw = "::".join("" if part is None else str(part) for part in parts)
|
|
return hashlib.sha1(raw.encode("utf-8")).hexdigest()
|
|
|
|
|
|
class RuntimeDb:
|
|
"""Small SQLite helper for append-mostly runtime data."""
|
|
|
|
def __init__(self, db_path: Path):
|
|
self.db_path = Path(db_path)
|
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
self._init_db()
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
return conn
|
|
|
|
def _init_db(self):
|
|
with self._connect() as conn:
|
|
conn.executescript(SCHEMA)
|
|
|
|
def insert_event(self, event: Dict[str, Any]):
|
|
payload = dict(event or {})
|
|
if not payload:
|
|
return
|
|
|
|
event_id = payload.get("id") or _hash_key(
|
|
payload.get("type"),
|
|
payload.get("timestamp"),
|
|
payload.get("agentId") or payload.get("agent_id"),
|
|
payload.get("content"),
|
|
payload.get("title"),
|
|
)
|
|
ticker = payload.get("ticker")
|
|
if not ticker and isinstance(payload.get("tickers"), list) and len(payload["tickers"]) == 1:
|
|
ticker = payload["tickers"][0]
|
|
|
|
with self._connect() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO events
|
|
(id, event_type, timestamp, agent_id, agent_name, ticker, title, content, payload_json, run_date)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
event_id,
|
|
payload.get("type"),
|
|
payload.get("timestamp"),
|
|
payload.get("agentId") or payload.get("agent_id"),
|
|
payload.get("agentName") or payload.get("agent_name"),
|
|
ticker,
|
|
payload.get("title"),
|
|
payload.get("content"),
|
|
_json_dumps(payload),
|
|
payload.get("date") or payload.get("trading_date") or payload.get("run_date"),
|
|
),
|
|
)
|
|
|
|
def get_recent_feed_events(
|
|
self,
|
|
*,
|
|
limit: int = 200,
|
|
event_types: Optional[Iterable[str]] = None,
|
|
) -> list[Dict[str, Any]]:
|
|
"""Return recent persisted feed events in newest-first order."""
|
|
event_types = tuple(event_types or ())
|
|
sql = """
|
|
SELECT payload_json
|
|
FROM events
|
|
"""
|
|
params: list[Any] = []
|
|
if event_types:
|
|
placeholders = ",".join("?" for _ in event_types)
|
|
sql += f" WHERE event_type IN ({placeholders})"
|
|
params.extend(event_types)
|
|
sql += " ORDER BY timestamp DESC LIMIT ?"
|
|
params.append(max(1, int(limit)))
|
|
|
|
with self._connect() as conn:
|
|
rows = conn.execute(sql, params).fetchall()
|
|
|
|
items: list[Dict[str, Any]] = []
|
|
for row in rows:
|
|
try:
|
|
payload = json.loads(row["payload_json"]) if row["payload_json"] else {}
|
|
except json.JSONDecodeError:
|
|
payload = {}
|
|
if payload:
|
|
items.append(payload)
|
|
return items
|
|
|
|
def get_last_day_feed_events(
|
|
self,
|
|
*,
|
|
current_date: Optional[str] = None,
|
|
limit: int = 200,
|
|
event_types: Optional[Iterable[str]] = None,
|
|
) -> list[Dict[str, Any]]:
|
|
"""Return latest trading day events in newest-first order for replay."""
|
|
event_types = tuple(event_types or ())
|
|
target_date = str(current_date or "").strip() or None
|
|
|
|
with self._connect() as conn:
|
|
if not target_date:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT run_date
|
|
FROM events
|
|
WHERE run_date IS NOT NULL AND TRIM(run_date) != ''
|
|
ORDER BY run_date DESC
|
|
LIMIT 1
|
|
"""
|
|
).fetchone()
|
|
target_date = row["run_date"] if row else None
|
|
|
|
if not target_date:
|
|
return []
|
|
|
|
sql = """
|
|
SELECT payload_json
|
|
FROM events
|
|
WHERE run_date = ?
|
|
"""
|
|
params: list[Any] = [target_date]
|
|
if event_types:
|
|
placeholders = ",".join("?" for _ in event_types)
|
|
sql += f" AND event_type IN ({placeholders})"
|
|
params.extend(event_types)
|
|
sql += " ORDER BY timestamp DESC LIMIT ?"
|
|
params.append(max(1, int(limit)))
|
|
rows = conn.execute(sql, params).fetchall()
|
|
|
|
items: list[Dict[str, Any]] = []
|
|
for row in rows:
|
|
try:
|
|
payload = json.loads(row["payload_json"]) if row["payload_json"] else {}
|
|
except json.JSONDecodeError:
|
|
payload = {}
|
|
if payload:
|
|
items.append(payload)
|
|
return items
|
|
|
|
def upsert_trade(self, trade: Dict[str, Any]):
|
|
payload = dict(trade or {})
|
|
if not payload:
|
|
return
|
|
|
|
trade_id = payload.get("id") or _hash_key(
|
|
payload.get("ticker"),
|
|
payload.get("timestamp") or payload.get("ts"),
|
|
payload.get("side"),
|
|
payload.get("qty"),
|
|
payload.get("price"),
|
|
)
|
|
with self._connect() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT OR REPLACE INTO trades
|
|
(id, ticker, side, qty, price, timestamp, trading_date, agent_id, meta_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
trade_id,
|
|
payload.get("ticker"),
|
|
payload.get("side"),
|
|
payload.get("qty"),
|
|
payload.get("price"),
|
|
payload.get("timestamp") or payload.get("ts"),
|
|
payload.get("trading_date"),
|
|
payload.get("agentId") or payload.get("agent_id"),
|
|
_json_dumps(payload),
|
|
),
|
|
)
|
|
|
|
def upsert_signal(self, signal: Dict[str, Any], *, agent_id: str, agent_name: str, role: str):
|
|
payload = dict(signal or {})
|
|
ticker = payload.get("ticker")
|
|
if not ticker:
|
|
return
|
|
|
|
signal_id = _hash_key(
|
|
agent_id,
|
|
ticker,
|
|
payload.get("date"),
|
|
payload.get("signal"),
|
|
payload.get("confidence"),
|
|
)
|
|
with self._connect() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT OR REPLACE INTO signals
|
|
(id, ticker, agent_id, agent_name, role, signal, confidence, reasoning_json,
|
|
real_return, is_correct, trade_date, created_at, meta_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
signal_id,
|
|
ticker,
|
|
agent_id,
|
|
agent_name,
|
|
role,
|
|
payload.get("signal"),
|
|
payload.get("confidence"),
|
|
_json_dumps(payload.get("reasoning")),
|
|
payload.get("real_return"),
|
|
None if payload.get("is_correct") is None else str(payload.get("is_correct")),
|
|
payload.get("date"),
|
|
payload.get("created_at") or payload.get("date"),
|
|
_json_dumps(payload),
|
|
),
|
|
)
|
|
|
|
def replace_signals_for_leaderboard(self, leaderboard: Iterable[Dict[str, Any]]):
|
|
with self._connect() as conn:
|
|
conn.execute("DELETE FROM signals")
|
|
for agent in leaderboard:
|
|
agent_id = agent.get("agentId")
|
|
agent_name = agent.get("name")
|
|
role = agent.get("role")
|
|
for signal in agent.get("signals", []) or []:
|
|
payload = dict(signal or {})
|
|
ticker = payload.get("ticker")
|
|
if not ticker:
|
|
continue
|
|
signal_id = _hash_key(
|
|
agent_id,
|
|
ticker,
|
|
payload.get("date"),
|
|
payload.get("signal"),
|
|
payload.get("confidence"),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO signals
|
|
(id, ticker, agent_id, agent_name, role, signal, confidence, reasoning_json,
|
|
real_return, is_correct, trade_date, created_at, meta_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
signal_id,
|
|
ticker,
|
|
agent_id,
|
|
agent_name,
|
|
role,
|
|
payload.get("signal"),
|
|
payload.get("confidence"),
|
|
_json_dumps(payload.get("reasoning")),
|
|
payload.get("real_return"),
|
|
None if payload.get("is_correct") is None else str(payload.get("is_correct")),
|
|
payload.get("date"),
|
|
payload.get("created_at") or payload.get("date"),
|
|
_json_dumps(payload),
|
|
),
|
|
)
|
|
|
|
def insert_price_point(
|
|
self,
|
|
*,
|
|
ticker: str,
|
|
timestamp: str,
|
|
price: float,
|
|
open_price: Optional[float] = None,
|
|
ret: Optional[float] = None,
|
|
source: Optional[str] = None,
|
|
meta: Optional[Dict[str, Any]] = None,
|
|
):
|
|
price_id = _hash_key(ticker, timestamp, price, open_price, ret)
|
|
with self._connect() as conn:
|
|
conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO price_points
|
|
(id, ticker, timestamp, price, open_price, ret, source, meta_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
price_id,
|
|
ticker,
|
|
timestamp,
|
|
price,
|
|
open_price,
|
|
ret,
|
|
source,
|
|
_json_dumps(meta or {}),
|
|
),
|
|
)
|
|
|
|
def get_stock_explain_snapshot(
|
|
self,
|
|
ticker: str,
|
|
*,
|
|
limit_events: int = 24,
|
|
limit_trades: int = 12,
|
|
limit_signals: int = 12,
|
|
) -> Dict[str, list[Dict[str, Any]]]:
|
|
"""Fetch query-oriented history for a single ticker."""
|
|
symbol = str(ticker or "").strip().upper()
|
|
if not symbol:
|
|
return {"events": [], "trades": [], "signals": []}
|
|
|
|
with self._connect() as conn:
|
|
trade_rows = conn.execute(
|
|
"""
|
|
SELECT * FROM trades
|
|
WHERE ticker = ?
|
|
ORDER BY timestamp DESC
|
|
LIMIT ?
|
|
""",
|
|
(symbol, limit_trades),
|
|
).fetchall()
|
|
signal_rows = conn.execute(
|
|
"""
|
|
SELECT * FROM signals
|
|
WHERE ticker = ?
|
|
ORDER BY trade_date DESC, created_at DESC
|
|
LIMIT ?
|
|
""",
|
|
(symbol, limit_signals),
|
|
).fetchall()
|
|
event_rows = conn.execute(
|
|
"""
|
|
SELECT * FROM events
|
|
WHERE payload_json LIKE ? OR content LIKE ? OR title LIKE ? OR ticker = ?
|
|
ORDER BY timestamp DESC
|
|
LIMIT ?
|
|
""",
|
|
(f"%{symbol}%", f"%{symbol}%", f"%{symbol}%", symbol, limit_events * 3),
|
|
).fetchall()
|
|
|
|
normalized_events = []
|
|
seen_event_ids: set[str] = set()
|
|
for row in event_rows:
|
|
payload = json.loads(row["payload_json"]) if row["payload_json"] else {}
|
|
content = str(row["content"] or payload.get("content") or "")
|
|
title = str(row["title"] or payload.get("title") or "")
|
|
if symbol not in f"{title} {content}".upper() and str(row["ticker"] or "").upper() != symbol:
|
|
continue
|
|
event_id = row["id"]
|
|
if event_id in seen_event_ids:
|
|
continue
|
|
seen_event_ids.add(event_id)
|
|
normalized_events.append(
|
|
{
|
|
"id": event_id,
|
|
"type": "mention",
|
|
"timestamp": row["timestamp"],
|
|
"title": title or f"{row['agent_name'] or '未知角色'}提及 {symbol}",
|
|
"meta": payload.get("conferenceTitle")
|
|
or payload.get("feedType")
|
|
or row["event_type"],
|
|
"body": content,
|
|
"tone": "neutral",
|
|
"agent": row["agent_name"] or payload.get("agentName") or payload.get("agent"),
|
|
},
|
|
)
|
|
if len(normalized_events) >= limit_events:
|
|
break
|
|
|
|
normalized_trades = [
|
|
{
|
|
"id": row["id"],
|
|
"type": "trade",
|
|
"timestamp": row["timestamp"],
|
|
"title": f"{row['side']} {int(row['qty'] or 0)} 股",
|
|
"meta": "交易执行",
|
|
"body": f"成交价 ${float(row['price'] or 0):.2f}",
|
|
"tone": "positive" if row["side"] == "LONG" else "negative" if row["side"] == "SHORT" else "neutral",
|
|
}
|
|
for row in trade_rows
|
|
]
|
|
|
|
normalized_signals = [
|
|
{
|
|
"id": row["id"],
|
|
"type": "signal",
|
|
"timestamp": f"{row['trade_date']}T08:00:00" if row["trade_date"] else row["created_at"],
|
|
"title": f"{row['agent_name']} 给出{row['signal'] or '中性'}信号",
|
|
"meta": row["role"],
|
|
"body": (
|
|
f"后验收益 {float(row['real_return']) * 100:+.2f}%"
|
|
if row["real_return"] is not None
|
|
else "该信号暂未完成后验评估"
|
|
),
|
|
"tone": "positive" if str(row["signal"] or "").lower() in {"bullish", "buy", "long"} else "negative" if str(row["signal"] or "").lower() in {"bearish", "sell", "short"} else "neutral",
|
|
}
|
|
for row in signal_rows
|
|
]
|
|
|
|
return {
|
|
"events": normalized_events,
|
|
"trades": normalized_trades,
|
|
"signals": normalized_signals,
|
|
}
|