cillin/evotraders
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: optimize system concurrency, I/O stability and fix WebSocket disconnects

Browse Source
This commit is contained in:
cillin
2026-04-07 13:58:49 +08:00
parent 62c7341cf6
commit 11849208ed
21 changed files with 357 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/services/gateway.py
View File

@@ -148,8 +148,9 @@ class Gateway:
self.handle_client,
host,
port,
ping_interval=30,
ping_timeout=60,
ping_interval=20,
ping_timeout=120,
max_size=10 * 1024 * 1024, # 10MB
)
logger.info(f"WebSocket server ready: ws://{host}:{port}")
@@ -833,12 +834,18 @@ class Gateway:
if not self.connected_clients:
return
message_json = json.dumps(message, ensure_ascii=False, default=str)
# Offload potentially heavy JSON serialization to thread
message_json = await asyncio.to_thread(
json.dumps, message, ensure_ascii=False, default=str
)
async with self.lock:
# Filter only active clients to minimize unnecessary send attempts
# In websockets v13+, we must check state.name == 'OPEN'
active_clients = [c for c in self.connected_clients if c.state.name == 'OPEN']
tasks = [
self._send_to_client(client, message_json)
for client in self.connected_clients.copy()
for client in active_clients
]
if tasks:
@@ -849,9 +856,14 @@ class Gateway:
client: ServerConnection,
message: str,
):
if client.state.name != 'OPEN':
async with self.lock:
self.connected_clients.discard(client)
return
try:
await client.send(message)
except websockets.ConnectionClosed:
except (websockets.ConnectionClosed, Exception):
async with self.lock:
self.connected_clients.discard(client)

18
backend/services/gateway_cycle_support.py
View File

@@ -253,7 +253,8 @@ async def finalize_cycle(gateway: Any, date: str) -> None:
async def get_market_caps(gateway: Any, tickers: list[str], date: str) -> dict[str, float]:
market_caps: dict[str, float] = {}
for ticker in tickers:
async def _get_one(ticker: str):
try:
market_cap = None
response = await gateway._call_trading_service(
@@ -263,12 +264,21 @@ async def get_market_caps(gateway: Any, tickers: list[str], date: str) -> dict[s
if response is not None:
market_cap = response.get("market_cap")
if market_cap is None:
payload = trading_domain.get_market_cap_payload(ticker=ticker, end_date=date)
payload = await asyncio.to_thread(
trading_domain.get_market_cap_payload,
ticker=ticker,
end_date=date,
)
market_cap = payload.get("market_cap")
market_caps[ticker] = market_cap if market_cap else 1e9
return ticker, (market_cap if market_cap else 1e9)
except Exception as exc:
logger.warning("Failed to get market cap for %s, using default 1e9: %s", ticker, exc)
market_caps[ticker] = 1e9
return ticker, 1e9
tasks = [_get_one(ticker) for ticker in tickers]
results = await asyncio.gather(*tasks)
for ticker, mc in results:
market_caps[ticker] = mc
return market_caps

212
backend/services/gateway_stock_handlers.py
View File

@@ -517,111 +517,129 @@ async def handle_get_stock_similar_days(gateway: Any, websocket: Any, data: dict
async def handle_get_stock_technical_indicators(gateway: Any, websocket: Any, data: dict[str, Any]) -> None:
ticker = normalize_symbol(data.get("ticker", ""))
if not ticker:
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": "ticker is required",
}, ensure_ascii=False))
return
ticker = normalize_symbol(data.get("ticker", ""))
if not ticker:
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": "ticker is required",
}, ensure_ascii=False))
return
try:
end_date = datetime.now()
start_date = end_date - timedelta(days=250)
try:
end_date = datetime.now()
# Reduced from 250 to 150 days to lower CPU/memory pressure while still supporting MA200 (approx 140 trading days)
start_date = end_date - timedelta(days=150)
prices = None
response = await gateway._call_trading_service(
"get_prices",
lambda client: client.get_prices(
ticker=ticker,
start_date=start_date.strftime("%Y-%m-%d"),
end_date=end_date.strftime("%Y-%m-%d"),
),
)
if response is not None:
prices = response.prices
prices = None
response = await gateway._call_trading_service(
"get_prices",
lambda client: client.get_prices(
ticker=ticker,
start_date=start_date.strftime("%Y-%m-%d"),
end_date=end_date.strftime("%Y-%m-%d"),
),
)
if response is not None:
prices = response.prices
if prices is None:
payload = trading_domain.get_prices_payload(
ticker=ticker,
start_date=start_date.strftime("%Y-%m-%d"),
end_date=end_date.strftime("%Y-%m-%d"),
)
prices = payload.get("prices") or []
if prices is None:
# Offload domain logic to thread
payload = await asyncio.to_thread(
trading_domain.get_prices_payload,
ticker=ticker,
start_date=start_date.strftime("%Y-%m-%d"),
end_date=end_date.strftime("%Y-%m-%d"),
)
prices = payload.get("prices") or []
if not prices or len(prices) < 20:
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": "Insufficient price data",
}, ensure_ascii=False))
return
if not prices or len(prices) < 20:
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": "Insufficient price data",
}, ensure_ascii=False))
return
df = prices_to_df(prices)
signal = gateway._technical_analyzer.analyze(ticker, df)
def _calc():
df = prices_to_df(prices)
signal = gateway._technical_analyzer.analyze(ticker, df)
df_sorted = df.sort_values("time").reset_index(drop=True)
df_sorted["returns"] = df_sorted["close"].pct_change()
v10 = float(df_sorted["returns"].tail(10).std() * (252**0.5) * 100) if len(df_sorted) >= 10 else None
v20 = float(df_sorted["returns"].tail(20).std() * (252**0.5) * 100) if len(df_sorted) >= 20 else None
v60 = float(df_sorted["returns"].tail(60).std() * (252**0.5) * 100) if len(df_sorted) >= 60 else None
df_sorted = df.sort_values("time").reset_index(drop=True)
df_sorted["returns"] = df_sorted["close"].pct_change()
vol_10 = float(df_sorted["returns"].tail(10).std() * (252**0.5) * 100) if len(df_sorted) >= 10 else None
vol_20 = float(df_sorted["returns"].tail(20).std() * (252**0.5) * 100) if len(df_sorted) >= 20 else None
vol_60 = float(df_sorted["returns"].tail(60).std() * (252**0.5) * 100) if len(df_sorted) >= 60 else None
ma_distance = {}
for ma_key in ["ma5", "ma10", "ma20", "ma50", "ma200"]:
ma_value = getattr(signal, ma_key, None)
ma_distance[ma_key] = ((signal.current_price - ma_value) / ma_value) * 100 if ma_value and ma_value > 0 else None
ma_dist = {}
for ma_key in ["ma5", "ma10", "ma20", "ma50", "ma200"]:
ma_val = getattr(signal, ma_key, None)
ma_dist[ma_key] = ((signal.current_price - ma_val) / ma_val) * 100 if ma_val and ma_val > 0 else None
indicators = {
"ticker": ticker,
"current_price": signal.current_price,
"ma": {
"ma5": signal.ma5,
"ma10": signal.ma10,
"ma20": signal.ma20,
"ma50": signal.ma50,
"ma200": signal.ma200,
"distance": ma_distance,
},
"rsi": {
"rsi14": signal.rsi14,
"status": "oversold" if signal.rsi14 < 30 else "overbought" if signal.rsi14 > 70 else "neutral",
},
"macd": {
"macd": signal.macd,
"signal": signal.macd_signal,
"histogram": signal.macd - signal.macd_signal,
},
"bollinger": {
"upper": signal.bollinger_upper,
"mid": signal.bollinger_mid,
"lower": signal.bollinger_lower,
},
"volatility": {
"vol_10d": vol_10,
"vol_20d": vol_20,
"vol_60d": vol_60,
"annualized": signal.annualized_volatility_pct,
"risk_level": signal.risk_level,
},
"trend": signal.trend,
"mean_reversion": signal.mean_reversion_signal,
}
return {
"ticker": ticker,
"current_price": signal.current_price,
"ma": {
"ma5": signal.ma5,
"ma10": signal.ma10,
"ma20": signal.ma20,
"ma50": signal.ma50,
"ma200": signal.ma200,
"distance": ma_dist,
},
"rsi": {
"rsi14": signal.rsi14,
"status": "oversold" if signal.rsi14 < 30 else "overbought" if signal.rsi14 > 70 else "neutral",
},
"macd": {
"macd": signal.macd,
"signal": signal.macd_signal,
"histogram": signal.macd - signal.macd_signal,
},
"bollinger": {
"upper": signal.bollinger_upper,
"mid": signal.bollinger_mid,
"lower": signal.bollinger_lower,
},
"volatility": {
"vol_10d": v10,
"vol_20d": v20,
"vol_60d": v60,
"annualized": signal.annualized_volatility_pct,
"risk_level": signal.risk_level,
},
"trend": signal.trend,
"mean_reversion": signal.mean_reversion_signal,
}
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": indicators,
}, ensure_ascii=False, default=str))
except Exception as exc:
logger.exception("Error getting technical indicators for %s", ticker)
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": str(exc),
}, ensure_ascii=False))
# Use a semaphore to prevent too many concurrent CPU-intensive calculations
# which can block the event loop heartbeats.
if not hasattr(gateway, "_calc_sem"):
gateway._calc_sem = asyncio.Semaphore(3)
async with gateway._calc_sem:
indicators = await asyncio.to_thread(_calc)
# Also offload JSON serialization to thread to avoid blocking main loop
msg = await asyncio.to_thread(json.dumps, {
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": indicators,
}, ensure_ascii=False, default=str)
if websocket.state.name == 'OPEN':
await websocket.send(msg)
else:
logger.warning("Websocket closed for %s, skipping indicator send", ticker)
except Exception as exc:
logger.exception("Error getting technical indicators for %s", ticker)
await websocket.send(json.dumps({
"type": "stock_technical_indicators_loaded",
"ticker": ticker,
"indicators": None,
"error": str(exc),
}, ensure_ascii=False))
async def handle_run_stock_enrich(gateway: Any, websocket: Any, data: dict[str, Any]) -> None:

31
backend/services/storage.py
View File

@@ -7,6 +7,7 @@ Handles reading/writing dashboard JSON files and portfolio state
import json
import logging
import os
import time
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
@@ -950,11 +951,14 @@ class StorageService:
def save_server_state(self, state: Dict[str, Any]):
"""
Save server state to file
Args:
state: Server state dictionary
Save server state to file with rate-limiting to avoid I/O storms.
"""
now = time.time()
# Ensure at least 2 seconds between physical disk writes
if hasattr(self, "_last_save_time") and (now - self._last_save_time) < 2.0:
return
self._last_save_time = now
state_to_save = {
**state,
"last_saved": datetime.now().isoformat(),
@@ -970,14 +974,17 @@ class StorageService:
if "trades" in state_to_save:
state_to_save["trades"] = state_to_save["trades"][:100]
with open(self.server_state_file, "w", encoding="utf-8") as f:
json.dump(
state_to_save,
f,
ensure_ascii=False,
indent=2,
default=str,
)
try:
with open(self.server_state_file, "w", encoding="utf-8") as f:
# Removed indent=2 to minimize file size and serialization overhead
json.dump(
state_to_save,
f,
ensure_ascii=False,
default=str,
)
except Exception as e:
logger.error(f"Failed to save server state: {e}")
logger.debug(f"Server state saved to: {self.server_state_file}")