f9f4560459
- FastAPI backend with SQLModel, Alembic migrations, AgentScope agents - Next.js 15 frontend with React 19, Tailwind, Zustand, React Flow - Multi-provider AI system (DashScope, Kling, MiniMax, Volcengine, OpenAI, etc.) - All HTTP clients migrated from sync requests to async httpx - Admin-managed API keys via environment variables - SSRF vulnerability fixed in ensure_url()
380 lines
14 KiB
Python
380 lines
14 KiB
Python
"""
|
|
Property-Based Tests for Health Monitoring
|
|
|
|
Tests correctness properties for health check and monitoring functionality.
|
|
Uses Hypothesis for property-based testing.
|
|
"""
|
|
import pytest
|
|
from hypothesis import given, strategies as st, settings, HealthCheck
|
|
from unittest.mock import Mock, patch, AsyncMock
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session, text
|
|
from src.main import app
|
|
from src.config.database import engine
|
|
|
|
|
|
# Test client
|
|
client = TestClient(app)
|
|
|
|
|
|
def unwrap_response_data(response):
|
|
payload = response.json()
|
|
return payload.get("data", payload)
|
|
|
|
|
|
# Strategies for generating test data
|
|
dependency_names = st.sampled_from(['database', 'redis', 'task_manager', 'model_registry', 'ai_services'])
|
|
health_statuses = st.sampled_from(['healthy', 'unhealthy', 'degraded', 'disabled'])
|
|
error_messages = st.text(min_size=1, max_size=100)
|
|
|
|
|
|
@given(
|
|
db_healthy=st.booleans(),
|
|
redis_healthy=st.booleans(),
|
|
task_manager_healthy=st.booleans()
|
|
)
|
|
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=20)
|
|
def test_property_24_health_check_dependency_validation(
|
|
db_healthy: bool,
|
|
redis_healthy: bool,
|
|
task_manager_healthy: bool
|
|
):
|
|
"""
|
|
Property 24: 健康检查依赖验证
|
|
|
|
对于任何不健康的依赖(数据库、缓存、外部服务),
|
|
健康检查端点应该返回相应的错误状态码和详细信息。
|
|
|
|
**Validates: Requirements 18.3**
|
|
"""
|
|
# Mock dependencies based on health status
|
|
with patch('src.api.health.Session') as mock_session_class, \
|
|
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
|
patch('src.api.health.task_manager') as mock_task_manager:
|
|
|
|
# Setup database mock
|
|
mock_session = Mock()
|
|
mock_session_class.return_value.__enter__.return_value = mock_session
|
|
|
|
if db_healthy:
|
|
mock_session.exec.return_value = None # Successful query
|
|
else:
|
|
mock_session.exec.side_effect = Exception("Database connection failed")
|
|
|
|
# Setup Redis mock
|
|
mock_cache = Mock()
|
|
mock_cache._connected = redis_healthy
|
|
if redis_healthy:
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(return_value=True)
|
|
mock_cache._redis.info = AsyncMock(return_value={
|
|
'redis_version': '7.0.0',
|
|
'connected_clients': 1,
|
|
'used_memory_human': '1M'
|
|
})
|
|
else:
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(side_effect=Exception("Redis connection failed"))
|
|
|
|
mock_cache_service.return_value = mock_cache
|
|
|
|
# Setup task manager mock
|
|
if task_manager_healthy:
|
|
mock_task_manager.get_stats.return_value = {
|
|
'total_tasks': 0,
|
|
'completed_tasks': 0,
|
|
'failed_tasks': 0,
|
|
'queue_size': 0
|
|
}
|
|
else:
|
|
mock_task_manager.get_stats.side_effect = Exception("Task manager error")
|
|
|
|
# Call the detailed health check endpoint
|
|
response = client.get("/health/detailed")
|
|
|
|
# Verify response structure
|
|
assert response.status_code == 200
|
|
data = unwrap_response_data(response)
|
|
|
|
# Verify response has required fields
|
|
assert "status" in data
|
|
assert "components" in data
|
|
assert "timestamp" in data
|
|
|
|
components = data["components"]
|
|
|
|
# Property: Database health should be reflected correctly
|
|
if "database" in components:
|
|
if db_healthy:
|
|
assert components["database"]["status"] == "healthy"
|
|
assert "latency_ms" in components["database"]
|
|
else:
|
|
assert components["database"]["status"] == "unhealthy"
|
|
assert "message" in components["database"]
|
|
assert "failed" in components["database"]["message"].lower()
|
|
|
|
# Property: Redis health should be reflected correctly
|
|
if "redis" in components:
|
|
if redis_healthy:
|
|
assert components["redis"]["status"] in ["healthy", "disabled"]
|
|
else:
|
|
assert components["redis"]["status"] in ["unhealthy", "disabled"]
|
|
|
|
# Property: Task manager health should be reflected correctly
|
|
if "task_manager" in components:
|
|
if task_manager_healthy:
|
|
assert components["task_manager"]["status"] == "healthy"
|
|
assert "stats" in components["task_manager"]
|
|
else:
|
|
assert components["task_manager"]["status"] == "unhealthy"
|
|
assert "message" in components["task_manager"]
|
|
|
|
# Property: Overall status should be unhealthy if any critical component is unhealthy
|
|
if not db_healthy:
|
|
assert data["status"] in ["unhealthy", "degraded"]
|
|
|
|
if not task_manager_healthy:
|
|
assert data["status"] in ["unhealthy", "degraded"]
|
|
|
|
|
|
@given(
|
|
db_ready=st.booleans(),
|
|
redis_enabled=st.booleans(),
|
|
redis_ready=st.booleans(),
|
|
task_manager_ready=st.booleans()
|
|
)
|
|
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=20)
|
|
def test_property_24_readiness_probe_dependency_validation(
|
|
db_ready: bool,
|
|
redis_enabled: bool,
|
|
redis_ready: bool,
|
|
task_manager_ready: bool
|
|
):
|
|
"""
|
|
Property 24: 就绪探针依赖验证
|
|
|
|
对于任何不就绪的依赖,就绪探针应该返回503状态码。
|
|
|
|
**Validates: Requirements 18.3**
|
|
"""
|
|
# Mock dependencies based on readiness status
|
|
with patch('src.api.health.Session') as mock_session_class, \
|
|
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
|
patch('src.api.health.task_manager') as mock_task_manager, \
|
|
patch('src.config.settings.REDIS_ENABLED', redis_enabled):
|
|
|
|
# Setup database mock
|
|
mock_session = Mock()
|
|
mock_session_class.return_value.__enter__.return_value = mock_session
|
|
|
|
if db_ready:
|
|
mock_session.exec.return_value = None
|
|
else:
|
|
mock_session.exec.side_effect = Exception("Database not ready")
|
|
|
|
# Setup Redis mock
|
|
mock_cache = Mock()
|
|
mock_cache._connected = redis_ready
|
|
if redis_ready:
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(return_value=True)
|
|
else:
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(side_effect=Exception("Redis not ready"))
|
|
|
|
mock_cache_service.return_value = mock_cache
|
|
|
|
# Setup task manager mock
|
|
if task_manager_ready:
|
|
mock_task_manager.get_stats.return_value = {
|
|
'total_tasks': 0,
|
|
'completed_tasks': 0,
|
|
'failed_tasks': 0,
|
|
'queue_size': 0
|
|
}
|
|
else:
|
|
mock_task_manager.get_stats.side_effect = Exception("Task manager not ready")
|
|
|
|
# Call the readiness probe endpoint
|
|
response = client.get("/health/ready")
|
|
|
|
# Property: Should return 200 if all critical dependencies are ready, 503 otherwise
|
|
# Critical dependencies: database, task_manager, and redis (only if enabled)
|
|
all_ready = db_ready and task_manager_ready and (not redis_enabled or redis_ready)
|
|
|
|
if all_ready:
|
|
assert response.status_code == 200
|
|
data = unwrap_response_data(response)
|
|
assert data["status"] == "ready"
|
|
assert "components" in data
|
|
else:
|
|
assert response.status_code == 503
|
|
data = response.json()
|
|
|
|
details = data.get("details") or data.get("detail") or {}
|
|
if isinstance(details, dict):
|
|
assert details["status"] == "not ready"
|
|
assert "components" in details
|
|
|
|
|
|
@given(
|
|
component_name=dependency_names,
|
|
is_healthy=st.booleans(),
|
|
error_msg=error_messages
|
|
)
|
|
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=15)
|
|
def test_property_24_component_error_details(
|
|
component_name: str,
|
|
is_healthy: bool,
|
|
error_msg: str
|
|
):
|
|
"""
|
|
Property 24: 组件错误详情
|
|
|
|
对于任何不健康的组件,健康检查应该提供详细的错误信息。
|
|
|
|
**Validates: Requirements 18.3**
|
|
"""
|
|
# Mock the specific component to be unhealthy
|
|
with patch('src.api.health.Session') as mock_session_class, \
|
|
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
|
patch('src.api.health.task_manager') as mock_task_manager, \
|
|
patch('src.api.health.ModelRegistry') as mock_registry, \
|
|
patch('src.api.health.health_monitor') as mock_health_monitor:
|
|
|
|
# Setup all mocks as healthy by default
|
|
mock_session = Mock()
|
|
mock_session_class.return_value.__enter__.return_value = mock_session
|
|
mock_session.exec.return_value = None
|
|
|
|
mock_cache = Mock()
|
|
mock_cache._connected = True
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(return_value=True)
|
|
mock_cache._redis.info = AsyncMock(return_value={
|
|
'redis_version': '7.0.0',
|
|
'connected_clients': 1,
|
|
'used_memory_human': '1M'
|
|
})
|
|
mock_cache_service.return_value = mock_cache
|
|
|
|
mock_task_manager.get_stats.return_value = {
|
|
'total_tasks': 0,
|
|
'completed_tasks': 0,
|
|
'failed_tasks': 0,
|
|
'queue_size': 0
|
|
}
|
|
|
|
mock_registry.list_models.return_value = {}
|
|
|
|
mock_health_monitor.get_health_summary.return_value = {
|
|
'total': 0,
|
|
'healthy': 0,
|
|
'unhealthy': 0,
|
|
'degraded': 0
|
|
}
|
|
|
|
# Make the specific component unhealthy
|
|
if not is_healthy:
|
|
if component_name == 'database':
|
|
mock_session.exec.side_effect = Exception(error_msg)
|
|
elif component_name == 'redis':
|
|
mock_cache._redis.ping.side_effect = Exception(error_msg)
|
|
elif component_name == 'task_manager':
|
|
mock_task_manager.get_stats.side_effect = Exception(error_msg)
|
|
elif component_name == 'model_registry':
|
|
mock_registry.list_models.side_effect = Exception(error_msg)
|
|
elif component_name == 'ai_services':
|
|
mock_health_monitor.get_health_summary.side_effect = Exception(error_msg)
|
|
|
|
# Call the detailed health check endpoint
|
|
response = client.get("/health/detailed")
|
|
|
|
assert response.status_code == 200
|
|
data = unwrap_response_data(response)
|
|
|
|
# Property: Unhealthy components should have error details
|
|
if not is_healthy and component_name in data["components"]:
|
|
component = data["components"][component_name]
|
|
|
|
# Should have status field
|
|
assert "status" in component
|
|
|
|
# Should have message field with error details
|
|
if component["status"] in ["unhealthy", "degraded"]:
|
|
assert "message" in component
|
|
# Error message should contain some information
|
|
assert len(component["message"]) > 0
|
|
|
|
|
|
@given(
|
|
num_unhealthy=st.integers(min_value=0, max_value=3)
|
|
)
|
|
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=10)
|
|
def test_property_24_overall_status_aggregation(num_unhealthy: int):
|
|
"""
|
|
Property 24: 整体状态聚合
|
|
|
|
整体健康状态应该正确反映所有组件的健康状态。
|
|
如果有任何组件不健康,整体状态应该是unhealthy或degraded。
|
|
|
|
**Validates: Requirements 18.3**
|
|
"""
|
|
with patch('src.api.health.Session') as mock_session_class, \
|
|
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
|
patch('src.api.health.task_manager') as mock_task_manager:
|
|
|
|
# Setup mocks
|
|
mock_session = Mock()
|
|
mock_session_class.return_value.__enter__.return_value = mock_session
|
|
|
|
mock_cache = Mock()
|
|
mock_cache._connected = True
|
|
mock_cache._redis = AsyncMock()
|
|
mock_cache._redis.ping = AsyncMock(return_value=True)
|
|
mock_cache._redis.info = AsyncMock(return_value={
|
|
'redis_version': '7.0.0',
|
|
'connected_clients': 1,
|
|
'used_memory_human': '1M'
|
|
})
|
|
mock_cache_service.return_value = mock_cache
|
|
|
|
# Make num_unhealthy components fail
|
|
components_to_fail = ['database', 'task_manager', 'redis'][:num_unhealthy]
|
|
|
|
if 'database' in components_to_fail:
|
|
mock_session.exec.side_effect = Exception("Database failed")
|
|
else:
|
|
mock_session.exec.return_value = None
|
|
|
|
if 'task_manager' in components_to_fail:
|
|
mock_task_manager.get_stats.side_effect = Exception("Task manager failed")
|
|
else:
|
|
mock_task_manager.get_stats.return_value = {
|
|
'total_tasks': 0,
|
|
'completed_tasks': 0,
|
|
'failed_tasks': 0,
|
|
'queue_size': 0
|
|
}
|
|
|
|
if 'redis' in components_to_fail:
|
|
mock_cache._redis.ping.side_effect = Exception("Redis failed")
|
|
|
|
# Call the detailed health check endpoint
|
|
response = client.get("/health/detailed")
|
|
|
|
assert response.status_code == 200
|
|
data = unwrap_response_data(response)
|
|
|
|
# Property: Overall status should reflect component health
|
|
if num_unhealthy == 0:
|
|
# All healthy - overall should be healthy
|
|
assert data["status"] == "healthy"
|
|
else:
|
|
# Some unhealthy - overall should be unhealthy or degraded
|
|
assert data["status"] in ["unhealthy", "degraded"]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v", "--tb=short"])
|