Initial commit: Pixel AI comic/video creation platform
- FastAPI backend with SQLModel, Alembic migrations, AgentScope agents - Next.js 15 frontend with React 19, Tailwind, Zustand, React Flow - Multi-provider AI system (DashScope, Kling, MiniMax, Volcengine, OpenAI, etc.) - All HTTP clients migrated from sync requests to async httpx - Admin-managed API keys via environment variables - SSRF vulnerability fixed in ensure_url()
This commit is contained in:
张鹏
379
backend/tests/test_health_monitoring_properties.py
Normal file
379
backend/tests/test_health_monitoring_properties.py
Normal file
@@ -0,0 +1,379 @@
|
||||
"""
|
||||
Property-Based Tests for Health Monitoring
|
||||
|
||||
Tests correctness properties for health check and monitoring functionality.
|
||||
Uses Hypothesis for property-based testing.
|
||||
"""
|
||||
import pytest
|
||||
from hypothesis import given, strategies as st, settings, HealthCheck
|
||||
from unittest.mock import Mock, patch, AsyncMock
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, text
|
||||
from src.main import app
|
||||
from src.config.database import engine
|
||||
|
||||
|
||||
# Test client
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def unwrap_response_data(response):
|
||||
payload = response.json()
|
||||
return payload.get("data", payload)
|
||||
|
||||
|
||||
# Strategies for generating test data
|
||||
dependency_names = st.sampled_from(['database', 'redis', 'task_manager', 'model_registry', 'ai_services'])
|
||||
health_statuses = st.sampled_from(['healthy', 'unhealthy', 'degraded', 'disabled'])
|
||||
error_messages = st.text(min_size=1, max_size=100)
|
||||
|
||||
|
||||
@given(
|
||||
db_healthy=st.booleans(),
|
||||
redis_healthy=st.booleans(),
|
||||
task_manager_healthy=st.booleans()
|
||||
)
|
||||
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=20)
|
||||
def test_property_24_health_check_dependency_validation(
|
||||
db_healthy: bool,
|
||||
redis_healthy: bool,
|
||||
task_manager_healthy: bool
|
||||
):
|
||||
"""
|
||||
Property 24: 健康检查依赖验证
|
||||
|
||||
对于任何不健康的依赖(数据库、缓存、外部服务),
|
||||
健康检查端点应该返回相应的错误状态码和详细信息。
|
||||
|
||||
**Validates: Requirements 18.3**
|
||||
"""
|
||||
# Mock dependencies based on health status
|
||||
with patch('src.api.health.Session') as mock_session_class, \
|
||||
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
||||
patch('src.api.health.task_manager') as mock_task_manager:
|
||||
|
||||
# Setup database mock
|
||||
mock_session = Mock()
|
||||
mock_session_class.return_value.__enter__.return_value = mock_session
|
||||
|
||||
if db_healthy:
|
||||
mock_session.exec.return_value = None # Successful query
|
||||
else:
|
||||
mock_session.exec.side_effect = Exception("Database connection failed")
|
||||
|
||||
# Setup Redis mock
|
||||
mock_cache = Mock()
|
||||
mock_cache._connected = redis_healthy
|
||||
if redis_healthy:
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(return_value=True)
|
||||
mock_cache._redis.info = AsyncMock(return_value={
|
||||
'redis_version': '7.0.0',
|
||||
'connected_clients': 1,
|
||||
'used_memory_human': '1M'
|
||||
})
|
||||
else:
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(side_effect=Exception("Redis connection failed"))
|
||||
|
||||
mock_cache_service.return_value = mock_cache
|
||||
|
||||
# Setup task manager mock
|
||||
if task_manager_healthy:
|
||||
mock_task_manager.get_stats.return_value = {
|
||||
'total_tasks': 0,
|
||||
'completed_tasks': 0,
|
||||
'failed_tasks': 0,
|
||||
'queue_size': 0
|
||||
}
|
||||
else:
|
||||
mock_task_manager.get_stats.side_effect = Exception("Task manager error")
|
||||
|
||||
# Call the detailed health check endpoint
|
||||
response = client.get("/health/detailed")
|
||||
|
||||
# Verify response structure
|
||||
assert response.status_code == 200
|
||||
data = unwrap_response_data(response)
|
||||
|
||||
# Verify response has required fields
|
||||
assert "status" in data
|
||||
assert "components" in data
|
||||
assert "timestamp" in data
|
||||
|
||||
components = data["components"]
|
||||
|
||||
# Property: Database health should be reflected correctly
|
||||
if "database" in components:
|
||||
if db_healthy:
|
||||
assert components["database"]["status"] == "healthy"
|
||||
assert "latency_ms" in components["database"]
|
||||
else:
|
||||
assert components["database"]["status"] == "unhealthy"
|
||||
assert "message" in components["database"]
|
||||
assert "failed" in components["database"]["message"].lower()
|
||||
|
||||
# Property: Redis health should be reflected correctly
|
||||
if "redis" in components:
|
||||
if redis_healthy:
|
||||
assert components["redis"]["status"] in ["healthy", "disabled"]
|
||||
else:
|
||||
assert components["redis"]["status"] in ["unhealthy", "disabled"]
|
||||
|
||||
# Property: Task manager health should be reflected correctly
|
||||
if "task_manager" in components:
|
||||
if task_manager_healthy:
|
||||
assert components["task_manager"]["status"] == "healthy"
|
||||
assert "stats" in components["task_manager"]
|
||||
else:
|
||||
assert components["task_manager"]["status"] == "unhealthy"
|
||||
assert "message" in components["task_manager"]
|
||||
|
||||
# Property: Overall status should be unhealthy if any critical component is unhealthy
|
||||
if not db_healthy:
|
||||
assert data["status"] in ["unhealthy", "degraded"]
|
||||
|
||||
if not task_manager_healthy:
|
||||
assert data["status"] in ["unhealthy", "degraded"]
|
||||
|
||||
|
||||
@given(
|
||||
db_ready=st.booleans(),
|
||||
redis_enabled=st.booleans(),
|
||||
redis_ready=st.booleans(),
|
||||
task_manager_ready=st.booleans()
|
||||
)
|
||||
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=20)
|
||||
def test_property_24_readiness_probe_dependency_validation(
|
||||
db_ready: bool,
|
||||
redis_enabled: bool,
|
||||
redis_ready: bool,
|
||||
task_manager_ready: bool
|
||||
):
|
||||
"""
|
||||
Property 24: 就绪探针依赖验证
|
||||
|
||||
对于任何不就绪的依赖,就绪探针应该返回503状态码。
|
||||
|
||||
**Validates: Requirements 18.3**
|
||||
"""
|
||||
# Mock dependencies based on readiness status
|
||||
with patch('src.api.health.Session') as mock_session_class, \
|
||||
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
||||
patch('src.api.health.task_manager') as mock_task_manager, \
|
||||
patch('src.config.settings.REDIS_ENABLED', redis_enabled):
|
||||
|
||||
# Setup database mock
|
||||
mock_session = Mock()
|
||||
mock_session_class.return_value.__enter__.return_value = mock_session
|
||||
|
||||
if db_ready:
|
||||
mock_session.exec.return_value = None
|
||||
else:
|
||||
mock_session.exec.side_effect = Exception("Database not ready")
|
||||
|
||||
# Setup Redis mock
|
||||
mock_cache = Mock()
|
||||
mock_cache._connected = redis_ready
|
||||
if redis_ready:
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(return_value=True)
|
||||
else:
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(side_effect=Exception("Redis not ready"))
|
||||
|
||||
mock_cache_service.return_value = mock_cache
|
||||
|
||||
# Setup task manager mock
|
||||
if task_manager_ready:
|
||||
mock_task_manager.get_stats.return_value = {
|
||||
'total_tasks': 0,
|
||||
'completed_tasks': 0,
|
||||
'failed_tasks': 0,
|
||||
'queue_size': 0
|
||||
}
|
||||
else:
|
||||
mock_task_manager.get_stats.side_effect = Exception("Task manager not ready")
|
||||
|
||||
# Call the readiness probe endpoint
|
||||
response = client.get("/health/ready")
|
||||
|
||||
# Property: Should return 200 if all critical dependencies are ready, 503 otherwise
|
||||
# Critical dependencies: database, task_manager, and redis (only if enabled)
|
||||
all_ready = db_ready and task_manager_ready and (not redis_enabled or redis_ready)
|
||||
|
||||
if all_ready:
|
||||
assert response.status_code == 200
|
||||
data = unwrap_response_data(response)
|
||||
assert data["status"] == "ready"
|
||||
assert "components" in data
|
||||
else:
|
||||
assert response.status_code == 503
|
||||
data = response.json()
|
||||
|
||||
details = data.get("details") or data.get("detail") or {}
|
||||
if isinstance(details, dict):
|
||||
assert details["status"] == "not ready"
|
||||
assert "components" in details
|
||||
|
||||
|
||||
@given(
|
||||
component_name=dependency_names,
|
||||
is_healthy=st.booleans(),
|
||||
error_msg=error_messages
|
||||
)
|
||||
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=15)
|
||||
def test_property_24_component_error_details(
|
||||
component_name: str,
|
||||
is_healthy: bool,
|
||||
error_msg: str
|
||||
):
|
||||
"""
|
||||
Property 24: 组件错误详情
|
||||
|
||||
对于任何不健康的组件,健康检查应该提供详细的错误信息。
|
||||
|
||||
**Validates: Requirements 18.3**
|
||||
"""
|
||||
# Mock the specific component to be unhealthy
|
||||
with patch('src.api.health.Session') as mock_session_class, \
|
||||
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
||||
patch('src.api.health.task_manager') as mock_task_manager, \
|
||||
patch('src.api.health.ModelRegistry') as mock_registry, \
|
||||
patch('src.api.health.health_monitor') as mock_health_monitor:
|
||||
|
||||
# Setup all mocks as healthy by default
|
||||
mock_session = Mock()
|
||||
mock_session_class.return_value.__enter__.return_value = mock_session
|
||||
mock_session.exec.return_value = None
|
||||
|
||||
mock_cache = Mock()
|
||||
mock_cache._connected = True
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(return_value=True)
|
||||
mock_cache._redis.info = AsyncMock(return_value={
|
||||
'redis_version': '7.0.0',
|
||||
'connected_clients': 1,
|
||||
'used_memory_human': '1M'
|
||||
})
|
||||
mock_cache_service.return_value = mock_cache
|
||||
|
||||
mock_task_manager.get_stats.return_value = {
|
||||
'total_tasks': 0,
|
||||
'completed_tasks': 0,
|
||||
'failed_tasks': 0,
|
||||
'queue_size': 0
|
||||
}
|
||||
|
||||
mock_registry.list_models.return_value = {}
|
||||
|
||||
mock_health_monitor.get_health_summary.return_value = {
|
||||
'total': 0,
|
||||
'healthy': 0,
|
||||
'unhealthy': 0,
|
||||
'degraded': 0
|
||||
}
|
||||
|
||||
# Make the specific component unhealthy
|
||||
if not is_healthy:
|
||||
if component_name == 'database':
|
||||
mock_session.exec.side_effect = Exception(error_msg)
|
||||
elif component_name == 'redis':
|
||||
mock_cache._redis.ping.side_effect = Exception(error_msg)
|
||||
elif component_name == 'task_manager':
|
||||
mock_task_manager.get_stats.side_effect = Exception(error_msg)
|
||||
elif component_name == 'model_registry':
|
||||
mock_registry.list_models.side_effect = Exception(error_msg)
|
||||
elif component_name == 'ai_services':
|
||||
mock_health_monitor.get_health_summary.side_effect = Exception(error_msg)
|
||||
|
||||
# Call the detailed health check endpoint
|
||||
response = client.get("/health/detailed")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = unwrap_response_data(response)
|
||||
|
||||
# Property: Unhealthy components should have error details
|
||||
if not is_healthy and component_name in data["components"]:
|
||||
component = data["components"][component_name]
|
||||
|
||||
# Should have status field
|
||||
assert "status" in component
|
||||
|
||||
# Should have message field with error details
|
||||
if component["status"] in ["unhealthy", "degraded"]:
|
||||
assert "message" in component
|
||||
# Error message should contain some information
|
||||
assert len(component["message"]) > 0
|
||||
|
||||
|
||||
@given(
|
||||
num_unhealthy=st.integers(min_value=0, max_value=3)
|
||||
)
|
||||
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture], max_examples=10)
|
||||
def test_property_24_overall_status_aggregation(num_unhealthy: int):
|
||||
"""
|
||||
Property 24: 整体状态聚合
|
||||
|
||||
整体健康状态应该正确反映所有组件的健康状态。
|
||||
如果有任何组件不健康,整体状态应该是unhealthy或degraded。
|
||||
|
||||
**Validates: Requirements 18.3**
|
||||
"""
|
||||
with patch('src.api.health.Session') as mock_session_class, \
|
||||
patch('src.services.cache_service.get_cache_service') as mock_cache_service, \
|
||||
patch('src.api.health.task_manager') as mock_task_manager:
|
||||
|
||||
# Setup mocks
|
||||
mock_session = Mock()
|
||||
mock_session_class.return_value.__enter__.return_value = mock_session
|
||||
|
||||
mock_cache = Mock()
|
||||
mock_cache._connected = True
|
||||
mock_cache._redis = AsyncMock()
|
||||
mock_cache._redis.ping = AsyncMock(return_value=True)
|
||||
mock_cache._redis.info = AsyncMock(return_value={
|
||||
'redis_version': '7.0.0',
|
||||
'connected_clients': 1,
|
||||
'used_memory_human': '1M'
|
||||
})
|
||||
mock_cache_service.return_value = mock_cache
|
||||
|
||||
# Make num_unhealthy components fail
|
||||
components_to_fail = ['database', 'task_manager', 'redis'][:num_unhealthy]
|
||||
|
||||
if 'database' in components_to_fail:
|
||||
mock_session.exec.side_effect = Exception("Database failed")
|
||||
else:
|
||||
mock_session.exec.return_value = None
|
||||
|
||||
if 'task_manager' in components_to_fail:
|
||||
mock_task_manager.get_stats.side_effect = Exception("Task manager failed")
|
||||
else:
|
||||
mock_task_manager.get_stats.return_value = {
|
||||
'total_tasks': 0,
|
||||
'completed_tasks': 0,
|
||||
'failed_tasks': 0,
|
||||
'queue_size': 0
|
||||
}
|
||||
|
||||
if 'redis' in components_to_fail:
|
||||
mock_cache._redis.ping.side_effect = Exception("Redis failed")
|
||||
|
||||
# Call the detailed health check endpoint
|
||||
response = client.get("/health/detailed")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = unwrap_response_data(response)
|
||||
|
||||
# Property: Overall status should reflect component health
|
||||
if num_unhealthy == 0:
|
||||
# All healthy - overall should be healthy
|
||||
assert data["status"] == "healthy"
|
||||
else:
|
||||
# Some unhealthy - overall should be unhealthy or degraded
|
||||
assert data["status"] in ["unhealthy", "degraded"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "--tb=short"])
|
||||
Reference in New Issue
Block a user