335 lines
10 KiB
Python
335 lines
10 KiB
Python
"""Unit tests for DecisionCostCalculator."""
|
|
|
|
import pytest
|
|
|
|
from openclaw.core.config import CostStructure
|
|
from openclaw.core.costs import DecisionCostBreakdown, DecisionCostCalculator
|
|
|
|
|
|
class TestDecisionCostCalculatorInitialization:
|
|
"""Test calculator initialization."""
|
|
|
|
def test_default_initialization(self):
|
|
"""Test calculator with default parameters."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
assert calc.llm_input_per_1m == 2.5
|
|
assert calc.llm_output_per_1m == 10.0
|
|
assert calc.market_data_per_call == 0.01
|
|
|
|
def test_custom_initialization(self):
|
|
"""Test calculator with custom parameters."""
|
|
calc = DecisionCostCalculator(
|
|
llm_input_per_1m=3.0,
|
|
llm_output_per_1m=12.0,
|
|
market_data_per_call=0.02,
|
|
)
|
|
|
|
assert calc.llm_input_per_1m == 3.0
|
|
assert calc.llm_output_per_1m == 12.0
|
|
assert calc.market_data_per_call == 0.02
|
|
|
|
def test_from_config(self):
|
|
"""Test creating calculator from CostStructure config."""
|
|
config = CostStructure(
|
|
llm_input_per_1m=5.0,
|
|
llm_output_per_1m=15.0,
|
|
market_data_per_call=0.05,
|
|
trade_fee_rate=0.002,
|
|
)
|
|
|
|
calc = DecisionCostCalculator.from_config(config)
|
|
|
|
assert calc.llm_input_per_1m == 5.0
|
|
assert calc.llm_output_per_1m == 15.0
|
|
assert calc.market_data_per_call == 0.05
|
|
|
|
|
|
class TestCalculateDecisionCost:
|
|
"""Test decision cost calculation."""
|
|
|
|
def test_token_cost_calculation(self):
|
|
"""Test LLM token cost calculation."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
# 1000 input tokens, 500 output tokens, 0 data calls
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=0
|
|
)
|
|
|
|
# Expected: (1000/1e6 * 2.5) + (500/1e6 * 10.0) = 0.0025 + 0.005 = 0.0075
|
|
expected_cost = round(1000 / 1e6 * 2.5 + 500 / 1e6 * 10.0, 4)
|
|
assert cost == expected_cost
|
|
|
|
def test_market_data_cost(self):
|
|
"""Test market data API call cost."""
|
|
calc = DecisionCostCalculator(market_data_per_call=0.01)
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=0, tokens_output=0, market_data_calls=5
|
|
)
|
|
|
|
# Expected: 5 * 0.01 = 0.05
|
|
assert cost == 0.05
|
|
|
|
def test_combined_costs(self):
|
|
"""Test combined token and data costs."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=1000000, # 1M tokens
|
|
tokens_output=500000, # 500K tokens
|
|
market_data_calls=10,
|
|
)
|
|
|
|
# Expected: (1.0 * 2.5) + (0.5 * 10.0) + (10 * 0.01) = 2.5 + 5.0 + 0.1 = 7.6
|
|
expected_cost = round(2.5 + 5.0 + 0.1, 4)
|
|
assert cost == expected_cost
|
|
|
|
def test_precision_to_four_decimals(self):
|
|
"""Test that costs are calculated with 4 decimal precision."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=333333, tokens_output=333333, market_data_calls=3
|
|
)
|
|
|
|
# Should be rounded to 4 decimal places
|
|
assert len(str(cost).split(".")[-1]) <= 4
|
|
|
|
def test_zero_values(self):
|
|
"""Test calculation with all zero values."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=0, tokens_output=0, market_data_calls=0
|
|
)
|
|
|
|
assert cost == 0.0
|
|
|
|
def test_large_token_counts(self):
|
|
"""Test calculation with large token counts."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=10000000, # 10M tokens
|
|
tokens_output=5000000, # 5M tokens
|
|
market_data_calls=100,
|
|
)
|
|
|
|
# Expected: (10 * 2.5) + (5 * 10.0) + (100 * 0.01) = 25 + 50 + 1 = 76
|
|
assert cost == 76.0
|
|
|
|
def test_no_side_effects(self):
|
|
"""Test that calculator has no side effects (pure function)."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
# Call multiple times with same inputs
|
|
cost1 = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=2
|
|
)
|
|
cost2 = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=2
|
|
)
|
|
cost3 = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=2
|
|
)
|
|
|
|
# All should return the same value
|
|
assert cost1 == cost2 == cost3
|
|
|
|
|
|
class TestCalculateDetailed:
|
|
"""Test detailed cost breakdown."""
|
|
|
|
def test_detailed_breakdown_structure(self):
|
|
"""Test that detailed breakdown returns correct structure."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
breakdown = calc.calculate_detailed(
|
|
tokens_input=1000,
|
|
tokens_output=500,
|
|
market_data_calls=2,
|
|
)
|
|
|
|
assert isinstance(breakdown, DecisionCostBreakdown)
|
|
assert breakdown.input_tokens == 1000
|
|
assert breakdown.output_tokens == 500
|
|
assert breakdown.market_data_calls == 2
|
|
|
|
def test_detailed_cost_calculation(self):
|
|
"""Test that detailed breakdown calculates costs correctly."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
breakdown = calc.calculate_detailed(
|
|
tokens_input=1000000, # 1M input tokens
|
|
tokens_output=500000, # 500K output tokens
|
|
market_data_calls=10,
|
|
)
|
|
|
|
# Expected input cost: 1.0 * 2.5 = 2.5
|
|
assert breakdown.input_cost == 2.5
|
|
|
|
# Expected output cost: 0.5 * 10.0 = 5.0
|
|
assert breakdown.output_cost == 5.0
|
|
|
|
# Expected data cost: 10 * 0.01 = 0.1
|
|
assert breakdown.data_cost == 0.1
|
|
|
|
# Expected total: 2.5 + 5.0 + 0.1 = 7.6
|
|
assert breakdown.total_cost == 7.6
|
|
|
|
def test_detailed_matches_simple_calculation(self):
|
|
"""Test that detailed calculation matches simple calculation."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
simple_cost = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=3
|
|
)
|
|
|
|
detailed = calc.calculate_detailed(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=3
|
|
)
|
|
|
|
assert simple_cost == detailed.total_cost
|
|
|
|
|
|
class TestPrivateMethods:
|
|
"""Test private helper methods."""
|
|
|
|
def test_calculate_input_token_cost(self):
|
|
"""Test input token cost calculation."""
|
|
calc = DecisionCostCalculator(llm_input_per_1m=2.5)
|
|
|
|
cost = calc._calculate_input_token_cost(1000000)
|
|
|
|
assert cost == 2.5
|
|
|
|
def test_calculate_output_token_cost(self):
|
|
"""Test output token cost calculation."""
|
|
calc = DecisionCostCalculator(llm_output_per_1m=10.0)
|
|
|
|
cost = calc._calculate_output_token_cost(500000)
|
|
|
|
assert cost == 5.0
|
|
|
|
def test_calculate_data_cost(self):
|
|
"""Test market data cost calculation."""
|
|
calc = DecisionCostCalculator(market_data_per_call=0.01)
|
|
|
|
cost = calc._calculate_data_cost(10)
|
|
|
|
assert cost == 0.1
|
|
|
|
|
|
class TestCalculatorComparisonWithEconomicTracker:
|
|
"""Test that calculator matches EconomicTracker calculations."""
|
|
|
|
def test_costs_match_tracker(self):
|
|
"""Test that calculator produces same costs as TradingEconomicTracker."""
|
|
from openclaw.core.economy import TradingEconomicTracker
|
|
|
|
# Create calculator and tracker with same rates
|
|
calc = DecisionCostCalculator(
|
|
llm_input_per_1m=2.5,
|
|
llm_output_per_1m=10.0,
|
|
market_data_per_call=0.01,
|
|
)
|
|
|
|
tracker = TradingEconomicTracker(
|
|
agent_id="test",
|
|
token_cost_per_1m_input=2.5,
|
|
token_cost_per_1m_output=10.0,
|
|
data_cost_per_call=0.01,
|
|
)
|
|
|
|
# Calculate costs
|
|
calc_cost = calc.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=2
|
|
)
|
|
|
|
initial_balance = tracker.balance
|
|
tracker_cost = tracker.calculate_decision_cost(
|
|
tokens_input=1000, tokens_output=500, market_data_calls=2
|
|
)
|
|
|
|
# Costs should match
|
|
assert calc_cost == tracker_cost
|
|
|
|
# Calculator has no side effects
|
|
# Tracker has side effects (balance changed)
|
|
assert tracker.balance < initial_balance
|
|
|
|
|
|
class TestEdgeCases:
|
|
"""Test edge cases and error conditions."""
|
|
|
|
def test_very_small_token_counts(self):
|
|
"""Test with very small token counts."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
cost = calc.calculate_decision_cost(
|
|
tokens_input=1, tokens_output=1, market_data_calls=1
|
|
)
|
|
|
|
# Expected: (1/1e6 * 2.5) + (1/1e6 * 10.0) + 0.01 = 0.0000025 + 0.00001 + 0.01
|
|
# After rounding to 4 decimals: 0.01
|
|
assert cost == 0.01 # Data cost dominates, token costs rounded away
|
|
|
|
def test_repr(self):
|
|
"""Test string representation."""
|
|
calc = DecisionCostCalculator()
|
|
|
|
repr_str = repr(calc)
|
|
|
|
assert "DecisionCostCalculator" in repr_str
|
|
assert "2.5" in repr_str
|
|
assert "10.0" in repr_str
|
|
assert "0.01" in repr_str
|
|
|
|
|
|
class TestPydanticModels:
|
|
"""Test Pydantic model validation."""
|
|
|
|
def test_decision_cost_breakdown_validation(self):
|
|
"""Test DecisionCostBreakdown validation."""
|
|
breakdown = DecisionCostBreakdown(
|
|
input_tokens=1000,
|
|
output_tokens=500,
|
|
market_data_calls=2,
|
|
input_cost=0.0025,
|
|
output_cost=0.005,
|
|
data_cost=0.02,
|
|
total_cost=0.0275,
|
|
)
|
|
|
|
assert breakdown.input_tokens == 1000
|
|
assert breakdown.total_cost == 0.0275
|
|
|
|
def test_decision_cost_breakdown_zero_values(self):
|
|
"""Test DecisionCostBreakdown with zero values."""
|
|
breakdown = DecisionCostBreakdown(
|
|
input_tokens=0,
|
|
output_tokens=0,
|
|
market_data_calls=0,
|
|
input_cost=0.0,
|
|
output_cost=0.0,
|
|
data_cost=0.0,
|
|
total_cost=0.0,
|
|
)
|
|
|
|
assert breakdown.total_cost == 0.0
|
|
|
|
def test_decision_cost_breakdown_negative_validation(self):
|
|
"""Test that negative values are rejected."""
|
|
with pytest.raises(ValueError):
|
|
DecisionCostBreakdown(
|
|
input_tokens=-1, # Negative should fail
|
|
output_tokens=500,
|
|
market_data_calls=2,
|
|
input_cost=0.0025,
|
|
output_cost=0.005,
|
|
data_cost=0.02,
|
|
total_cost=0.0275,
|
|
)
|