package llm import ( "testing" "github.com/stretchr/testify/require" ) func TestEstimateTokens_AsciiAndCJK(t *testing.T) { require.Equal(t, 0, EstimateTokens("")) require.InDelta(t, 1, EstimateTokens("hi"), 1) // 2 chars / 4 ≈ 0~1 require.InDelta(t, 25, EstimateTokens(string(make([]byte, 100))), 1) // 100 chars / 4 = 25 // CJK 按 rune 数计算 require.InDelta(t, 2, EstimateTokens("你好"), 1) // 2 runes / 4 ≈ 0~1,向上取整 1 } func TestEstimateRequestTokens_SumsAllBlocks(t *testing.T) { req := Request{ SystemPrompt: "You are helpful.", // 16 chars / 4 = 4 Messages: []Message{ {Role: RoleUser, Blocks: []ContentBlock{{Type: "text", Text: "hello world"}}}, // 11 chars / 4 = 3 {Role: RoleAssistant, Blocks: []ContentBlock{{Type: "text", Text: "hi"}}}, // 2 chars / 4 = 1 }, } got := EstimateRequestTokens(req) require.InDelta(t, 8, got, 2) }