You've already forked agentic-coding-workflow
28 lines
904 B
Go
28 lines
904 B
Go
package llm
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEstimateTokens_AsciiAndCJK(t *testing.T) {
|
|
require.Equal(t, 0, EstimateTokens(""))
|
|
require.InDelta(t, 1, EstimateTokens("hi"), 1) // 2 chars / 4 ≈ 0~1
|
|
require.InDelta(t, 25, EstimateTokens(string(make([]byte, 100))), 1) // 100 chars / 4 = 25
|
|
// CJK 按 rune 数计算
|
|
require.InDelta(t, 2, EstimateTokens("你好"), 1) // 2 runes / 4 ≈ 0~1，向上取整 1
|
|
}
|
|
|
|
func TestEstimateRequestTokens_SumsAllBlocks(t *testing.T) {
|
|
req := Request{
|
|
SystemPrompt: "You are helpful.", // 16 chars / 4 = 4
|
|
Messages: []Message{
|
|
{Role: RoleUser, Blocks: []ContentBlock{{Type: "text", Text: "hello world"}}}, // 11 chars / 4 = 3
|
|
{Role: RoleAssistant, Blocks: []ContentBlock{{Type: "text", Text: "hi"}}}, // 2 chars / 4 = 1
|
|
},
|
|
}
|
|
got := EstimateRequestTokens(req)
|
|
require.InDelta(t, 8, got, 2)
|
|
}
|