feat(infra/llm): neutral types + LLMClient interface + estimate tokenizer + fake

internal/infra/llm/tokenizer_test.go

@@ -0,0 +1,27 @@
+package llm
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestEstimateTokens_AsciiAndCJK(t *testing.T) {
+	require.Equal(t, 0, EstimateTokens(""))
+	require.InDelta(t, 1, EstimateTokens("hi"), 1)        // 2 chars / 4 ≈ 0~1
+	require.InDelta(t, 25, EstimateTokens(string(make([]byte, 100))), 1) // 100 chars / 4 = 25
+	// CJK 按 rune 数计算
+	require.InDelta(t, 2, EstimateTokens("你好"), 1)        // 2 runes / 4 ≈ 0~1，向上取整 1
+}
+
+func TestEstimateRequestTokens_SumsAllBlocks(t *testing.T) {
+	req := Request{
+		SystemPrompt: "You are helpful.", // 16 chars / 4 = 4
+		Messages: []Message{
+			{Role: RoleUser, Blocks: []ContentBlock{{Type: "text", Text: "hello world"}}}, // 11 chars / 4 = 3
+			{Role: RoleAssistant, Blocks: []ContentBlock{{Type: "text", Text: "hi"}}},     // 2 chars / 4 = 1
+		},
+	}
+	got := EstimateRequestTokens(req)
+	require.InDelta(t, 8, got, 2)
+}