bugfix

internal/codeindex/chunker_test.go

@@ -0,0 +1,91 @@
+package codeindex
+
+import (
+	"crypto/sha256"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestChunker_WindowsAndLineNumbers(t *testing.T) {
+	c := &Chunker{WindowLines: 10, OverlapLines: 2, MaxFileBytes: 1 << 20, MaxChunkTokens: 10000}
+	// 25 numbered lines.
+	var sb strings.Builder
+	for i := 1; i <= 25; i++ {
+		sb.WriteString("line")
+		sb.WriteByte(byte('0' + i%10))
+		sb.WriteByte('\n')
+	}
+	chunks := c.Chunk("a.go", []byte(sb.String()))
+	require.NotEmpty(t, chunks)
+
+	// First window: lines 1..10.
+	require.Equal(t, 1, chunks[0].StartLine)
+	require.Equal(t, 10, chunks[0].EndLine)
+	require.Equal(t, "go", chunks[0].Lang)
+
+	// step = window - overlap = 8 → next window starts at line 9.
+	require.Equal(t, 9, chunks[1].StartLine)
+	require.Equal(t, 18, chunks[1].EndLine)
+
+	// Last chunk must end exactly at line 25.
+	last := chunks[len(chunks)-1]
+	require.Equal(t, 25, last.EndLine)
+}
+
+func TestChunker_ContentSHAStable(t *testing.T) {
+	c := NewChunker()
+	content := []byte("package x\n\nfunc Y() {}\n")
+	a := c.Chunk("x.go", content)
+	b := c.Chunk("x.go", content)
+	require.Equal(t, len(a), len(b))
+	require.Equal(t, a[0].ContentSHA, b[0].ContentSHA)
+	// SHA must be sha256 of the chunk content.
+	sum := sha256.Sum256([]byte(a[0].Content))
+	require.Equal(t, sum[:], a[0].ContentSHA)
+}
+
+func TestChunker_SkipsBinary(t *testing.T) {
+	c := NewChunker()
+	bin := []byte("text\x00more\x00binary")
+	require.Nil(t, c.Chunk("data.bin", bin))
+	require.True(t, c.Skippable("data.bin", len(bin), bin))
+}
+
+func TestChunker_SkipsOversize(t *testing.T) {
+	c := &Chunker{WindowLines: 10, MaxFileBytes: 8}
+	big := []byte("aaaaaaaaaaaaaaaaaaaa\n") // > 8 bytes
+	require.Nil(t, c.Chunk("big.txt", big))
+}
+
+func TestChunker_SkipsVendored(t *testing.T) {
+	c := NewChunker()
+	content := []byte("package vendored\n")
+	require.Nil(t, c.Chunk("vendor/foo/bar.go", content))
+	require.Nil(t, c.Chunk("node_modules/pkg/index.js", content))
+	require.True(t, isVendored("a/b/node_modules/x.js"))
+	require.False(t, isVendored("internal/codeindex/chunker.go"))
+}
+
+func TestChunker_EmptyInput(t *testing.T) {
+	c := NewChunker()
+	require.Nil(t, c.Chunk("e.go", nil))
+	require.Nil(t, c.Chunk("e.go", []byte("")))
+}
+
+func TestChunker_TokenCapTruncates(t *testing.T) {
+	// One very long line that exceeds the token cap is truncated.
+	c := &Chunker{WindowLines: 60, OverlapLines: 10, MaxFileBytes: 1 << 20, MaxChunkTokens: 5}
+	long := strings.Repeat("x", 1000)
+	chunks := c.Chunk("min.js", []byte(long+"\n"))
+	require.Len(t, chunks, 1)
+	require.LessOrEqual(t, len(chunks[0].Content), 5*4) // ~maxTok*4 bytes
+}
+
+func TestLangForPath(t *testing.T) {
+	require.Equal(t, "go", langForPath("a/b.go"))
+	require.Equal(t, "typescript", langForPath("c.tsx"))
+	require.Equal(t, "python", langForPath("x.py"))
+	require.Equal(t, "", langForPath("Makefile"))
+}