You've already forked agentic-coding-workflow
92 lines
2.8 KiB
Go
92 lines
2.8 KiB
Go
package codeindex
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestChunker_WindowsAndLineNumbers(t *testing.T) {
|
|
c := &Chunker{WindowLines: 10, OverlapLines: 2, MaxFileBytes: 1 << 20, MaxChunkTokens: 10000}
|
|
// 25 numbered lines.
|
|
var sb strings.Builder
|
|
for i := 1; i <= 25; i++ {
|
|
sb.WriteString("line")
|
|
sb.WriteByte(byte('0' + i%10))
|
|
sb.WriteByte('\n')
|
|
}
|
|
chunks := c.Chunk("a.go", []byte(sb.String()))
|
|
require.NotEmpty(t, chunks)
|
|
|
|
// First window: lines 1..10.
|
|
require.Equal(t, 1, chunks[0].StartLine)
|
|
require.Equal(t, 10, chunks[0].EndLine)
|
|
require.Equal(t, "go", chunks[0].Lang)
|
|
|
|
// step = window - overlap = 8 → next window starts at line 9.
|
|
require.Equal(t, 9, chunks[1].StartLine)
|
|
require.Equal(t, 18, chunks[1].EndLine)
|
|
|
|
// Last chunk must end exactly at line 25.
|
|
last := chunks[len(chunks)-1]
|
|
require.Equal(t, 25, last.EndLine)
|
|
}
|
|
|
|
func TestChunker_ContentSHAStable(t *testing.T) {
|
|
c := NewChunker()
|
|
content := []byte("package x\n\nfunc Y() {}\n")
|
|
a := c.Chunk("x.go", content)
|
|
b := c.Chunk("x.go", content)
|
|
require.Equal(t, len(a), len(b))
|
|
require.Equal(t, a[0].ContentSHA, b[0].ContentSHA)
|
|
// SHA must be sha256 of the chunk content.
|
|
sum := sha256.Sum256([]byte(a[0].Content))
|
|
require.Equal(t, sum[:], a[0].ContentSHA)
|
|
}
|
|
|
|
func TestChunker_SkipsBinary(t *testing.T) {
|
|
c := NewChunker()
|
|
bin := []byte("text\x00more\x00binary")
|
|
require.Nil(t, c.Chunk("data.bin", bin))
|
|
require.True(t, c.Skippable("data.bin", len(bin), bin))
|
|
}
|
|
|
|
func TestChunker_SkipsOversize(t *testing.T) {
|
|
c := &Chunker{WindowLines: 10, MaxFileBytes: 8}
|
|
big := []byte("aaaaaaaaaaaaaaaaaaaa\n") // > 8 bytes
|
|
require.Nil(t, c.Chunk("big.txt", big))
|
|
}
|
|
|
|
func TestChunker_SkipsVendored(t *testing.T) {
|
|
c := NewChunker()
|
|
content := []byte("package vendored\n")
|
|
require.Nil(t, c.Chunk("vendor/foo/bar.go", content))
|
|
require.Nil(t, c.Chunk("node_modules/pkg/index.js", content))
|
|
require.True(t, isVendored("a/b/node_modules/x.js"))
|
|
require.False(t, isVendored("internal/codeindex/chunker.go"))
|
|
}
|
|
|
|
func TestChunker_EmptyInput(t *testing.T) {
|
|
c := NewChunker()
|
|
require.Nil(t, c.Chunk("e.go", nil))
|
|
require.Nil(t, c.Chunk("e.go", []byte("")))
|
|
}
|
|
|
|
func TestChunker_TokenCapTruncates(t *testing.T) {
|
|
// One very long line that exceeds the token cap is truncated.
|
|
c := &Chunker{WindowLines: 60, OverlapLines: 10, MaxFileBytes: 1 << 20, MaxChunkTokens: 5}
|
|
long := strings.Repeat("x", 1000)
|
|
chunks := c.Chunk("min.js", []byte(long+"\n"))
|
|
require.Len(t, chunks, 1)
|
|
require.LessOrEqual(t, len(chunks[0].Content), 5*4) // ~maxTok*4 bytes
|
|
}
|
|
|
|
func TestLangForPath(t *testing.T) {
|
|
require.Equal(t, "go", langForPath("a/b.go"))
|
|
require.Equal(t, "typescript", langForPath("c.tsx"))
|
|
require.Equal(t, "python", langForPath("x.py"))
|
|
require.Equal(t, "", langForPath("Makefile"))
|
|
}
|