bugfix

2026-06-22 08:55:57 +08:00
parent dbb87823e8
commit 6ade6e8fa9
325 changed files with 41131 additions and 855 deletions
@@ -0,0 +1,216 @@
+package codeindex
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"path/filepath"
+	"strings"
+
+	"github.com/yan1h/agent-coding-workflow/internal/infra/llm"
+)
+
+// Chunker config defaults. Windows are line-based; the token cap (estimated via
+// llm.EstimateTokens) hard-bounds embedding cost on pathological long lines.
+const (
+	DefaultWindowLines    = 60
+	DefaultOverlapLines   = 10
+	DefaultMaxFileBytes   = 512 * 1024 // skip files larger than this
+	DefaultMaxChunkTokens = 1024       // estimated; oversize windows are truncated
+)
+
+// Chunker splits file content into overlapping line windows.
+type Chunker struct {
+	WindowLines    int
+	OverlapLines   int
+	MaxFileBytes   int
+	MaxChunkTokens int
+}
+
+// NewChunker returns a Chunker with platform defaults.
+func NewChunker() *Chunker {
+	return &Chunker{
+		WindowLines:    DefaultWindowLines,
+		OverlapLines:   DefaultOverlapLines,
+		MaxFileBytes:   DefaultMaxFileBytes,
+		MaxChunkTokens: DefaultMaxChunkTokens,
+	}
+}
+
+func (c *Chunker) window() int {
+	if c.WindowLines > 0 {
+		return c.WindowLines
+	}
+	return DefaultWindowLines
+}
+
+func (c *Chunker) overlap() int {
+	if c.OverlapLines >= 0 && c.OverlapLines < c.window() {
+		return c.OverlapLines
+	}
+	return DefaultOverlapLines
+}
+
+func (c *Chunker) maxFileBytes() int {
+	if c.MaxFileBytes > 0 {
+		return c.MaxFileBytes
+	}
+	return DefaultMaxFileBytes
+}
+
+func (c *Chunker) maxChunkTokens() int {
+	if c.MaxChunkTokens > 0 {
+		return c.MaxChunkTokens
+	}
+	return DefaultMaxChunkTokens
+}
+
+// Skippable reports whether a file should not be indexed (binary, oversize, or
+// vendored). Callers should check this before reading large files.
+func (c *Chunker) Skippable(filePath string, size int, content []byte) bool {
+	if size > c.maxFileBytes() {
+		return true
+	}
+	if isVendored(filePath) {
+		return true
+	}
+	if isBinary(content) {
+		return true
+	}
+	return false
+}
+
+// Chunk splits content into overlapping line windows. Returns nil for skippable
+// or empty input. ContentSHA is the sha256 of the chunk's exact content so the
+// build runner can reuse already-embedded chunks across commits.
+func (c *Chunker) Chunk(filePath string, content []byte) []Chunk {
+	if len(content) == 0 {
+		return nil
+	}
+	if c.Skippable(filePath, len(content), content) {
+		return nil
+	}
+	lines := strings.Split(string(content), "\n")
+	// Drop a trailing empty element from a final newline so end_line is accurate.
+	if n := len(lines); n > 0 && lines[n-1] == "" {
+		lines = lines[:n-1]
+	}
+	if len(lines) == 0 {
+		return nil
+	}
+	lang := langForPath(filePath)
+	window := c.window()
+	step := window - c.overlap()
+	if step <= 0 {
+		step = window
+	}
+	maxTok := c.maxChunkTokens()
+
+	var out []Chunk
+	for start := 0; start < len(lines); start += step {
+		end := start + window
+		if end > len(lines) {
+			end = len(lines)
+		}
+		body := strings.Join(lines[start:end], "\n")
+		// Hard token cap: truncate runaway windows (minified/one-line files).
+		if llm.EstimateTokens(body) > maxTok {
+			body = truncateToTokens(body, maxTok)
+		}
+		sum := sha256.Sum256([]byte(body))
+		out = append(out, Chunk{
+			StartLine:  start + 1, // 1-based, inclusive
+			EndLine:    end,       // 1-based, inclusive
+			Lang:       lang,
+			Content:    body,
+			ContentSHA: sum[:],
+		})
+		if end == len(lines) {
+			break
+		}
+	}
+	return out
+}
+
+// truncateToTokens cuts s down to roughly maxTok estimated tokens (4 chars/token).
+func truncateToTokens(s string, maxTok int) string {
+	maxBytes := maxTok * 4
+	if len(s) <= maxBytes {
+		return s
+	}
+	// Cut on a UTF-8 boundary.
+	for maxBytes > 0 && (s[maxBytes]&0xC0) == 0x80 {
+		maxBytes--
+	}
+	return s[:maxBytes]
+}
+
+// isBinary reports whether content looks binary (NUL byte in the first 8KB).
+func isBinary(content []byte) bool {
+	n := len(content)
+	if n > 8192 {
+		n = 8192
+	}
+	return bytes.IndexByte(content[:n], 0) >= 0
+}
+
+// vendoredDirs are path segments whose contents are excluded from the index.
+var vendoredDirs = []string{
+	"vendor/", "node_modules/", ".git/", "dist/", "build/", "third_party/",
+	"testdata/", ".venv/", "__pycache__/",
+}
+
+func isVendored(p string) bool {
+	norm := filepath.ToSlash(p)
+	for _, d := range vendoredDirs {
+		if strings.HasPrefix(norm, d) || strings.Contains(norm, "/"+d) {
+			return true
+		}
+	}
+	return false
+}
+
+// langForPath maps a file extension to a coarse language label (best-effort).
+func langForPath(p string) string {
+	switch strings.ToLower(filepath.Ext(p)) {
+	case ".go":
+		return "go"
+	case ".ts", ".tsx":
+		return "typescript"
+	case ".js", ".jsx", ".mjs", ".cjs":
+		return "javascript"
+	case ".vue":
+		return "vue"
+	case ".py":
+		return "python"
+	case ".rs":
+		return "rust"
+	case ".java":
+		return "java"
+	case ".c", ".h":
+		return "c"
+	case ".cpp", ".cc", ".hpp":
+		return "cpp"
+	case ".rb":
+		return "ruby"
+	case ".php":
+		return "php"
+	case ".sql":
+		return "sql"
+	case ".sh", ".bash":
+		return "shell"
+	case ".md", ".markdown":
+		return "markdown"
+	case ".json":
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
+	case ".toml":
+		return "toml"
+	case ".html", ".htm":
+		return "html"
+	case ".css", ".scss":
+		return "css"
+	default:
+		return ""
+	}
+}