35: several improvements on packer, rag and ask packages

pkg/app/app.go

@@ -17,8 +17,11 @@ import (
 var usageText string = `tlm suggest "<prompt>"
 tlm s --model=qwen2.5-coder:1.5b --style=stable "<prompt>"
 
-tlm explain "<command>"
-tlm e --model=llama3.2:1b --style=balanced "<command>"`
+tlm explain "<command>" # explain a command 
+tlm e --model=llama3.2:1b --style=balanced "<command>" # explain a command with a overrided model
+
+tlm ask "<prompt>" # ask a question
+tlm ask --context . --include *.md "<prompt>" # ask a question with a context`
 
 type TlmApp struct {
 	App *cli.App

pkg/ask/ask.go

@@ -8,6 +8,7 @@ import (
 
 type Ask struct {
 	api     *ollama.Client
+	user    string
 	version string
 	model   string
 	style   string

pkg/ask/cli.go

@@ -2,6 +2,8 @@ package ask
 
 import (
 	"errors"
+	"fmt"
+	"os/user"
 
 	"github.com/urfave/cli/v2"
 	"github.com/yusufcanb/tlm/pkg/packer"
@@ -14,17 +16,24 @@ func (a *Ask) before(c *cli.Context) error {
 		return errors.New("message is required")
 	}
 
+	user, err := user.Current()
+	if err != nil {
+		a.user = "User"
+	}
+	a.user = user.Username
+
 	return nil
 }
 
 func (a *Ask) action(c *cli.Context) error {
+	isInteractive := c.Bool("interactive")
 	contextDir := c.Path("context")
-	var chatContext string // chat context
+	var chatContext string    // chat context
+	var numCtx int = 1024 * 8 // num_ctx in Ollama API
 
 	if contextDir != "" {
 		includePatterns := c.StringSlice("include")
 		excludePatterns := c.StringSlice("exclude")
-
 		// fmt.Printf("include=%v, exclude=%v\n\n", includePatterns, excludePatterns)
 
 		// Pack files under the context directory
@@ -47,13 +56,36 @@ func (a *Ask) action(c *cli.Context) error {
 		}
 	}
 
+	fmt.Printf("\n🤖 %s\n───────────────────\n", a.model)
+
 	message := c.Args().First()
 	rag := rag.NewRAGChat(a.api, chatContext)
-	_, err := rag.Send(message)
+	_, err := rag.Send(message, numCtx)
 	if err != nil {
 		return err
 	}
 
+	user.Current()
+
+	if isInteractive {
+		for {
+			fmt.Printf("\n\n👤 %s\n───────────────────\n", a.user)
+			var input string
+			fmt.Scanln(&input)
+
+			if input == "exit" {
+				break
+			}
+
+			_, err := rag.Send(input, numCtx)
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("\n\n")
+		}
+	}
+
 	return nil
 }
 
@@ -78,12 +110,12 @@ func (a *Ask) Command() *cli.Command {
 			&cli.StringSliceFlag{
 				Name:    "include",
 				Aliases: []string{"i"},
-				Usage:   "Include patterns. E.g. --include=*.txt",
+				Usage:   "Include patterns. e.g. --include=*.txt",
 			},
 			&cli.StringSliceFlag{
 				Name:    "exclude",
 				Aliases: []string{"e"},
-				Usage:   "Exclude patterns. E.g. --exclude=*.binary",
+				Usage:   "Exclude patterns. e.g. --exclude=*.binary",
 			},
 			&cli.BoolFlag{
 				Name:    "interactive",

pkg/packer/directory.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"path"
+	"path/filepath"
 	"sort"
 	"text/template"
 	"time"
@@ -16,29 +18,28 @@ type directoryPacker struct {
 	files    []File
 }
 
-func (dp *directoryPacker) Pack(path any, includePatterns []string, ignorePatterns []string) (*Result, error) {
+func (dp *directoryPacker) Pack(contextPath any, includePatterns []string, ignorePatterns []string) (*Result, error) {
 	var numTokens int = 0
 	var numChars int = 0
 
-	path, ok := path.(string)
+	_, ok := contextPath.(string)
 	if !ok {
 		return nil, errors.New("path is not a string")
 	}
 
-	filePaths, err := internal.GetContextFilePaths(path.(string), includePatterns, ignorePatterns)
+	filePaths, err := internal.GetContextFilePaths(contextPath.(string), includePatterns, ignorePatterns)
 	if err != nil {
 		return nil, errors.New("failed to get context files")
 	}
 
 	for _, fp := range filePaths {
-		content, tokens, chars, err := internal.GetFileContent(fp)
+		content, tokens, chars, err := internal.GetFileContent(contextPath.(string), fp)
 		if err != nil {
-			fmt.Printf("Error reading file: %v\n", err)
-			return nil, errors.New("failed to get file content")
+			return nil, fmt.Errorf("failed to get file content %s : %s", fp, err.Error())
 		}
 
 		dp.files = append(dp.files, File{
-			Path:    fp,
+			Path:    path.Join(filepath.ToSlash(contextPath.(string)), fp),
 			Content: content,
 			Chars:   chars,
 			Tokens:  tokens,
@@ -67,26 +68,25 @@ func (dp *directoryPacker) Render(result *Result) (string, error) {
 	// Parse and execute the template.
 	tmpl, err := template.New("packed").Parse(dp.template)
 	if err != nil {
-		return "", fmt.Errorf("Error parsing template: %v", err)
+		return "", fmt.Errorf("error parsing template: %v", err)
 	}
 	var buf bytes.Buffer
 	if err := tmpl.Execute(&buf, data); err != nil {
-		return "", fmt.Errorf("Error executing template: %v", err)
+		return "", fmt.Errorf("error executing template: %v", err)
 	}
 
 	return buf.String(), nil
 }
 
-func (dp *directoryPacker) PrintTopFiles(result *Result, top int) {
-	// Print the top 5 files
-	fmt.Printf("📈 Top 5 Files by Character Count and Token Count:\n──────────────────────────────────────────────────\n")
+func (dp *directoryPacker) PrintTopFiles(result *Result, n int) {
+	fmt.Printf("📈 Top %d Files by Character Count and Token Count:\n──────────────────────────────────────────────────\n", n)
 	sort.Slice(result.Files, func(i, j int) bool {
 		return result.Files[i].Tokens > result.Files[j].Tokens
 	})
 
-	// Print the top 5 files
+	// Print the top n files
 	for i, file := range result.Files {
-		if i >= 5 {
+		if i >= n {
 			break
 		}
 		fmt.Printf("%d.  %s (%d chars, %d tokens)\n", i+1, file.Path, file.Chars, file.Tokens)

pkg/packer/internal/exclude.go

@@ -1,17 +1,101 @@
 package internal
 
-import (
-	"path/filepath"
-
-	"github.com/bmatcuk/doublestar/v4"
-)
+var defaultIgnoreList = []string{
+	".git/**",
+	".hg/**",
+	".hgignore",
+	".svn/**",
+	"node_modules/**",
+	"**/node_modules/**",
+	"bower_components/**",
+	"**/bower_components/**",
+	"jspm_packages/**",
+	"**/jspm_packages/**",
+	"vendor/**",
+	".bundle/**",
+	".gradle/**",
+	"target/**",
+	"logs/**",
+	"**/*.log",
+	"**/npm-debug.log*",
+	"**/yarn-debug.log*",
+	"**/yarn-error.log*",
+	"pids/**",
+	"*.pid",
+	"*.seed",
+	"*.pid.lock",
+	"lib-cov/**",
+	"coverage/**",
+	".nyc_output/**",
+	".grunt/**",
+	".lock-wscript",
+	"build/Release/**",
+	"typings/**",
+	"**/.npm/**",
+	".eslintcache",
+	".rollup.cache/**",
+	".webpack.cache/**",
+	".parcel-cache/**",
+	".sass-cache/**",
+	"*.cache",
+	".node_repl_history",
+	"*.tgz",
+	"**/.yarn/**",
+	"**/.yarn-integrity",
+	".env",
+	".next/**",
+	".nuxt/**",
+	".vuepress/dist/**",
+	".serverless/**",
+	".fusebox/**",
+	".dynamodb/**",
+	"dist/**",
+	"**/.DS_Store",
+	"**/Thumbs.db",
+	".idea/**",
+	".vscode/**",
+	"**/*.swp",
+	"**/*.swo",
+	"**/*.swn",
+	"**/*.bak",
+	"build/**",
+	"out/**",
+	"tmp/**",
+	"temp/**",
+	"repomix-output.*",
+	"repopack-output.*",
+	"**/package-lock.json",
+	"**/yarn-error.log",
+	"**/yarn.lock",
+	"**/pnpm-lock.yaml",
+	"**/bun.lockb",
+	"**/__pycache__/**",
+	"**/*.py[cod]",
+	"**/venv/**",
+	"**/.venv/**",
+	"**/.pytest_cache/**",
+	"**/.mypy_cache/**",
+	"**/.ipynb_checkpoints/**",
+	"**/Pipfile.lock",
+	"**/poetry.lock",
+	"**/Cargo.lock",
+	"**/Cargo.toml.orig",
+	"**/target/**",
+	"**/*.rs.bk",
+	"**/composer.lock",
+	"**/Gemfile.lock",
+	"**/go.sum",
+	"**/mix.lock",
+	"**/stack.yaml.lock",
+	"**/cabal.project.freeze",
+}
 
 // shouldExclude returns true if the file path matches any of the exclude patterns.
-func shouldExclude(path string, patterns []string) bool {
-	for _, pattern := range patterns {
-		if match, _ := doublestar.Match(pattern, filepath.ToSlash(path)); match {
-			return true
-		}
-	}
-	return false
-}
+// func shouldExclude(path string, patterns []string) bool {
+// 	for _, pattern := range patterns {
+// 		if match, _ := doublestar.Match(pattern, filepath.ToSlash(path)); match {
+// 			return true
+// 		}
+// 	}
+// 	return false
+// }

pkg/packer/internal/file.go

@@ -1,15 +1,18 @@
 package internal
 
 import (
+	"fmt"
+	"io/fs"
 	"os"
+	"path"
 
 	"github.com/bmatcuk/doublestar/v4"
 )
 
 // GetContextFilePaths returns the file paths in the given directory that match the include patterns and do not match the exclude patterns.
 func GetContextFilePaths(path string, includePatterns []string, excludePatterns []string) ([]string, error) {
 	dirFs := os.DirFS(path)
-	filePaths := make([]string, 0)
+	filePathSet := make(map[string]struct{})
 
 	if len(includePatterns) == 0 {
 		includePatterns = []string{"**/*"}
@@ -21,41 +24,54 @@ func GetContextFilePaths(path string, includePatterns []string, excludePatterns
 		if err != nil {
 			return nil, err
 		}
-		filePaths = append(filePaths, paths...)
+		for _, p := range paths {
+			// Check if the path corresponds to a file.
+			info, err := fs.Stat(dirFs, p)
+			if err != nil {
+				return nil, fmt.Errorf("failed to stat %s: %w", p, err)
+			}
+			// Skip if it's a directory.
+			if info.IsDir() {
+				continue
+			}
+			filePathSet[p] = struct{}{}
+		}
 	}
 
 	// Exclude files that match the exclude patterns
-	for _, ep := range excludePatterns {
-		for _, fp := range filePaths {
+	for _, ep := range append(excludePatterns, defaultIgnoreList...) {
+		for fp := range filePathSet {
 			shouldExclude, err := doublestar.PathMatch(ep, fp)
 			if err != nil {
 				return nil, err
 			}
 
 			// if the file should be excluded, remove it from the list
 			if shouldExclude {
-
-				for i, path := range filePaths {
-					if path == fp {
-						filePaths = append(filePaths[:i], filePaths[i+1:]...)
-						break
-					}
-				}
-
-				// fmt.Println("Excluded file: ", fp)
+				delete(filePathSet, fp)
 			}
 		}
 	}
 
+	// Convert the set into a slice.
+	filePaths := make([]string, 0, len(filePathSet))
+	for fp := range filePathSet {
+		filePaths = append(filePaths, fp)
+		delete(filePathSet, fp)
+	}
+
 	return filePaths, nil
 }
 
 // GetFileContent returns the content of a file and the number of tokens in it.
-func GetFileContent(filePath string) (string, int, int, error) {
-	// Get the file content
-	b, err := os.ReadFile(filePath)
+func GetFileContent(baseDir string, filePath string) (string, int, int, error) {
+	b, err := os.ReadFile(path.Join(baseDir, filePath))
 	if err != nil {
-		return "", -1, -1, err
+		return "", -1, -1, fmt.Errorf("failed to read file content: %s ", err.Error())
+	}
+
+	if isBinary(b) {
+		return "__BINARY__", 0, 0, nil
 	}
 
 	return string(b), GetTokenCount(string(b)), len(b), nil