v3.0.12: add chat -e (edit) flag, don't remove chat file backup

Add edit parameter to Chat, pass to continueChat method

summary of diff --git  a/v3/core/api.go b/v3/core/api.go

- Add 'edit' boolean parameter to `Chat` function signature
- Pass 'edit' parameter to `history.continueChat` method call within `Chat` function

Refactor chat functionality, enhance edit feature, improve logging

summary of diff --git  a/v3/core/chat.go b/v3/core/chat.go

- Add `time` package to imports
- Rename parameter `level` to `contextLevel` in `continueChat` function signature and usage
- Introduce `edit` boolean parameter in `continueChat` function to allow editing behavior
- Implement prompt retrieval from most recent message if `edit` is true
- Append the user's prompt to messages conditionally based on `edit` status
- Include error checking and debug logging for `continueChat` execution
- Adjust role attribution from "AI" to "USER" in `parseChat` for preamble content
- Trim and uppercase `role` in `fixRole` before switching on its value
- Adapt backup file naming in `Save` to include a timestamp
- Optionally preserve backup file based on a condition check (commented out with XXX marker)

Add false parameter to grok.Chat and continueChat calls

summary of diff --git  a/v3/core/chat_test.go b/v3/core/chat_test.go

- Append an additional false parameter to the grok.Chat function call in the test for starting a chat and checking the color of the widget's center
- Add a false argument to the continueChat method call in multiple instances to specify a new behavior or flag
- Ensure all grok.Chat and history.continueChat function calls within TestChatSummarization consistently include this new false parameter

Import go-shlex, add edit flag, implement file editing.

summary of diff --git  a/v3/core/cli.go b/v3/core/cli.go

- Import `github.com/anmitsu/go-shlex` in `v3/core/cli.go`
- Add `Edit` boolean field with a short flag `-e` and help text to `cmdChat` struct
- Implement feature to open the chat file in `GROKKER_EDITOR` for editing if the `-e` flag is set
- Pass `edit` flag to `grok.Chat` function call to possibly incorporate editing logic
- Define `EditFile` function to facilitate opening and editing the chat file in the specified editor
- Ensure `EditFile` appends a `### USER` heading to the chat file if it's not present at the end before opening in editor
- Use `shlex.Split` to correctly split the editor command allowing for commands with spaces

Add print statement for temp directory path in TestCliChat

summary of diff --git  a/v3/core/cli_test.go b/v3/core/cli_test.go

- Add print statement to output temporary directory path in TestCliChat function

Update version in grokker.go to 3.0.12

summary of diff --git  a/v3/core/grokker.go b/v3/core/grokker.go

- Update version from 3.0.11 to 3.0.12 in grokker.go

Uncomment debug line to print messages with Debug, Spprint

summary of diff --git  a/v3/core/openai.go b/v3/core/openai.go

- Uncomment the debug line to print message contents using `Debug` function and `Spprint` for formatting

Add go-shlex, assert/v2, and repr as dependencies

summary of diff --git  a/v3/go.mod b/v3/go.mod

- Add `github.com/anmitsu/go-shlex` dependency with version `v0.0.0-20200514113438-38f4b401e2be`
- Include `github.com/alecthomas/assert/v2` as an indirect dependency with version `v2.3.0`
- Include `github.com/alecthomas/repr` as an indirect dependency with version `v0.2.0`

Update assert and repr, add go-shlex with go.mod files

summary of diff --git  a/v3/go.sum b/v3/go.sum

- Update github.com/alecthomas/assert from v2.1.0 to v2.3.0
- Add go.mod for github.com/alecthomas/assert v2.3.0
- Update github.com/alecthomas/repr from v0.1.0 to v0.2.0
- Add go.mod for github.com/alecthomas/repr v0.2.0
- Add github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
- Add go.mod for github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be

v3/core/api.go

@@ -91,7 +91,7 @@ func (g *GrokkerInternal) ForgetDocument(path string) (err error) {
 
 // Chat uses the given sysmsg and prompt along with context from the
 // knowledge base and message history file to generate a response.
-func (g *GrokkerInternal) Chat(sysmsg, prompt, fileName string, level util.ContextLevel, infiles []string, outfiles []FileLang, extract, promptTokenLimit int, extractToStdout, addToDb bool) (resp string, err error) {
+func (g *GrokkerInternal) Chat(sysmsg, prompt, fileName string, level util.ContextLevel, infiles []string, outfiles []FileLang, extract, promptTokenLimit int, extractToStdout, addToDb, edit bool) (resp string, err error) {
 	defer Return(&err)
 	// open the message history file
 	history, err := g.OpenChatHistory(sysmsg, fileName)
@@ -107,7 +107,7 @@ func (g *GrokkerInternal) Chat(sysmsg, prompt, fileName string, level util.Conte
 		return
 	}
 	// get response
-	resp, _, err = history.continueChat(prompt, level, infiles, outfiles, promptTokenLimit)
+	resp, _, err = history.continueChat(prompt, level, infiles, outfiles, promptTokenLimit, edit)
 	Ck(err)
 	return
 }

v3/core/chat.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 
 	. "github.com/stevegt/goadapt"
 	"github.com/stevegt/grokker/v3/util"
@@ -96,7 +97,7 @@ func (g *GrokkerInternal) OpenChatHistory(sysmsg, relPath string) (history *Chat
 // continueChat continues a chat history.  The debug map contains
 // interesting statistics about the process, for testing and debugging
 // purposes.
-func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel, infiles []string, outfiles []FileLang, promptTokenLimit int) (resp string, debug map[string]int, err error) {
+func (history *ChatHistory) continueChat(prompt string, contextLevel util.ContextLevel, infiles []string, outfiles []FileLang, promptTokenLimit int, edit bool) (resp string, debug map[string]int, err error) {
 	defer Return(&err)
 	g := history.g
 
@@ -107,7 +108,7 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 	}
 
 	Debug("continueChat: prompt len=%d", len(prompt))
-	Debug("continueChat: context level=%s", level)
+	Debug("continueChat: context level=%s", contextLevel)
 
 	// create a temporary slice of messages to work with
 	var msgs []ChatMsg
@@ -116,7 +117,7 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 	getContext := false
 	appendMsgs := false
 	var files []string
-	switch level {
+	switch contextLevel {
 	case util.ContextNone:
 		// no context
 	case util.ContextRecent:
@@ -135,7 +136,7 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 		getContext = true
 		appendMsgs = true
 	default:
-		Assert(false, "invalid context level: %s", level)
+		Assert(false, "invalid context level: %s", contextLevel)
 	}
 
 	if getContext {
@@ -161,8 +162,14 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 		msgs = append(msgs, history.msgs...)
 	}
 
-	// append the prompt to the context
-	msgs = append(msgs, ChatMsg{Role: "USER", Txt: prompt})
+	if edit {
+		// get the prompt from the most recent message
+		Assert(len(prompt) == 0, "edit mode expects an empty prompt")
+		prompt = history.msgs[len(history.msgs)-1].Txt
+	} else {
+		// append the prompt to the context
+		msgs = append(msgs, ChatMsg{Role: "USER", Txt: prompt})
+	}
 
 	peakCount, err := history.tokenCount(msgs)
 	Ck(err)
@@ -175,7 +182,7 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 	if promptTokenLimit > 0 {
 		maxTokens = promptTokenLimit
 	}
-	msgs, err = history.summarize(prompt, msgs, maxTokens, level)
+	msgs, err = history.summarize(prompt, msgs, maxTokens, contextLevel)
 	Ck(err)
 
 	finalCount, err := history.tokenCount(msgs)
@@ -203,7 +210,11 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 
 	// generate the response
 	Fpf(os.Stderr, "Sending %d tokens to OpenAI...\n", finalCount)
+	Debug("continueChat: sysmsg len=%d", len(sysmsg))
+	Debug("continueChat: msgs len=%d", len(msgs))
 	resp, err = g.completeChat(sysmsg, msgs)
+	Ck(err)
+	Debug("continueChat: resp len=%d", len(resp))
 
 	// append the prompt and response to the stored messages
 	history.msgs = append(history.msgs, ChatMsg{Role: "USER", Txt: prompt})
@@ -217,7 +228,7 @@ func (history *ChatHistory) continueChat(prompt string, level util.ContextLevel,
 
 // summarize summarizes a chat history until it is within
 // maxTokens.  It always leaves the last message intact.
-func (history *ChatHistory) summarize(prompt string, msgs []ChatMsg, maxTokens int, level util.ContextLevel) (summarized []ChatMsg, err error) {
+func (history *ChatHistory) summarize(prompt string, msgs []ChatMsg, maxTokens int, contextLevel util.ContextLevel) (summarized []ChatMsg, err error) {
 	defer Return(&err)
 	g := history.g
 
@@ -239,7 +250,7 @@ func (history *ChatHistory) summarize(prompt string, msgs []ChatMsg, maxTokens i
 	}
 
 	var sysmsg string
-	if level > util.ContextRecent {
+	if contextLevel > util.ContextRecent {
 		Fpf(os.Stderr, "Summarizing %d tokens...\n", msgsCount)
 		// generate a sysmsg that includes a short summary of the prompt
 		topic, err := g.Msg("Summarize the topic in one sentence.", prompt)
@@ -325,7 +336,7 @@ func (history *ChatHistory) summarize(prompt string, msgs []ChatMsg, maxTokens i
 	// second half of the messages
 	msgs[secondHalfI] = msg2
 
-	if level > util.ContextRecent {
+	if contextLevel > util.ContextRecent {
 		for len(summarized) == 0 {
 			// summarize the first half of the messages by converting them to
 			// a text format that GPT-4 can understand, sending them to GPT-4,
@@ -345,7 +356,7 @@ func (history *ChatHistory) summarize(prompt string, msgs []ChatMsg, maxTokens i
 	summarized = append(summarized, msgs[secondHalfI:]...)
 
 	// recurse
-	return history.summarize(prompt, summarized, maxTokens, level)
+	return history.summarize(prompt, summarized, maxTokens, contextLevel)
 }
 
 // chat2txt returns the given history messages as a text string.
@@ -385,8 +396,8 @@ func (history *ChatHistory) parseChat(txt string) (msgs []ChatMsg) {
 			continue
 		}
 		if msg == nil {
-			// we are in some sort of preamble -- credit it to the AI
-			msg = &ChatMsg{Role: "AI"}
+			// we are in some sort of preamble -- credit it to the user
+			msg = &ChatMsg{Role: "USER"}
 		}
 		// if we get here, we are in the middle of a message
 		msg.Txt += line + "\n"
@@ -401,6 +412,7 @@ func (history *ChatHistory) parseChat(txt string) (msgs []ChatMsg) {
 // fixRole fixes the role in a chat history file.  This might be
 // necessary if GPT-4 changes the role names.
 func (history *ChatHistory) fixRole(role string) (fixed string) {
+	role = strings.TrimSpace(role)
 	role = strings.ToUpper(role)
 	switch role {
 	case "USER":
@@ -437,28 +449,37 @@ func (history *ChatHistory) Save(addToDb bool) (err error) {
 	// close the temp file
 	err = fh.Close()
 	Ck(err)
-	// replace the chat history file with the temp file
 	Assert(history.relPath != "", "relPath is required")
-	// path := filepath.Join(history.g.Root, history.relPath)
 	path := history.relPath
-	// move the chat history file to a backup file
+	// move the existing chat history file to a backup file
 	var backup string
 	_, err = os.Stat(path)
 	if err == nil {
 		// file exists
-		backup = path + ".bak"
+		// get a timestamp string
+		ts := time.Now().Format("20060102-150405")
+		// split on dots
+		parts := strings.Split(path, ".")
+		// insert the timestamp before the file extension
+		parts = append(parts[:len(parts)-1], ts, parts[len(parts)-1])
+		// join the parts
+		backup = strings.Join(parts, ".")
 		err = os.Rename(path, backup)
 		Ck(err)
 	}
 	// move the temp file to the chat history file
 	err = os.Rename(fh.Name(), path)
 	Ck(err)
-	// remove the backup file
-	_, err = os.Stat(backup)
-	if err == nil {
-		// file exists
-		err = os.Remove(backup)
-		Ck(err)
+
+	// XXX this should be a flag
+	if false {
+		// remove the backup file
+		_, err = os.Stat(backup)
+		if err == nil {
+			// file exists
+			err = os.Remove(backup)
+			Ck(err)
+		}
 	}
 
 	if addToDb {

v3/core/chat_test.go

@@ -69,7 +69,7 @@ func TestChatSummarization(t *testing.T) {
 	defer grok.Save()
 
 	// start a chat by mentioning something not in GPT-4's global context
-	res, err := grok.Chat("", "Pretend a blue widget has a red center.", "chat1", util.ContextAll, nil, nil, 0, 0, false, true)
+	res, err := grok.Chat("", "Pretend a blue widget has a red center.", "chat1", util.ContextAll, nil, nil, 0, 0, false, true, false)
 	Tassert(t, err == nil, "error starting chat: %v", err)
 	// check that the response contains the expected output
 	match = cimatch(res, "red")
@@ -87,7 +87,7 @@ func TestChatSummarization(t *testing.T) {
 	Pl("testing large context")
 	history, err := grok.OpenChatHistory("", "chat1")
 	Tassert(t, err == nil, "error opening chat history: %v", err)
-	res, debug, err := history.continueChat("Talk about complex systems.", util.ContextAll, nil, nil, 0)
+	res, debug, err := history.continueChat("Talk about complex systems.", util.ContextAll, nil, nil, 0, false)
 	Tassert(t, err == nil, "error continuing chat: %v", err)
 	err = history.Save(true)
 	Tassert(t, err == nil, "error saving chat history: %v", err)
@@ -99,7 +99,7 @@ func TestChatSummarization(t *testing.T) {
 		ctx, err := grok.Context("system", 3000, false, false)
 		Tassert(t, err == nil, "error getting context: %v", err)
 		prompt := Spf("%s\n\nDiscuss complex systems more.", ctx)
-		res, debug, err = history.continueChat(prompt, util.ContextAll, nil, nil, 0)
+		res, debug, err = history.continueChat(prompt, util.ContextAll, nil, nil, 0, false)
 		Tassert(t, err == nil, "error continuing chat: %v", err)
 		err = history.Save(true)
 		Ck(err)
@@ -119,7 +119,7 @@ func TestChatSummarization(t *testing.T) {
 	Tassert(t, ok, "peak token count never exceeded token limit: %v", debug)
 
 	// check that we still remember the blue widget
-	res, err = grok.Chat("", "What color is the center of the blue widget?", "chat1", util.ContextAll, nil, nil, 0, 0, false, true)
+	res, err = grok.Chat("", "What color is the center of the blue widget?", "chat1", util.ContextAll, nil, nil, 0, 0, false, true, false)
 	match = cimatch(res, "red")
 	Tassert(t, match, "CLI did not return expected output: %s", res)
 
@@ -140,7 +140,7 @@ func TestChatSummarization(t *testing.T) {
 			ctx, err := grok.Context("system", 3000, false, false)
 			Tassert(t, err == nil, "error getting context: %v", err)
 			prompt := Spf("Discuss this topic more:\n%s\n\n", ctx)
-			res, _, err = history.continueChat(prompt, util.ContextAll, nil, nil, 0)
+			res, _, err = history.continueChat(prompt, util.ContextAll, nil, nil, 0, false)
 			Tassert(t, err == nil, "error continuing chat: %v", err)
 			err = history.Save(true)
 			Ck(err)
@@ -161,7 +161,7 @@ func TestChatSummarization(t *testing.T) {
 	Tassert(t, ok, "chat1 file never exceeded token limit: %v", debug)
 
 	// check that we still remember the blue widget
-	res, err = grok.Chat("", "What color is the center of the blue widget?", "chat1", util.ContextAll, nil, nil, 0, 0, false, true)
+	res, err = grok.Chat("", "What color is the center of the blue widget?", "chat1", util.ContextAll, nil, nil, 0, 0, false, true, false)
 	match = cimatch(res, "red")
 	Tassert(t, match, "CLI did not return expected output: %s", res)
 

v3/core/cli.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/alecthomas/kong"
+	"github.com/anmitsu/go-shlex"
 	"github.com/gofrs/flock"
 	. "github.com/stevegt/goadapt"
 	"github.com/stevegt/grokker/v3/util"
@@ -111,6 +112,7 @@ type cmdChat struct {
 	ContextRepo      bool     `short:"C" help:"Add context from the entire grokker repository (includes chat file)."`
 	ContextChat      bool     `short:"c" help:"Add context from the entire chat file."`
 	ContextNone      bool     `short:"N" help:"Do not add any context."`
+	Edit             bool     `short:"e" help:"Open the chat file in GROKKER_EDITOR for editing."`
 	Prompt           string   `short:"m" help:"Prompt message to use instead of stdin."`
 	InputFiles       []string `short:"i" type:"string" help:"Input files to be provided in the prompt."`
 	OutputFiles      []string `short:"o" type:"string" help:"Output files to be created or overwritten."`
@@ -377,9 +379,14 @@ func Cli(args []string, config *CliConfig) (rc int, err error) {
 		}
 		var prompt string
 		extract := cli.Chat.Extract
+		edit := cli.Chat.Edit
 		if extract < 1 {
 			if cli.Chat.Prompt != "" {
 				prompt = cli.Chat.Prompt
+			} else if edit {
+				// open the chat file in the editor
+				err = EditFile(cli.Chat.ChatFile)
+				Ck(err)
 			} else {
 				// get text from stdin and print the response
 				buf, err := ioutil.ReadAll(config.Stdin)
@@ -423,7 +430,7 @@ func Cli(args []string, config *CliConfig) (rc int, err error) {
 			}
 		}
 		// get the response
-		outtxt, err := grok.Chat(cli.Chat.Sysmsg, prompt, cli.Chat.ChatFile, level, infiles, outfiles, extract, cli.Chat.PromptTokenLimit, cli.Chat.ExtractToStdout, !cli.Chat.NoAddToDb)
+		outtxt, err := grok.Chat(cli.Chat.Sysmsg, prompt, cli.Chat.ChatFile, level, infiles, outfiles, extract, cli.Chat.PromptTokenLimit, cli.Chat.ExtractToStdout, !cli.Chat.NoAddToDb, edit)
 		Ck(err)
 		Pl(outtxt)
 		// save the grok file
@@ -696,3 +703,62 @@ func commitMessage(grok *GrokkerInternal, args ...string) (summary string, err e
 
 	return
 }
+
+// EditFile opens the chat file in the editor
+func EditFile(fn string) (err error) {
+	defer Return(&err)
+
+	// first append a ### USER heading to the file if it doesn't exist
+	_, err = os.Stat(fn)
+	if err != nil {
+		// file does not exist
+		err = ioutil.WriteFile(fn, []byte("### USER\n"), 0644)
+		Ck(err)
+	} else {
+		// file exists -- check for the heading at the end of the file
+		buf, err := ioutil.ReadFile(fn)
+		Ck(err)
+		lines := strings.Split(string(buf), "\n")
+		for i := len(lines) - 1; i >= 0; i-- {
+			if lines[i] == "### USER" {
+				// heading exists
+				break
+			}
+			if lines[i] == "" {
+				// empty line
+				continue
+			}
+			// heading does not exist -- append it
+			fh, err := os.OpenFile(fn, os.O_APPEND|os.O_WRONLY, 0644)
+			Ck(err)
+			defer fh.Close()
+			_, err = fh.WriteString("\n\n### USER\n\n")
+			Ck(err)
+			break
+		}
+	}
+
+	// open the file in the editor
+	editor := os.Getenv("GROKKER_EDITOR")
+	if editor == "" {
+		editor = "vi +"
+	}
+	// use shlex to split the editor command
+	cmdline, err := shlex.Split(editor, true)
+	Ck(err)
+	editor = cmdline[0]
+	var args []string
+	if len(cmdline) == 1 {
+		args = []string{fn}
+	} else {
+		args = append(cmdline[1:], fn)
+	}
+	cmd := exec.Command(editor, args...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	Ck(err)
+
+	return
+}

v3/core/cli_test.go

@@ -531,6 +531,7 @@ func TestCliChat(t *testing.T) {
 	// create a temporary directory
 	dir, err := os.MkdirTemp("", "grokker")
 	Ck(err)
+	Pf("dir: %s\n", dir)
 	// defer os.RemoveAll(dir)
 	// cd into the temporary directory
 	cd(t, dir)

v3/core/grokker.go

@@ -50,7 +50,7 @@ import (
 const (
 	// See the "Semantic Versioning" section of the README for
 	// information on API and db stability and versioning.
-	version = "3.0.11"
+	version = "3.0.12"
 )
 
 type GrokkerInternal struct {

v3/core/openai.go

@@ -73,7 +73,7 @@ func (g *GrokkerInternal) createEmbeddings(texts []string) (embeddings [][]float
 func (g *GrokkerInternal) completeChat(sysmsg string, msgs []ChatMsg) (response string, err error) {
 	defer Return(&err)
 
-	// Debug("msgs: %s", Spprint(msgs))
+	Debug("msgs: %s", Spprint(msgs))
 
 	omsgs := []oai.ChatCompletionMessage{
 		{