Adding Sound Effects (#88)

* path fix issue

* proof of concept sound effects

* adding boxing bell ring

* boxing bell ring

* refactor sounds to work with state machine, writing becomes typing

* also adding specific openai model

assets/animator.js

@@ -18,9 +18,9 @@ class Animator {
       this.img.src = '/matt-pending.png';
       this.img.style.animation = 'talking 0.25s infinite';
     }
-    if (this.state === 'writing') {
+    if (this.state === 'typing') {
       this.img.src = '/matt-typing.png';
-      this.img.style.animation = 'writing 0.25s infinite';
+      this.img.style.animation = 'typing 0.25s infinite';
     }
     if (this.state === 'thinking') {
       this.img.src = '/matt-thinking.png';

assets/index.html

@@ -62,8 +62,8 @@
         }
       }
 
-      /* Writing animation using vertical transform */
-      @keyframes writing {
+      /* Typing animation using vertical transform */
+      @keyframes typing {
         0% {
           transform: translateY(0px);
         }

config.sample.js

@@ -6,6 +6,7 @@ module.exports = {
   tts: 'piper', // say, coqui, piper, elevenlabs, elevenlabsSync, playht
   showSubtitles: false, // show subtitles of what is being said
   ollamaModel: 'llama2:70b', // model to use with ollama provider (only used if model is ollama)
+  openAImodel: 'gpt-4-turbo', // replace with fine-tuned model (only used if model is openai)
   replicateApiToken: 'your-replicate-api-token-here',
   openAIApiToken: 'your-openai-api-token-here',
   geminiApiToken: 'your-gemini-api-token-here',

src/commands/listenToHuman.js

@@ -64,13 +64,15 @@ class ListenToHumanCommand extends Command {
 
   async stopListening() {
     this.recording.stop();
+    const agent = getAgent();
+    agent.webserver.sendStatus('thinking');
+
     // this.statusBarItem.text = '$(mute) Not listening';
     setStatusbarText('$(loading~spin) Transcribing...');
     let output = await transcribe(
       path.join(__dirname, '../../', this.filename)
     );
     this.listening = false;
-    const agent = getAgent();
     const startingPrompt = output.text;
     agent.prompt(startingPrompt);
   }

src/lib/agent/Agent.js

@@ -10,7 +10,7 @@ const { query: queryForContext } = require('../../util/semantic-retrieval');
 // States
 const IdleState = require('./states/IdleState');
 const PromptingState = require('./states/promptingState');
-const WritingState = require('./states/writingState');
+const TypingState = require('./states/typingState');
 const ThinkingState = require('./states/thinkingState');
 const TalkingState = require('./states/talkingState');
 
@@ -21,7 +21,7 @@ class Agent extends StateMachine {
     // Registering states
     this.addState(new IdleState(this));
     this.addState(new PromptingState(this));
-    this.addState(new WritingState(this));
+    this.addState(new TypingState(this));
     this.addState(new ThinkingState(this));
     this.addState(new TalkingState(this));
 
@@ -271,7 +271,7 @@ class Agent extends StateMachine {
         .replace(/\r\n/g, '\n');
       const diffs = Diff.diffWordsWithSpace(currentEditorCode, step.content);
 
-      this.goToState('writing');
+      this.goToState('typing');
       await applyDiffs(editor, diffs);
     } else if (step.type === 'SPEAK') {
       let content = step.content.trim();

src/lib/agent/providers/openaiProvider.js

@@ -12,7 +12,7 @@ class OpenAIProvider extends ModelProvider {
   async query(prompt) {
     this.messageHistory.push({ role: 'user', content: prompt });
     const response = await this.openai.chat.completions.create({
-      model: 'gpt-4-turbo-preview',
+      model: config.openAImodel,
       messages: [
         {
           role: 'system',

src/lib/agent/states/thinkingState.js

@@ -1,10 +1,12 @@
 const { setStatusbarText } = require('../../../extension');
 const State = require('../../../util/statemachine/State');
+const { playSound } = require('../../../util/sound-effects');
 
 class ThinkingState extends State {
   onActivate() {
     setStatusbarText('$(loading~spin) waiting for chunks...');
     this.stateMachine.webserver.sendStatus('thinking'); // state machine is our agent in this case
+    playSound('thinking');
   }
 }
 

src/lib/agent/states/typingState.js

@@ -0,0 +1,13 @@
+const { setStatusbarText } = require('../../../extension');
+const State = require('../../../util/statemachine/State');
+const { playSound } = require('../../../util/sound-effects');
+
+class TypingState extends State {
+  onActivate() {
+    setStatusbarText('$(record-keys) Typing code...');
+    this.stateMachine.webserver.sendStatus('typing'); // state machine is our agent in this case
+    playSound('typing');
+  }
+}
+
+module.exports = TypingState;

src/util/sound-effects.js

@@ -0,0 +1,39 @@
+const player = require('play-sound')();
+const { join } = require('path');
+
+let sounds = {
+  thinking: join(
+    __dirname,
+    '../../sounds/ES_Computer Tone 3 - SFX Producer.mp3'
+  ),
+  typing: join(
+    __dirname,
+    '../../sounds/ES_Keyboard Typing 3 - SFX Producer.mp3'
+  ),
+  challenge: join(
+    __dirname,
+    '../../sounds/ES_Boxing Bell Ring 2 - SFX Producer.mp3'
+  ),
+};
+
+function playSound(sound) {
+  const filepath = sounds[sound];
+  return new Promise((resolve, reject) => {
+    player.play(filepath, (err) => {
+      if (err) {
+        console.error('Failed to play:', err);
+        reject(err);
+      } else {
+        // console.log('Audio playback finished.');
+        // Hanging on this, not sure why
+        // fs.unlinkSync(tempFilePath);
+        resolve();
+      }
+    });
+  });
+}
+
+module.exports = {
+  // sounds,
+  playSound,
+};