Merge branch 'develop'

.github/workflows/release.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Build with PyInstaller
         run: |
           .\venv\Scripts\Activate
-          pyinstaller main.py --name WingmanAiCore --noconfirm --icon assets/wingman-ai.ico --paths venv/Lib/site-packages --add-data "venv/Lib/site-packages/azure/cognitiveservices/speech;azure/cognitiveservices/speech" --add-data "assets;assets" --add-data "services;services" --add-data "wingmen;wingmen" --add-data "configs;configs" --add-data "audio_samples;audio_samples" --add-data "LICENSE;."
+          pyinstaller main.py --name WingmanAiCore --noconfirm --icon assets/wingman-ai.ico --paths venv/Lib/site-packages --add-data "venv/Lib/site-packages/azure/cognitiveservices/speech;azure/cognitiveservices/speech" --add-data "assets;assets" --add-data "services;services" --add-data "wingmen;wingmen" --add-data "templates;templates" --add-data "audio_samples;audio_samples" --add-data "LICENSE;."
 
       - name: Upload Windows Exe Artifact
         uses: actions/upload-artifact@v4

.gitignore

@@ -178,3 +178,4 @@ configs/system/secrets.yaml
 .frontmatter/database/taxonomyDb.json
 
 frontmatter.json
+skills/**/dependencies

.vscode/launch.json

@@ -6,18 +6,18 @@
     "configurations": [
         {
             "name": "Python: Main",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "main.py",
             "console": "integratedTerminal",
             "justMyCode": true,
-            // "args": [
-            //     "--host",
-            //     "127.0.0.1",
-            //     "--port",
-            //     "9000",
-            //     "--sidecar"
-            // ]
+            "args": [
+                "--host",
+                "0.0.0.0",
+                //"--port",
+                //"10001",
+                // "--sidecar"
+            ]
         }
     ]
 }

api/commands.py

@@ -43,6 +43,7 @@ class LogCommand(WebSocketCommandModel):
     source_name: Optional[str] = None
     source: LogSource = "system"
     tag: Optional[CommandTag] = None
+    skill_name: Optional[str] = None
 
 
 class PromptSecretCommand(WebSocketCommandModel):

api/enums.py

@@ -41,6 +41,13 @@ class CommandTag(Enum):
     UNAUTHORIZED = "unauthorized"
 
 
+class CustomPropertyType(Enum):
+    STRING = "string"
+    NUMBER = "number"
+    BOOLEAN = "boolean"
+    SINGLE_SELECT = "single_select"
+
+
 class AzureApiVersion(Enum):
     A2023_12_01_PREVIEW = "2023-12-01-preview"
     A2024_02_15_PREVIEW = "2024-02-15-preview"
@@ -64,28 +71,33 @@ class TtsVoiceGender(Enum):
 
 
 class OpenAiModel(Enum):
-    GPT_35_TURBO = "gpt-3.5-turbo"
-    GPT_4_TURBO_PREVIEW = "gpt-4-turbo-preview"
+    """https://platform.openai.com/docs/models/overview"""
+
+    GPT_4O = "gpt-4o"
 
 
 class MistralModel(Enum):
+    """https://docs.mistral.ai/getting-started/models/"""
+
     MISTRAL_7B = "open-mistral-7b"
     OPEN_MIXTRAL_8X7B = "open-mixtral-8x7b"
+    OPEN_MIXTRAL_8X22B = "open-mixtral-8x22b"
     MISTRAL_SMALL = "mistral-small-latest"
     MISTRAL_MEDIUM = "mistral-medium-latest"
     MISTRAL_LARGE = "mistral-large-latest"
 
 
-class LlamaModel(Enum):
-    LLAMA3_8B = "llama3-8b"
+class GroqModel(Enum):
+    """https://console.groq.com/docs/models"""
+
     LLAMA3_8B_8192 = "llama3-8b-8192"
-    LLAMA3_70B = "llama3-70b"
     LLAMA3_70B_8192 = "llama3-70b-8192"
+    MIXTRAL_8X7B = "mixtral-8x7b-32768"
+    GEMMA_7B = "gemma-7b-it"
 
 
 class WingmanProAzureDeployment(Enum):
-    GPT_35_TURBO = "gpt-35-turbo"
-    GPT_4_TURBO = "gpt-4-turbo"
+    GPT_4O = "gpt-4o"
     MISTRAL_LARGE = "mistral-large-latest"
     LLAMA3_8B = "llama3-8b"
     LLAMA3_70B = "llama3-70b"
@@ -136,15 +148,19 @@ class VoiceActivationSttProvider(Enum):
 class ConversationProvider(Enum):
     OPENAI = "openai"
     MISTRAL = "mistral"
-    LLAMA = "llama"
+    GROQ = "groq"
+    OPENROUTER = "openrouter"
+    LOCAL_LLM = "local_llm"
     AZURE = "azure"
     WINGMAN_PRO = "wingman_pro"
 
 
 class SummarizeProvider(Enum):
     OPENAI = "openai"
     MISTRAL = "mistral"
-    LLAMA = "llama"
+    GROQ = "groq"
+    OPENROUTER = "openrouter"
+    LOCAL_LLM = "local_llm"
     AZURE = "azure"
     WINGMAN_PRO = "wingman_pro"
 
@@ -199,6 +215,10 @@ class CommandTagEnumModel(BaseEnumModel):
     command_tag: CommandTag
 
 
+class CustomPropertyTypeEnumModel(BaseEnumModel):
+    property_type: CustomPropertyType
+
+
 class AzureApiVersionEnumModel(BaseEnumModel):
     api_version: AzureApiVersion
 
@@ -223,8 +243,8 @@ class MistralModelEnumModel(BaseEnumModel):
     model: MistralModel
 
 
-class LlamaModelEnumModel(BaseEnumModel):
-    model: LlamaModel
+class GroqModelEnumModel(BaseEnumModel):
+    model: GroqModel
 
 
 class WingmanProAzureDeploymentEnumModel(BaseEnumModel):
@@ -285,13 +305,14 @@ class WingmanProTtsProviderModel(BaseEnumModel):
     "ToastType": ToastTypeEnumModel,
     "WingmanInitializationErrorType": WingmanInitializationErrorTypeModel,
     "CommandTag": CommandTagEnumModel,
+    "CustomPropertyType": CustomPropertyTypeEnumModel,
     "AzureApiVersion": AzureApiVersionEnumModel,
     "AzureRegion": AzureRegionEnumModel,
     "ElevenlabsModel": ElevenlabsModelEnumModel,
     "TtsVoiceGender": TtsVoiceGenderEnumModel,
     "OpenAiModel": OpenAiModelEnumModel,
     "MistralModel": MistralModelEnumModel,
-    "LLamaModel": LlamaModelEnumModel,
+    "GroqModel": GroqModelEnumModel,
     "WingmanProAzureDeployment": WingmanProAzureDeploymentEnumModel,
     "OpenAiTtsVoice": OpenAiTtsVoiceEnumModel,
     "SoundEffect": SoundEffectEnumModel,

api/interface.py

@@ -9,8 +9,9 @@
     AzureApiVersion,
     AzureRegion,
     ConversationProvider,
-    LlamaModel,
+    GroqModel,
     MistralModel,
+    CustomPropertyType,
     TtsVoiceGender,
     ElevenlabsModel,
     OpenAiModel,
@@ -243,21 +244,12 @@ class XVASynthTtsConfig(BaseModel):
 
 
 class OpenAiConfig(BaseModel):
-    context: Optional[str] = None
-    """The "context" for the wingman. Here's where you can tell the AI how to behave.
-    This is probably what you want to play around with the most.
-    """
-
     conversation_model: OpenAiModel
     """ The model to use for conversations aka "chit-chat" and for function calls.
-    gpt-4 is more powerful than gpt-3.5 but also 10x more expensive.
-    gpt-3.5 is the default and should be enough for most use cases.
-    If something is not working as expected, you might want to test it with gpt-4.
     """
 
     summarize_model: OpenAiModel
-    """ This model summarizes function responses, like API call responses etc.
-    In most cases gpt-3.5 should be enough.
+    """ This model summarizes function responses, like API call responses etc. This can be a less capable model.
     """
 
     tts_voice: OpenAiTtsVoice
@@ -276,15 +268,35 @@ class OpenAiConfig(BaseModel):
     """If you have an organization key, you can set it here."""
 
 
+class PromptConfig(BaseModel):
+    system_prompt: str
+    """The read-only "context template" for the Wingman. Contains variables that will be replaced by the user (backstory) and/or skills."""
+
+    backstory: Optional[str] = None
+    """The backstory of the Wingman. Edit this to control how your Wingman should behave."""
+
+
 class MistralConfig(BaseModel):
     conversation_model: MistralModel
     summarize_model: MistralModel
     endpoint: str
 
 
-class LlamaConfig(BaseModel):
-    conversation_model: LlamaModel
-    summarize_model: LlamaModel
+class GroqConfig(BaseModel):
+    conversation_model: GroqModel
+    summarize_model: GroqModel
+    endpoint: str
+
+
+class OpenRouterConfig(BaseModel):
+    conversation_model: str
+    summarize_model: str
+    endpoint: str
+
+
+class LocalLlmConfig(BaseModel):
+    conversation_model: Optional[str] = None
+    summarize_model: Optional[str] = None
     endpoint: str
 
 
@@ -335,9 +347,6 @@ class FeaturesConfig(BaseModel):
     Note that the other providers may have additional config blocks. These are only used if the provider is set here.
     """
 
-    debug_mode: bool
-    """If enabled, the Wingman will skip executing any keypresses. It will also print more debug messages and benchmark results."""
-
     tts_provider: TtsProvider
     stt_provider: SttProvider
     conversation_provider: ConversationProvider
@@ -426,32 +435,72 @@ class CustomWingmanClassConfig(BaseModel):
     """The name of your class within your file/module."""
 
 
-class NestedConfig(BaseModel):
-    sound: SoundConfig
-    features: FeaturesConfig
-    openai: OpenAiConfig
-    mistral: MistralConfig
-    llama: LlamaConfig
-    edge_tts: EdgeTtsConfig
-    elevenlabs: ElevenlabsConfig
-    azure: AzureConfig
-    xvasynth: XVASynthTtsConfig
-    whispercpp: WhispercppSttConfig
-    wingman_pro: WingmanProConfig
-    commands: Optional[list[CommandConfig]] = None
+class LabelValuePair(BaseModel):
+    label: str
+    value: str | int | float | bool
 
 
 class CustomWingmanProperty(BaseModel):
     id: str
     """The name of the property. Has to be unique"""
     name: str
     """The "friendly" name of the property, displayed in the UI."""
-    value: str
+    value: str | int | float | bool | None
     """The value of the property"""
     hint: Optional[str] = None
     """A hint for the user, displayed in the UI."""
     required: Optional[bool] = False
     """Marks the property as required in the UI."""
+    property_type: Optional[CustomPropertyType] = CustomPropertyType.STRING
+    """Determines the type of the property and which controls to render in the UI."""
+    options: Optional[list[LabelValuePair]] = None
+    """If property_type is set to 'single_select', you can provide options here."""
+
+
+class LocalizedMetadata(BaseModel):
+    en: str
+    de: Optional[str] = None
+
+
+class SkillExample(BaseModel):
+    question: LocalizedMetadata
+    answer: LocalizedMetadata
+
+
+class SkillConfig(CustomWingmanClassConfig):
+    prompt: Optional[str] = None
+    """An additional prompt that extends the system prompt of the Wingman."""
+    custom_properties: Optional[list[CustomWingmanProperty]] = None
+    """You can add custom properties here to use in your custom skill class."""
+    description: Optional[LocalizedMetadata] = None
+    hint: Optional[LocalizedMetadata] = None
+    examples: Optional[list[SkillExample]] = None
+    commands: Optional[list[CommandConfig]] = None
+
+
+class SkillBase(BaseModel):
+    name: str
+    config: SkillConfig
+    logo: Optional[Annotated[str, Base64Str]] = None
+
+
+class NestedConfig(BaseModel):
+    prompts: PromptConfig
+    sound: SoundConfig
+    features: FeaturesConfig
+    openai: OpenAiConfig
+    mistral: MistralConfig
+    groq: GroqConfig
+    openrouter: OpenRouterConfig
+    local_llm: LocalLlmConfig
+    edge_tts: EdgeTtsConfig
+    elevenlabs: ElevenlabsConfig
+    azure: AzureConfig
+    xvasynth: XVASynthTtsConfig
+    whispercpp: WhispercppSttConfig
+    wingman_pro: WingmanProConfig
+    commands: Optional[list[CommandConfig]] = None
+    skills: Optional[list[SkillConfig]] = None
 
 
 class WingmanConfig(NestedConfig):
@@ -513,3 +562,4 @@ class SettingsConfig(BaseModel):
     audio: Optional[AudioSettings] = None
     voice_activation: VoiceActivationSettings
     wingman_pro: WingmanProSettings
+    debug_mode: bool = False