From ac4f394b1f9fe9af246b43092218a062c4367d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B5=E3=81=81=E3=83=BC?= <47295014+ymuichiro@users.noreply.github.com> Date: Wed, 18 Sep 2024 05:06:38 +0900 Subject: [PATCH 1/3] Python: Fixed an issue where the schema property was missing when using the OpenAPI plugin with the get method. (#8502) Fixed an issue where the schema property was missing when using the OpenAPI plugin with the get method. ### Motivation and Context This Pull Request is based on Issue https://github.com/microsoft/semantic-kernel/issues/8423. When loading an openapi plugin that uses the get method, properties other than the "type" key in parameters.schema is missing. ### Description The parameters.schema includes properties other than the "type" key, such as "description", which is used for the llm to make judgments during function calling. Therefore, I modified it to include this information. [openapi schema](https://github.com/OAI/OpenAPI-Specification/blob/4e9d2b3ec859beef08309c414f895c73a793b958/schemas/v3.0/schema.yaml#L658) ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --------- Co-authored-by: Tao Chen Co-authored-by: Evan Mattson <35585003+moonbox3@users.noreply.github.com> --- .../models/rest_api_operation_parameter.py | 2 +- .../openapi_plugin/openapi_parser.py | 17 ++--- .../unit/connectors/openapi/openapi_todo.yaml | 57 +++++++++++++++++ .../connectors/openapi/test_openapi_parser.py | 62 ++++++++++++++++++- 4 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 python/tests/unit/connectors/openapi/openapi_todo.yaml diff --git a/python/semantic_kernel/connectors/openapi_plugin/models/rest_api_operation_parameter.py b/python/semantic_kernel/connectors/openapi_plugin/models/rest_api_operation_parameter.py index 761e390c9d4c..0f8745e08f2e 100644 --- a/python/semantic_kernel/connectors/openapi_plugin/models/rest_api_operation_parameter.py +++ b/python/semantic_kernel/connectors/openapi_plugin/models/rest_api_operation_parameter.py @@ -28,7 +28,7 @@ def __init__( description: str | None = None, is_required: bool = False, default_value: Any | None = None, - schema: str | None = None, + schema: str | dict | None = None, response: RestApiOperationExpectedResponse | None = None, ): """Initialize the RestApiOperationParameter.""" diff --git a/python/semantic_kernel/connectors/openapi_plugin/openapi_parser.py b/python/semantic_kernel/connectors/openapi_plugin/openapi_parser.py index 85f13a096908..984f120e837b 100644 --- a/python/semantic_kernel/connectors/openapi_plugin/openapi_parser.py +++ b/python/semantic_kernel/connectors/openapi_plugin/openapi_parser.py @@ -63,26 +63,27 @@ def _parse_parameters(self, parameters: list[dict[str, Any]]): """Parse the parameters from the OpenAPI document.""" result: list[RestApiOperationParameter] = [] for param in parameters: - name = param["name"] - type = param["schema"]["type"] + name: str = param["name"] if not param.get("in"): raise PluginInitializationError(f"Parameter {name} is missing 'in' field") + if param.get("content", None) is not None: + # The schema and content fields are mutually exclusive. + raise PluginInitializationError(f"Parameter {name} cannot have a 'content' field. Expected: schema.") location = RestApiOperationParameterLocation(param["in"]) - description = param.get("description", None) - is_required = param.get("required", False) + description: str = param.get("description", None) + is_required: bool = param.get("required", False) default_value = param.get("default", None) - schema = param.get("schema", None) - schema_type = schema.get("type", None) if schema else "string" + schema: dict[str, Any] | None = param.get("schema", None) result.append( RestApiOperationParameter( name=name, - type=type, + type=schema.get("type", "string") if schema else "string", location=location, description=description, is_required=is_required, default_value=default_value, - schema=schema_type, + schema=schema if schema else {"type": "string"}, ) ) return result diff --git a/python/tests/unit/connectors/openapi/openapi_todo.yaml b/python/tests/unit/connectors/openapi/openapi_todo.yaml new file mode 100644 index 000000000000..3afd713b809e --- /dev/null +++ b/python/tests/unit/connectors/openapi/openapi_todo.yaml @@ -0,0 +1,57 @@ +openapi: 3.0.0 +info: + title: Todo List API + version: 1.0.0 + description: API for managing todo lists +paths: + /list: + get: + summary: Get todo list + operationId: get_todo_list + description: get todo list from specific group + parameters: + - name: listName + in: query + required: true + description: todo list group name description + schema: + type: string + description: todo list group name + responses: + "200": + description: Successful response + content: + application/json: + schema: + type: array + items: + type: object + properties: + task: + type: string + listName: + type: string + + /add: + post: + summary: Add a task to a list + operationId: add_todo_list + description: add todo to specific group + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - task + properties: + task: + type: string + description: task name + listName: + type: string + description: task group name + responses: + "201": + description: Task added successfully diff --git a/python/tests/unit/connectors/openapi/test_openapi_parser.py b/python/tests/unit/connectors/openapi/test_openapi_parser.py index 71548537e30a..0e4d278a1667 100644 --- a/python/tests/unit/connectors/openapi/test_openapi_parser.py +++ b/python/tests/unit/connectors/openapi/test_openapi_parser.py @@ -1,10 +1,14 @@ # Copyright (c) Microsoft. All rights reserved. +import os import pytest -from semantic_kernel.connectors.openapi_plugin.openapi_manager import OpenApiParser +from semantic_kernel.connectors.openapi_plugin.openapi_manager import OpenApiParser, create_functions_from_openapi from semantic_kernel.exceptions.function_exceptions import PluginInitializationError +from semantic_kernel.functions import KernelFunctionFromMethod, KernelFunctionMetadata, KernelParameterMetadata + +current_dir = os.path.dirname(os.path.abspath(__file__)) def test_parse_parameters_missing_in_field(): @@ -14,6 +18,62 @@ def test_parse_parameters_missing_in_field(): parser._parse_parameters(parameters) +def test_parse_parameters_get_query(): + """Verify whether the get request query parameter can be successfully parsed""" + openapi_fcs: list[KernelFunctionFromMethod] = create_functions_from_openapi( + plugin_name="todo", + openapi_document_path=os.path.join(current_dir, "openapi_todo.yaml"), + execution_settings=None, + ) + + get_todo_list: list[KernelFunctionMetadata] = [ + f.metadata for f in openapi_fcs if f.metadata.name == "get_todo_list" + ] + + assert get_todo_list + + get_todo_params: list[KernelParameterMetadata] = get_todo_list[0].parameters + assert get_todo_params + assert get_todo_params[0].name == "listName" + assert get_todo_params[0].description == "todo list group name description" + assert get_todo_params[0].is_required + assert get_todo_params[0].schema_data + assert get_todo_params[0].schema_data.get("type") == "string" + assert get_todo_params[0].schema_data.get("description") == "todo list group name" + + +def test_parse_parameters_post_request_body(): + """Verify whether the post request body parameter can be successfully parsed""" + openapi_fcs: list[KernelFunctionFromMethod] = create_functions_from_openapi( + plugin_name="todo", + openapi_document_path=os.path.join(current_dir, "openapi_todo.yaml"), + execution_settings=None, + ) + + add_todo_list: list[KernelFunctionMetadata] = [ + f.metadata for f in openapi_fcs if f.metadata.name == "add_todo_list" + ] + + assert add_todo_list + + add_todo_params: list[KernelParameterMetadata] = add_todo_list[0].parameters + + assert add_todo_params + assert add_todo_params[0].name == "task" + assert add_todo_params[0].description == "task name" + assert add_todo_params[0].is_required + assert add_todo_params[0].schema_data + assert add_todo_params[0].schema_data.get("type") == "string" + assert add_todo_params[0].schema_data.get("description") == "task name" + + assert add_todo_params[1].name == "listName" + assert add_todo_params[1].description == "task group name" + assert not add_todo_params[1].is_required + assert add_todo_params[1].schema_data + assert add_todo_params[1].schema_data.get("type") == "string" + assert add_todo_params[1].schema_data.get("description") == "task group name" + + def test_get_payload_properties_schema_none(): parser = OpenApiParser() properties = parser._get_payload_properties("operation_id", None, []) From 00f3a6b24186063ec216f51b7f547576c429f516 Mon Sep 17 00:00:00 2001 From: Chris <66376200+crickman@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:12:53 -0700 Subject: [PATCH 2/3] .Net Agents - Streaming Bug Fix and Support Additional Assistant Option (#8852) ### Motivation and Context Respond to two customer identified issues: 1. Add support for `AdditionalInstructions` for creating an assistant as well as invocation override. Fixes #8715 2. Fix issue with duplicated tool-call result when using `ChatCompletionAgent` streaming Fixes #8825 ### Description 1. `AdditionalInstructions` option wasn't included in the V2 migration as oversight. This is a pure addition. 2. Unit-tests added for new `AdditionalInstructions` option. 4. Duplication of the terminated function result addressed within `ChatCompletionAgent` 5. Streaming cases added to existing sample demonstrating use of `IAutoFunctionInvocationFilter` : `Concepts/Agents/ChatCompletion_FunctionTermination` ### Contribution Checklist - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone :smile: --- .../ChatCompletion_FunctionTermination.cs | 148 ++++++++++++++++-- dotnet/src/Agents/Core/ChatCompletionAgent.cs | 10 +- .../Internal/AssistantRunOptionsFactory.cs | 1 + .../OpenAI/Internal/AssistantThreadActions.cs | 1 - .../OpenAI/OpenAIAssistantExecutionOptions.cs | 6 + .../OpenAIAssistantInvocationOptions.cs | 6 + .../AssistantRunOptionsFactoryTests.cs | 9 ++ .../OpenAI/OpenAIAssistantDefinitionTests.cs | 2 + .../OpenAIAssistantInvocationOptionsTests.cs | 3 + .../Agents/ChatCompletionAgentTests.cs | 9 +- .../Agents/MixedAgentTests.cs | 3 +- .../Agents/OpenAIAssistantAgentTests.cs | 5 +- ...penAIChatCompletionFunctionCallingTests.cs | 2 +- ...enAIChatCompletion_FunctionCallingTests.cs | 2 +- 14 files changed, 177 insertions(+), 30 deletions(-) diff --git a/dotnet/samples/Concepts/Agents/ChatCompletion_FunctionTermination.cs b/dotnet/samples/Concepts/Agents/ChatCompletion_FunctionTermination.cs index f208fc8a7634..e1612bfc83c1 100644 --- a/dotnet/samples/Concepts/Agents/ChatCompletion_FunctionTermination.cs +++ b/dotnet/samples/Concepts/Agents/ChatCompletion_FunctionTermination.cs @@ -38,14 +38,8 @@ public async Task UseAutoFunctionInvocationFilterWithAgentInvocationAsync() await InvokeAgentAsync("What is the special drink?"); await InvokeAgentAsync("Thank you"); - // Display the chat history. - Console.WriteLine("================================"); - Console.WriteLine("CHAT HISTORY"); - Console.WriteLine("================================"); - foreach (ChatMessageContent message in chat) - { - this.WriteAgentChatMessage(message); - } + // Display the entire chat history. + WriteChatHistory(chat); // Local function to invoke agent and display the conversation messages. async Task InvokeAgentAsync(string input) @@ -91,15 +85,8 @@ public async Task UseAutoFunctionInvocationFilterWithAgentChatAsync() await InvokeAgentAsync("What is the special drink?"); await InvokeAgentAsync("Thank you"); - // Display the chat history. - Console.WriteLine("================================"); - Console.WriteLine("CHAT HISTORY"); - Console.WriteLine("================================"); - ChatMessageContent[] history = await chat.GetChatMessagesAsync().ToArrayAsync(); - for (int index = history.Length; index > 0; --index) - { - this.WriteAgentChatMessage(history[index - 1]); - } + // Display the entire chat history. + WriteChatHistory(await chat.GetChatMessagesAsync().ToArrayAsync()); // Local function to invoke agent and display the conversation messages. async Task InvokeAgentAsync(string input) @@ -115,6 +102,133 @@ async Task InvokeAgentAsync(string input) } } + [Fact] + public async Task UseAutoFunctionInvocationFilterWithStreamingAgentInvocationAsync() + { + // Define the agent + ChatCompletionAgent agent = + new() + { + Instructions = "Answer questions about the menu.", + Kernel = CreateKernelWithFilter(), + Arguments = new KernelArguments(new OpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }), + }; + + KernelPlugin plugin = KernelPluginFactory.CreateFromType(); + agent.Kernel.Plugins.Add(plugin); + + /// Create the chat history to capture the agent interaction. + ChatHistory chat = []; + + // Respond to user input, invoking functions where appropriate. + await InvokeAgentAsync("Hello"); + await InvokeAgentAsync("What is the special soup?"); + await InvokeAgentAsync("What is the special drink?"); + await InvokeAgentAsync("Thank you"); + + // Display the entire chat history. + WriteChatHistory(chat); + + // Local function to invoke agent and display the conversation messages. + async Task InvokeAgentAsync(string input) + { + ChatMessageContent message = new(AuthorRole.User, input); + chat.Add(message); + this.WriteAgentChatMessage(message); + + int historyCount = chat.Count; + + bool isFirst = false; + await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(chat)) + { + if (string.IsNullOrEmpty(response.Content)) + { + continue; + } + + if (!isFirst) + { + Console.WriteLine($"\n# {response.Role} - {response.AuthorName ?? "*"}:"); + isFirst = true; + } + + Console.WriteLine($"\t > streamed: '{response.Content}'"); + } + + if (historyCount <= chat.Count) + { + for (int index = historyCount; index < chat.Count; index++) + { + this.WriteAgentChatMessage(chat[index]); + } + } + } + } + + [Fact] + public async Task UseAutoFunctionInvocationFilterWithStreamingAgentChatAsync() + { + // Define the agent + ChatCompletionAgent agent = + new() + { + Instructions = "Answer questions about the menu.", + Kernel = CreateKernelWithFilter(), + Arguments = new KernelArguments(new OpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }), + }; + + KernelPlugin plugin = KernelPluginFactory.CreateFromType(); + agent.Kernel.Plugins.Add(plugin); + + // Create a chat for agent interaction. + AgentGroupChat chat = new(); + + // Respond to user input, invoking functions where appropriate. + await InvokeAgentAsync("Hello"); + await InvokeAgentAsync("What is the special soup?"); + await InvokeAgentAsync("What is the special drink?"); + await InvokeAgentAsync("Thank you"); + + // Display the entire chat history. + WriteChatHistory(await chat.GetChatMessagesAsync().ToArrayAsync()); + + // Local function to invoke agent and display the conversation messages. + async Task InvokeAgentAsync(string input) + { + ChatMessageContent message = new(AuthorRole.User, input); + chat.AddChatMessage(message); + this.WriteAgentChatMessage(message); + + bool isFirst = false; + await foreach (StreamingChatMessageContent response in chat.InvokeStreamingAsync(agent)) + { + if (string.IsNullOrEmpty(response.Content)) + { + continue; + } + + if (!isFirst) + { + Console.WriteLine($"\n# {response.Role} - {response.AuthorName ?? "*"}:"); + isFirst = true; + } + + Console.WriteLine($"\t > streamed: '{response.Content}'"); + } + } + } + + private void WriteChatHistory(IEnumerable chat) + { + Console.WriteLine("================================"); + Console.WriteLine("CHAT HISTORY"); + Console.WriteLine("================================"); + foreach (ChatMessageContent message in chat) + { + this.WriteAgentChatMessage(message); + } + } + private Kernel CreateKernelWithFilter() { IKernelBuilder builder = Kernel.CreateBuilder(); diff --git a/dotnet/src/Agents/Core/ChatCompletionAgent.cs b/dotnet/src/Agents/Core/ChatCompletionAgent.cs index cd56f42a9e20..37982a17613c 100644 --- a/dotnet/src/Agents/Core/ChatCompletionAgent.cs +++ b/dotnet/src/Agents/Core/ChatCompletionAgent.cs @@ -94,7 +94,7 @@ public override async IAsyncEnumerable InvokeStream StringBuilder builder = new(); await foreach (StreamingChatMessageContent message in messages.ConfigureAwait(false)) { - role ??= message.Role; + role = message.Role; message.Role ??= AuthorRole.Assistant; message.AuthorName = this.Name; @@ -103,8 +103,6 @@ public override async IAsyncEnumerable InvokeStream yield return message; } - chat.Add(new(role ?? AuthorRole.Assistant, builder.ToString()) { AuthorName = this.Name }); - // Capture mutated messages related function calling / tools for (int messageIndex = messageCount; messageIndex < chat.Count; messageIndex++) { @@ -114,6 +112,12 @@ public override async IAsyncEnumerable InvokeStream history.Add(message); } + + // Do not duplicate terminated function result to history + if (role != AuthorRole.Tool) + { + history.Add(new(role ?? AuthorRole.Assistant, builder.ToString()) { AuthorName = this.Name }); + } } internal static (IChatCompletionService service, PromptExecutionSettings? executionSettings) GetChatCompletionService(Kernel kernel, KernelArguments? arguments) diff --git a/dotnet/src/Agents/OpenAI/Internal/AssistantRunOptionsFactory.cs b/dotnet/src/Agents/OpenAI/Internal/AssistantRunOptionsFactory.cs index 981c646254af..9cef36da3fa3 100644 --- a/dotnet/src/Agents/OpenAI/Internal/AssistantRunOptionsFactory.cs +++ b/dotnet/src/Agents/OpenAI/Internal/AssistantRunOptionsFactory.cs @@ -24,6 +24,7 @@ public static RunCreationOptions GenerateOptions(OpenAIAssistantDefinition defin RunCreationOptions options = new() { + AdditionalInstructions = invocationOptions?.AdditionalInstructions ?? definition.ExecutionOptions?.AdditionalInstructions, MaxCompletionTokens = ResolveExecutionSetting(invocationOptions?.MaxCompletionTokens, definition.ExecutionOptions?.MaxCompletionTokens), MaxPromptTokens = ResolveExecutionSetting(invocationOptions?.MaxPromptTokens, definition.ExecutionOptions?.MaxPromptTokens), ModelOverride = invocationOptions?.ModelName, diff --git a/dotnet/src/Agents/OpenAI/Internal/AssistantThreadActions.cs b/dotnet/src/Agents/OpenAI/Internal/AssistantThreadActions.cs index 933ed120ae2e..643f9dbb1dcc 100644 --- a/dotnet/src/Agents/OpenAI/Internal/AssistantThreadActions.cs +++ b/dotnet/src/Agents/OpenAI/Internal/AssistantThreadActions.cs @@ -11,7 +11,6 @@ using Azure; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.ChatCompletion; - using OpenAI; using OpenAI.Assistants; diff --git a/dotnet/src/Agents/OpenAI/OpenAIAssistantExecutionOptions.cs b/dotnet/src/Agents/OpenAI/OpenAIAssistantExecutionOptions.cs index 074b92831c92..845cecb0956c 100644 --- a/dotnet/src/Agents/OpenAI/OpenAIAssistantExecutionOptions.cs +++ b/dotnet/src/Agents/OpenAI/OpenAIAssistantExecutionOptions.cs @@ -11,6 +11,12 @@ namespace Microsoft.SemanticKernel.Agents.OpenAI; /// public sealed class OpenAIAssistantExecutionOptions { + /// + /// Appends additional instructions. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AdditionalInstructions { get; init; } + /// /// The maximum number of completion tokens that may be used over the course of the run. /// diff --git a/dotnet/src/Agents/OpenAI/OpenAIAssistantInvocationOptions.cs b/dotnet/src/Agents/OpenAI/OpenAIAssistantInvocationOptions.cs index 0653c83a13e2..c06921a6f0d0 100644 --- a/dotnet/src/Agents/OpenAI/OpenAIAssistantInvocationOptions.cs +++ b/dotnet/src/Agents/OpenAI/OpenAIAssistantInvocationOptions.cs @@ -18,6 +18,12 @@ public sealed class OpenAIAssistantInvocationOptions [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ModelName { get; init; } + /// + /// Appends additional instructions. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AdditionalInstructions { get; init; } + /// /// Set if code_interpreter tool is enabled. /// diff --git a/dotnet/src/Agents/UnitTests/OpenAI/Internal/AssistantRunOptionsFactoryTests.cs b/dotnet/src/Agents/UnitTests/OpenAI/Internal/AssistantRunOptionsFactoryTests.cs index d6bcf91b8a94..e3aa50473e81 100644 --- a/dotnet/src/Agents/UnitTests/OpenAI/Internal/AssistantRunOptionsFactoryTests.cs +++ b/dotnet/src/Agents/UnitTests/OpenAI/Internal/AssistantRunOptionsFactoryTests.cs @@ -23,6 +23,11 @@ public void AssistantRunOptionsFactoryExecutionOptionsNullTest() new("gpt-anything") { Temperature = 0.5F, + ExecutionOptions = + new() + { + AdditionalInstructions = "test", + }, }; // Act @@ -32,6 +37,7 @@ public void AssistantRunOptionsFactoryExecutionOptionsNullTest() Assert.NotNull(options); Assert.Null(options.Temperature); Assert.Null(options.NucleusSamplingFactor); + Assert.Equal("test", options.AdditionalInstructions); Assert.Empty(options.Metadata); } @@ -77,6 +83,7 @@ public void AssistantRunOptionsFactoryExecutionOptionsOverrideTest() ExecutionOptions = new() { + AdditionalInstructions = "test1", TruncationMessageCount = 5, }, }; @@ -84,6 +91,7 @@ public void AssistantRunOptionsFactoryExecutionOptionsOverrideTest() OpenAIAssistantInvocationOptions invocationOptions = new() { + AdditionalInstructions = "test2", Temperature = 0.9F, TruncationMessageCount = 8, EnableJsonResponse = true, @@ -96,6 +104,7 @@ public void AssistantRunOptionsFactoryExecutionOptionsOverrideTest() Assert.NotNull(options); Assert.Equal(0.9F, options.Temperature); Assert.Equal(8, options.TruncationStrategy.LastMessages); + Assert.Equal("test2", options.AdditionalInstructions); Assert.Equal(AssistantResponseFormat.JsonObject, options.ResponseFormat); Assert.Null(options.NucleusSamplingFactor); } diff --git a/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantDefinitionTests.cs b/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantDefinitionTests.cs index f8547f375f13..5c28373744a8 100644 --- a/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantDefinitionTests.cs +++ b/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantDefinitionTests.cs @@ -62,6 +62,7 @@ public void VerifyOpenAIAssistantDefinitionAssignment() ExecutionOptions = new() { + AdditionalInstructions = "test instructions", MaxCompletionTokens = 1000, MaxPromptTokens = 1000, ParallelToolCallsEnabled = false, @@ -83,6 +84,7 @@ public void VerifyOpenAIAssistantDefinitionAssignment() Assert.Equal(2, definition.Temperature); Assert.Equal(0, definition.TopP); Assert.NotNull(definition.ExecutionOptions); + Assert.Equal("test instructions", definition.ExecutionOptions.AdditionalInstructions); Assert.Equal(1000, definition.ExecutionOptions.MaxCompletionTokens); Assert.Equal(1000, definition.ExecutionOptions.MaxPromptTokens); Assert.Equal(12, definition.ExecutionOptions.TruncationMessageCount); diff --git a/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantInvocationOptionsTests.cs b/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantInvocationOptionsTests.cs index 99cbe012f183..a07690f42245 100644 --- a/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantInvocationOptionsTests.cs +++ b/dotnet/src/Agents/UnitTests/OpenAI/OpenAIAssistantInvocationOptionsTests.cs @@ -22,6 +22,7 @@ public void OpenAIAssistantInvocationOptionsInitialState() // Assert Assert.Null(options.ModelName); + Assert.Null(options.AdditionalInstructions); Assert.Null(options.Metadata); Assert.Null(options.Temperature); Assert.Null(options.TopP); @@ -48,6 +49,7 @@ public void OpenAIAssistantInvocationOptionsAssignment() new() { ModelName = "testmodel", + AdditionalInstructions = "test instructions", Metadata = new Dictionary() { { "a", "1" } }, MaxCompletionTokens = 1000, MaxPromptTokens = 1000, @@ -62,6 +64,7 @@ public void OpenAIAssistantInvocationOptionsAssignment() // Assert Assert.Equal("testmodel", options.ModelName); + Assert.Equal("test instructions", options.AdditionalInstructions); Assert.Equal(2, options.Temperature); Assert.Equal(0, options.TopP); Assert.Equal(1000, options.MaxCompletionTokens); diff --git a/dotnet/src/IntegrationTests/Agents/ChatCompletionAgentTests.cs b/dotnet/src/IntegrationTests/Agents/ChatCompletionAgentTests.cs index fd8b815b84ab..cf4c7867a0b8 100644 --- a/dotnet/src/IntegrationTests/Agents/ChatCompletionAgentTests.cs +++ b/dotnet/src/IntegrationTests/Agents/ChatCompletionAgentTests.cs @@ -12,6 +12,7 @@ using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.OpenAI; using SemanticKernel.IntegrationTests.TestSettings; +using xRetry; using Xunit; namespace SemanticKernel.IntegrationTests.Agents; @@ -32,7 +33,7 @@ public sealed class ChatCompletionAgentTests() /// Integration test for using function calling /// and targeting Azure OpenAI services. /// - [Theory] + [RetryTheory(typeof(HttpOperationException))] [InlineData("What is the special soup?", "Clam Chowder", false)] [InlineData("What is the special soup?", "Clam Chowder", true)] public async Task AzureChatCompletionAgentAsync(string input, string expectedAnswerContains, bool useAutoFunctionTermination) @@ -96,7 +97,7 @@ public async Task AzureChatCompletionAgentAsync(string input, string expectedAns /// Integration test for using new function calling model /// and targeting Azure OpenAI services. /// - [Theory] + [RetryTheory(typeof(HttpOperationException))] [InlineData("What is the special soup?", "Clam Chowder", false)] [InlineData("What is the special soup?", "Clam Chowder", true)] public async Task AzureChatCompletionAgentUsingNewFunctionCallingModelAsync(string input, string expectedAnswerContains, bool useAutoFunctionTermination) @@ -160,7 +161,7 @@ public async Task AzureChatCompletionAgentUsingNewFunctionCallingModelAsync(stri /// Integration test for using function calling /// and targeting Azure OpenAI services. /// - [Fact] + [RetryFact(typeof(HttpOperationException))] public async Task AzureChatCompletionStreamingAsync() { // Arrange @@ -206,7 +207,7 @@ public async Task AzureChatCompletionStreamingAsync() /// Integration test for using new function calling model /// and targeting Azure OpenAI services. /// - [Fact] + [RetryFact(typeof(HttpOperationException))] public async Task AzureChatCompletionStreamingUsingNewFunctionCallingModelAsync() { // Arrange diff --git a/dotnet/src/IntegrationTests/Agents/MixedAgentTests.cs b/dotnet/src/IntegrationTests/Agents/MixedAgentTests.cs index 4c4792b7694e..94e9b6c34eaf 100644 --- a/dotnet/src/IntegrationTests/Agents/MixedAgentTests.cs +++ b/dotnet/src/IntegrationTests/Agents/MixedAgentTests.cs @@ -11,6 +11,7 @@ using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.OpenAI; using SemanticKernel.IntegrationTests.TestSettings; +using xRetry; using Xunit; namespace SemanticKernel.IntegrationTests.Agents; @@ -50,7 +51,7 @@ await this.VerifyAgentExecutionAsync( /// Integration test for using function calling /// and targeting Azure OpenAI services. /// - [Theory] + [RetryTheory(typeof(HttpOperationException))] [InlineData(false)] [InlineData(true)] public async Task AzureOpenAIMixedAgentAsync(bool useNewFunctionCallingModel) diff --git a/dotnet/src/IntegrationTests/Agents/OpenAIAssistantAgentTests.cs b/dotnet/src/IntegrationTests/Agents/OpenAIAssistantAgentTests.cs index 62388488d483..e2d1ef2b1bfe 100644 --- a/dotnet/src/IntegrationTests/Agents/OpenAIAssistantAgentTests.cs +++ b/dotnet/src/IntegrationTests/Agents/OpenAIAssistantAgentTests.cs @@ -11,6 +11,7 @@ using Microsoft.SemanticKernel.Agents.OpenAI; using Microsoft.SemanticKernel.ChatCompletion; using SemanticKernel.IntegrationTests.TestSettings; +using xRetry; using Xunit; namespace SemanticKernel.IntegrationTests.Agents; @@ -48,7 +49,7 @@ await this.ExecuteAgentAsync( /// Integration test for using function calling /// and targeting Azure OpenAI services. /// - [Theory] + [RetryTheory(typeof(HttpOperationException))] [InlineData("What is the special soup?", "Clam Chowder")] public async Task AzureOpenAIAssistantAgentAsync(string input, string expectedAnswerContains) { @@ -84,7 +85,7 @@ await this.ExecuteStreamingAgentAsync( /// Integration test for using function calling /// and targeting Azure OpenAI services. /// - [Theory/*(Skip = "No supported endpoint configured.")*/] + [RetryTheory(typeof(HttpOperationException))] [InlineData("What is the special soup?", "Clam Chowder")] public async Task AzureOpenAIAssistantAgentStreamingAsync(string input, string expectedAnswerContains) { diff --git a/dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletionFunctionCallingTests.cs b/dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletionFunctionCallingTests.cs index a149d5075db3..c3616167b2c8 100644 --- a/dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletionFunctionCallingTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletionFunctionCallingTests.cs @@ -482,7 +482,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu Assert.NotNull(getWeatherForCityFunctionCallResult.Result); } - [Fact] + [Fact(Skip = "Weather in Boston (USA) is not supported.")] public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForManualFunctionCallingForStreamingAsync() { // Arrange diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIChatCompletion_FunctionCallingTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIChatCompletion_FunctionCallingTests.cs index 38dc4b52daff..0423323cfab0 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIChatCompletion_FunctionCallingTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIChatCompletion_FunctionCallingTests.cs @@ -482,7 +482,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu Assert.NotNull(getWeatherForCityFunctionCallResult.Result); } - [Fact] + [Fact(Skip = "Weather in Boston (USA) is not supported.")] public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForManualFunctionCallingForStreamingAsync() { // Arrange From 4c00b79542d25f52524c2a2c4ba277d165263ed1 Mon Sep 17 00:00:00 2001 From: SergeyMenshykh <68852919+SergeyMenshykh@users.noreply.github.com> Date: Tue, 17 Sep 2024 21:24:45 +0100 Subject: [PATCH 3/3] .Net: Sample demonstrating function advertisement depending on context (#8842) Closes: https://github.com/microsoft/semantic-kernel/issues/8840 --- .../ContextDependentAdvertising.cs | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 dotnet/samples/Concepts/FunctionCalling/ContextDependentAdvertising.cs diff --git a/dotnet/samples/Concepts/FunctionCalling/ContextDependentAdvertising.cs b/dotnet/samples/Concepts/FunctionCalling/ContextDependentAdvertising.cs new file mode 100644 index 000000000000..7d7adc2e5f7d --- /dev/null +++ b/dotnet/samples/Concepts/FunctionCalling/ContextDependentAdvertising.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; + +namespace FunctionCalling; + +/// +/// These samples demonstrate how to advertise functions to AI model based on a context. +/// +public class ContextDependentAdvertising(ITestOutputHelper output) : BaseTest(output) +{ + /// + /// This sample demonstrates how to advertise functions to AI model based on the context of the chat history. + /// It advertises functions to the AI model based on the game state. + /// For example, if the maze has not been created, advertise the create maze function only to prevent the AI model + /// from adding traps or treasures to the maze before it is created. + /// + [Fact] + public async Task AdvertiseFunctionsDependingOnContextPerUserInteractionAsync() + { + Kernel kernel = CreateKernel(); + + IChatCompletionService chatCompletionService = kernel.GetRequiredService(); + + // Tracking number of iterations to avoid infinite loop. + int maxIteration = 10; + int iteration = 0; + + // Define the functions for AI model to call. + var gameUtils = kernel.ImportPluginFromType(); + KernelFunction createMaze = gameUtils["CreateMaze"]; + KernelFunction addTraps = gameUtils["AddTrapsToMaze"]; + KernelFunction addTreasures = gameUtils["AddTreasuresToMaze"]; + KernelFunction playGame = gameUtils["PlayGame"]; + + ChatHistory chatHistory = []; + chatHistory.AddUserMessage("I would like to play a maze game with a lot of tricky traps and shiny treasures."); + + // Loop until the game has started or the max iteration is reached. + while (!chatHistory.Any(item => item.Content?.Contains("Game started.") ?? false) && iteration < maxIteration) + { + List functionsToAdvertise = new(); + + // Decide game state based on chat history. + bool mazeCreated = chatHistory.Any(item => item.Content?.Contains("Maze created.") ?? false); + bool trapsAdded = chatHistory.Any(item => item.Content?.Contains("Traps added to the maze.") ?? false); + bool treasuresAdded = chatHistory.Any(item => item.Content?.Contains("Treasures added to the maze.") ?? false); + + // The maze has not been created yet so advertise the create maze function. + if (!mazeCreated) + { + functionsToAdvertise.Add(createMaze); + } + // The maze has been created so advertise the adding traps and treasures functions. + else if (mazeCreated && (!trapsAdded || !treasuresAdded)) + { + functionsToAdvertise.Add(addTraps); + functionsToAdvertise.Add(addTreasures); + } + // Both traps and treasures have been added so advertise the play game function. + else if (treasuresAdded && trapsAdded) + { + functionsToAdvertise.Add(playGame); + } + + // Provide the functions to the AI model. + OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Required(functionsToAdvertise) }; + + // Prompt the AI model. + ChatMessageContent result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + + Console.WriteLine(result); + + iteration++; + } + } + + private static Kernel CreateKernel() + { + // Create kernel + IKernelBuilder builder = Kernel.CreateBuilder(); + + builder.AddOpenAIChatCompletion(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey); + + return builder.Build(); + } + + private sealed class GameUtils + { + [KernelFunction] + public static string CreateMaze() => "Maze created."; + + [KernelFunction] + public static string AddTrapsToMaze() => "Traps added to the maze."; + + [KernelFunction] + public static string AddTreasuresToMaze() => "Treasures added to the maze."; + + [KernelFunction] + public static string PlayGame() => "Game started."; + } +}