From cbf31ee9c74e5c6dfaf4fff7e8fe17428f751ea7 Mon Sep 17 00:00:00 2001 From: Teresa Hoang <125500434+teresaqhoang@users.noreply.github.com> Date: Wed, 22 Nov 2023 14:38:09 -0800 Subject: [PATCH] .Net: Handlebars planner complex types (#3502) ### Motivation and Context Support for complex types in Handlebars Planner ### Description Native Plugin with Complex Types ![image](https://github.com/microsoft/semantic-kernel/assets/125500434/01c9d5a0-b98b-4d88-ae5a-f38425a2d1ba) Remote Plugin with Complex Types ![image](https://github.com/microsoft/semantic-kernel/assets/125500434/ebd3afb7-1c49-483a-b4ea-6f41783837bf) ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --- .../Example65_HandlebarsPlanner.cs | 142 ++++---- .../ComplexParamsDictionaryPlugin.cs | 105 ++++++ .../StringParamsDictionaryPlugin.cs | 43 +++ .../Plugins/DictionaryPlugin/openapi.json | 101 ++++++ .../KernelOpenApiPluginExtensions.cs | 2 +- .../HandlebarsPlannerTests.cs | 6 +- ...HandlebarsTemplateEngineExtensionsTests.cs | 35 +- .../SKParameterMetadataExtensionsTests.cs | 314 ++++++++++++++++++ ...handlebars => CreatePlanPrompt.handlebars} | 147 +++++--- .../HandlebarsPlannerExtensions.cs | 2 + .../SKParameterMetadataExtensions.cs | 148 +++++++++ .../Handlebars/HandlebarsParameterTypeView.cs | 89 ----- .../Handlebars/HandlebarsPlan.cs | 28 +- .../Handlebars/HandlebarsPlanner.cs | 121 +++++-- .../Handlebars/HandlebarsPlannerConfig.cs | 1 + .../HandlebarsTemplateEngineExtensions.cs | 229 +++++++++---- .../Models/HandlebarsParameterTypeMetadata.cs | 82 +++++ .../Handlebars/NoLoops/skPrompt.handlebars | 130 -------- .../Planners.Core/Planners.Core.csproj | 5 +- 19 files changed, 1278 insertions(+), 452 deletions(-) create mode 100644 dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/ComplexParamsDictionaryPlugin.cs create mode 100644 dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/StringParamsDictionaryPlugin.cs create mode 100644 dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/openapi.json rename dotnet/src/IntegrationTests/Planners/{HandlebarsPlanner => Handlebars}/HandlebarsPlannerTests.cs (94%) rename dotnet/src/IntegrationTests/Planners/{HandlebarsPlanner => Handlebars}/HandlebarsTemplateEngineExtensionsTests.cs (91%) create mode 100644 dotnet/src/IntegrationTests/Planners/Handlebars/SKParameterMetadataExtensionsTests.cs rename dotnet/src/Planners/Planners.Core/Handlebars/{skPrompt.handlebars => CreatePlanPrompt.handlebars} (50%) rename dotnet/src/Planners/Planners.Core/Handlebars/{ => Extensions}/HandlebarsPlannerExtensions.cs (96%) create mode 100644 dotnet/src/Planners/Planners.Core/Handlebars/Extensions/SKParameterMetadataExtensions.cs delete mode 100644 dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsParameterTypeView.cs create mode 100644 dotnet/src/Planners/Planners.Core/Handlebars/Models/HandlebarsParameterTypeMetadata.cs delete mode 100644 dotnet/src/Planners/Planners.Core/Handlebars/NoLoops/skPrompt.handlebars diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs index 533ea8a516f3..48dfa82e0d8a 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs @@ -3,12 +3,12 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Functions.OpenAPI.Extensions; using Microsoft.SemanticKernel.Planning.Handlebars; +using Plugins.DictionaryPlugin; using RepoUtils; /** @@ -16,28 +16,34 @@ */ public static class Example65_HandlebarsPlanner { - private static int s_sampleCount; + private static int s_sampleIndex; + + private const string CourseraPluginName = "CourseraPlugin"; /// /// Show how to create a plan with Handlebars and execute it. /// public static async Task RunAsync() { - s_sampleCount = 0; - Console.WriteLine($"======== {nameof(Example65_HandlebarsPlanner)} ========"); + s_sampleIndex = 1; + bool shouldPrintPrompt = true; + // Using primitive types as inputs and outputs await PlanNotPossibleSampleAsync(); - await RunDictionarySampleAsync(); + await RunDictionaryWithBasicTypesSampleAsync(); await RunPoetrySampleAsync(); await RunBookSampleAsync(); + + // Using Complex Types as inputs and outputs + await RunLocalDictionaryWithComplexTypesSampleAsync(shouldPrintPrompt); } private static void WriteSampleHeadingToConsole(string name) { - Console.WriteLine($"======== [Handlebars Planner] Sample {s_sampleCount++} - Create and Execute {name} Plan ========"); + Console.WriteLine($"======== [Handlebars Planner] Sample {s_sampleIndex++} - Create and Execute {name} Plan ========"); } - private static async Task RunSampleAsync(string goal, params string[] pluginDirectoryNames) + private static async Task RunSampleAsync(string goal, bool shouldPrintPrompt = false, params string[] pluginDirectoryNames) { string apiKey = TestConfiguration.AzureOpenAI.ApiKey; string chatDeploymentName = TestConfiguration.AzureOpenAI.ChatDeploymentName; @@ -58,9 +64,20 @@ private static async Task RunSampleAsync(string goal, params string[] pluginDire apiKey: apiKey) .Build(); - if (pluginDirectoryNames[0] == DictionaryPlugin.PluginName) + if (pluginDirectoryNames[0] == StringParamsDictionaryPlugin.PluginName) { - kernel.ImportPluginFromObject(new DictionaryPlugin(), DictionaryPlugin.PluginName); + kernel.ImportPluginFromObject(new StringParamsDictionaryPlugin(), StringParamsDictionaryPlugin.PluginName); + } + else if (pluginDirectoryNames[0] == ComplexParamsDictionaryPlugin.PluginName) + { + kernel.ImportPluginFromObject(new ComplexParamsDictionaryPlugin(), ComplexParamsDictionaryPlugin.PluginName); + } + else if (pluginDirectoryNames[0] == CourseraPluginName) + { + await kernel.ImportPluginFromOpenApiAsync( + CourseraPluginName, + new Uri("https://www.coursera.org/api/rest/v1/search/openapi.yaml") + ); } else { @@ -72,14 +89,28 @@ private static async Task RunSampleAsync(string goal, params string[] pluginDire } } - // The gpt-35-turbo model does not handle loops well in the plans. - var allowLoopsInPlan = chatDeploymentName.Contains("gpt-35-turbo", StringComparison.OrdinalIgnoreCase) ? false : true; + // Use gpt-4 or newer models if you want to test with loops. + // Older models like gpt-35-turbo are less recommended. They do handle loops but are more prone to syntax errors. + var allowLoopsInPlan = chatDeploymentName.Contains("gpt-4", StringComparison.OrdinalIgnoreCase); + var planner = new HandlebarsPlanner( + kernel, + new HandlebarsPlannerConfig() + { + // Change this if you want to test with loops regardless of model selection. + AllowLoops = allowLoopsInPlan + }); - var planner = new HandlebarsPlanner(kernel, new HandlebarsPlannerConfig() { AllowLoops = allowLoopsInPlan }); Console.WriteLine($"Goal: {goal}"); // Create the plan var plan = await planner.CreatePlanAsync(goal); + + if (shouldPrintPrompt) + { + // Print the prompt template + Console.WriteLine($"\nPrompt template:\n{plan.Prompt}"); + } + Console.WriteLine($"\nOriginal plan:\n{plan}"); // Execute the plan @@ -87,14 +118,14 @@ private static async Task RunSampleAsync(string goal, params string[] pluginDire Console.WriteLine($"\nResult:\n{result.GetValue()}\n"); } - private static async Task PlanNotPossibleSampleAsync() + private static async Task PlanNotPossibleSampleAsync(bool shouldPrintPrompt = false) { WriteSampleHeadingToConsole("Plan Not Possible"); try { // Load additional plugins to enable planner but not enough for the given goal. - await RunSampleAsync("Send Mary an email with the list of meetings I have scheduled today.", "SummarizePlugin"); + await RunSampleAsync("Send Mary an email with the list of meetings I have scheduled today.", shouldPrintPrompt, "SummarizePlugin"); } catch (SKException e) { @@ -111,10 +142,10 @@ Additional helpers may be required. } } - private static async Task RunDictionarySampleAsync() + private static async Task RunDictionaryWithBasicTypesSampleAsync(bool shouldPrintPrompt = false) { WriteSampleHeadingToConsole("Dictionary"); - await RunSampleAsync("Get a random word and its definition.", DictionaryPlugin.PluginName); + await RunSampleAsync("Get a random word and its definition.", shouldPrintPrompt, StringParamsDictionaryPlugin.PluginName); /* Original plan: {{!-- Step 1: Get a random word --}} @@ -131,10 +162,44 @@ private static async Task RunDictionarySampleAsync() */ } - private static async Task RunPoetrySampleAsync() + private static async Task RunLocalDictionaryWithComplexTypesSampleAsync(bool shouldPrintPrompt = false) + { + WriteSampleHeadingToConsole("Complex Types with Local Dictionary Plugin"); + await RunSampleAsync("Teach me two random words and their definition.", shouldPrintPrompt, ComplexParamsDictionaryPlugin.PluginName); + /* + Original Plan: + {{!-- Step 1: Get two random dictionary entries --}} + {{set "entry1" (DictionaryPlugin-GetRandomEntry)}} + {{set "entry2" (DictionaryPlugin-GetRandomEntry)}} + + {{!-- Step 2: Extract words from the entries --}} + {{set "word1" (DictionaryPlugin-GetWord entry=(get "entry1"))}} + {{set "word2" (DictionaryPlugin-GetWord entry=(get "entry2"))}} + + {{!-- Step 3: Extract definitions for the words --}} + {{set "definition1" (DictionaryPlugin-GetDefinition word=(get "word1"))}} + {{set "definition2" (DictionaryPlugin-GetDefinition word=(get "word2"))}} + + {{!-- Step 4: Display the words and their definitions --}} + Word 1: {{json (get "word1")}} + Definition: {{json (get "definition1")}} + + Word 2: {{json (get "word2")}} + Definition: {{json (get "definition2")}} + + Result: + Word 1: apple + Definition 1: a round fruit with red, green, or yellow skin and a white flesh + + Word 2: dog + Definition 2: a domesticated animal with four legs, a tail, and a keen sense of smell that is often used for hunting or companionship + */ + } + + private static async Task RunPoetrySampleAsync(bool shouldPrintPrompt = false) { WriteSampleHeadingToConsole("Poetry"); - await RunSampleAsync("Write a poem about John Doe, then translate it into Italian.", "SummarizePlugin", "WriterPlugin"); + await RunSampleAsync("Write a poem about John Doe, then translate it into Italian.", shouldPrintPrompt, "SummarizePlugin", "WriterPlugin"); /* Original plan: {{!-- Step 1: Initialize the scenario for the poem --}} @@ -158,10 +223,10 @@ Al mistero che lo faceva brillare. */ } - private static async Task RunBookSampleAsync() + private static async Task RunBookSampleAsync(bool shouldPrintPrompt = false) { WriteSampleHeadingToConsole("Book Creation"); - await RunSampleAsync("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'", "WriterPlugin", "MiscPlugin"); + await RunSampleAsync("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'", shouldPrintPrompt, "WriterPlugin", "MiscPlugin"); /* Original plan: {{!-- Step 1: Initialize the book title and chapter count --}} @@ -185,39 +250,4 @@ private static async Task RunBookSampleAsync() {{/each}} */ } - - /// - /// Plugin example with two native functions, where one function gets a random word and the other returns a definition for a given word. - /// - private sealed class DictionaryPlugin - { - public const string PluginName = nameof(DictionaryPlugin); - - private readonly Dictionary _dictionary = new() - { - {"apple", "a round fruit with red, green, or yellow skin and a white flesh"}, - {"book", "a set of printed or written pages bound together along one edge"}, - {"cat", "a small furry animal with whiskers and a long tail that is often kept as a pet"}, - {"dog", "a domesticated animal with four legs, a tail, and a keen sense of smell that is often used for hunting or companionship"}, - {"elephant", "a large gray mammal with a long trunk, tusks, and ears that lives in Africa and Asia"} - }; - - [SKFunction, SKName("GetRandomWord"), System.ComponentModel.Description("Gets a random word from a dictionary of common words and their definitions.")] - public string GetRandomWord() - { - // Get random number - var index = RandomNumberGenerator.GetInt32(0, this._dictionary.Count - 1); - - // Return the word at the random index - return this._dictionary.ElementAt(index).Key; - } - - [SKFunction, SKName("GetDefinition"), System.ComponentModel.Description("Gets the definition for a given word.")] - public string GetDefinition([System.ComponentModel.Description("Word to get definition for.")] string word) - { - return this._dictionary.TryGetValue(word, out var definition) - ? definition - : "Word not found"; - } - } } diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/ComplexParamsDictionaryPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/ComplexParamsDictionaryPlugin.cs new file mode 100644 index 000000000000..be5d15f7da6c --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/ComplexParamsDictionaryPlugin.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.SemanticKernel; + +namespace Plugins.DictionaryPlugin; + +/// +/// Plugin example with two Local functions, where one function gets a random word and the other returns a definition for a given word. +/// +public sealed class ComplexParamsDictionaryPlugin +{ + public const string PluginName = nameof(ComplexParamsDictionaryPlugin); + + private readonly List _dictionary = new() + { + new DictionaryEntry("apple", "a round fruit with red, green, or yellow skin and a white flesh"), + new DictionaryEntry("book", "a set of printed or written pages bound together along one edge"), + new DictionaryEntry("cat", "a small furry animal with whiskers and a long tail that is often kept as a pet"), + new DictionaryEntry("dog", "a domesticated animal with four legs, a tail, and a keen sense of smell that is often used for hunting or companionship"), + new DictionaryEntry("elephant", "a large gray mammal with a long trunk, tusks, and ears that lives in Africa and Asia") + }; + + [SKFunction, SKName("GetRandomEntry"), System.ComponentModel.Description("Gets a random word from a dictionary of common words and their definitions.")] + public DictionaryEntry GetRandomEntry() + { + // Get random number + var index = RandomNumberGenerator.GetInt32(0, this._dictionary.Count - 1); + + // Return the word at the random index + return this._dictionary[index]; + } + + [SKFunction, SKName("GetWord"), System.ComponentModel.Description("Gets the word for a given dictionary entry.")] + public string GetWord([System.ComponentModel.Description("Word to get definition for.")] DictionaryEntry entry) + { + // Return the definition or a default message + return this._dictionary.FirstOrDefault(e => e.Word == entry.Word)?.Word ?? "Entry not found"; + } + + [SKFunction, SKName("GetDefinition"), System.ComponentModel.Description("Gets the definition for a given word.")] + public string GetDefinition([System.ComponentModel.Description("Word to get definition for.")] string word) + { + // Return the definition or a default message + return this._dictionary.FirstOrDefault(e => e.Word == word)?.Definition ?? "Word not found"; + } +} + +/// +/// In order to use custom types, should be specified, +/// that will convert object instance to string representation. +/// +/// +/// is used to represent complex object as meaningful string, so +/// it can be passed to AI for further processing using semantic functions. +/// It's possible to choose any format (e.g. XML, JSON, YAML) to represent your object. +/// +[TypeConverter(typeof(DictionaryEntryConverter))] +public sealed class DictionaryEntry +{ + public string Word { get; set; } = string.Empty; + public string Definition { get; set; } = string.Empty; + + public DictionaryEntry(string word, string definition) + { + this.Word = word; + this.Definition = definition; + } +} + +/// +/// Implementation of for . +/// In this example, object instance is serialized with from System.Text.Json, +/// but it's possible to convert object to string using any other serialization logic. +/// +#pragma warning disable CA1812 // instantiated by Kernel +public sealed class DictionaryEntryConverter : TypeConverter +#pragma warning restore CA1812 +{ + public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) => true; + + /// + /// This method is used to convert object from string to actual type. This will allow to pass object to + /// Local function which requires it. + /// + public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) + { + return JsonSerializer.Deserialize((string)value); + } + + /// + /// This method is used to convert actual type to string representation, so it can be passed to AI + /// for further processing. + /// + public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType) + { + return JsonSerializer.Serialize(value); + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/StringParamsDictionaryPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/StringParamsDictionaryPlugin.cs new file mode 100644 index 000000000000..7b12ff2024e6 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/StringParamsDictionaryPlugin.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using Microsoft.SemanticKernel; + +namespace Plugins.DictionaryPlugin; + +/// +/// Plugin example with two native functions, where one function gets a random word and the other returns a definition for a given word. +/// +public sealed class StringParamsDictionaryPlugin +{ + public const string PluginName = nameof(StringParamsDictionaryPlugin); + + private readonly Dictionary _dictionary = new() + { + {"apple", "a round fruit with red, green, or yellow skin and a white flesh"}, + {"book", "a set of printed or written pages bound together along one edge"}, + {"cat", "a small furry animal with whiskers and a long tail that is often kept as a pet"}, + {"dog", "a domesticated animal with four legs, a tail, and a keen sense of smell that is often used for hunting or companionship"}, + {"elephant", "a large gray mammal with a long trunk, tusks, and ears that lives in Africa and Asia"} + }; + + [SKFunction, SKName("GetRandomWord"), System.ComponentModel.Description("Gets a random word from a dictionary of common words and their definitions.")] + public string GetRandomWord() + { + // Get random number + var index = RandomNumberGenerator.GetInt32(0, this._dictionary.Count - 1); + + // Return the word at the random index + return this._dictionary.ElementAt(index).Key; + } + + [SKFunction, SKName("GetDefinition"), System.ComponentModel.Description("Gets the definition for a given word.")] + public string GetDefinition([System.ComponentModel.Description("Word to get definition for.")] string word) + { + return this._dictionary.TryGetValue(word, out var definition) + ? definition + : "Word not found"; + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/openapi.json b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/openapi.json new file mode 100644 index 000000000000..bd5a0fec8bbd --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/DictionaryPlugin/openapi.json @@ -0,0 +1,101 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "DictionaryPlugin", + "version": "1.0.0", + "description": "A plugin that provides dictionary functions for common words and their definitions." + }, + "paths": { + "/GetRandomEntry": { + "get": { + "summary": "Gets a random word from a dictionary of common words and their definitions.", + "operationId": "GetRandomEntry", + "responses": { + "200": { + "description": "A successful response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DictionaryEntry" + } + } + } + } + } + } + }, + "/GetWord": { + "get": { + "summary": "Gets the word for a given dictionary entry.", + "operationId": "GetWord", + "parameters": [ + { + "name": "entry", + "in": "query", + "description": "Word to get definition for.", + "required": true, + "schema": { + "$ref": "#/components/schemas/DictionaryEntry" + } + } + ], + "responses": { + "200": { + "description": "A successful response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/GetDefinition": { + "get": { + "summary": "Gets the definition for a given word.", + "operationId": "GetDefinition", + "parameters": [ + { + "name": "word", + "in": "query", + "description": "Word to get definition for.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A successful response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "DictionaryEntry": { + "type": "object", + "properties": { + "Word": { + "type": "string" + }, + "Definition": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelOpenApiPluginExtensions.cs b/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelOpenApiPluginExtensions.cs index 750ea69932ea..f1c77b421efb 100644 --- a/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelOpenApiPluginExtensions.cs +++ b/dotnet/src/Functions/Functions.OpenAPI/Extensions/KernelOpenApiPluginExtensions.cs @@ -322,7 +322,7 @@ async Task ExecuteAsync(SKContext context, Cancellatio var parameters = restOperationParameters .Select(p => new SKParameterMetadata(p.AlternativeName ?? p.Name) { - Description = $"{p.Description ?? p.Name}{(p.IsRequired ? " (required)" : string.Empty)}", + Description = $"{p.Description ?? p.Name}", DefaultValue = p.DefaultValue ?? string.Empty, IsRequired = p.IsRequired, Schema = p.Schema ?? (p.Type is null ? null : SKJsonSchema.Parse($"{{\"type\":\"{p.Type}\"}}")), diff --git a/dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsPlannerTests.cs b/dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsPlannerTests.cs similarity index 94% rename from dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsPlannerTests.cs rename to dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsPlannerTests.cs index 9deeac34b20f..a5d18a4d6967 100644 --- a/dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsPlannerTests.cs +++ b/dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsPlannerTests.cs @@ -13,9 +13,7 @@ using Xunit; using Xunit.Abstractions; -#pragma warning disable IDE0130 // Namespace does not match folder structure namespace SemanticKernel.IntegrationTests.Planners.Handlebars; -#pragma warning restore IDE0130 public sealed class HandlebarsPlannerTests : IDisposable { @@ -39,7 +37,7 @@ public async Task CreatePlanFunctionFlowAsync(bool useChatModel, string prompt, { // Arrange bool useEmbeddings = false; - Kernel kernel = this.InitializeKernel(useEmbeddings, useChatModel); + var kernel = this.InitializeKernel(useEmbeddings, useChatModel); kernel.ImportPluginFromObject(new EmailPluginFake(), expectedPlugin); TestHelpers.ImportSamplePlugins(kernel, "FunPlugin"); @@ -57,7 +55,7 @@ public async Task CreatePlanFunctionFlowAsync(bool useChatModel, string prompt, } [RetryTheory] - [InlineData("Write a novel about software development that is 3 chapters long.", "NovelOutline", "WriterPlugin")] + [InlineData("Outline a novel about software development that is 3 chapters long.", "NovelOutline", "WriterPlugin")] public async Task CreatePlanWithDefaultsAsync(string prompt, string expectedFunction, string expectedPlugin) { // Arrange diff --git a/dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsTemplateEngineExtensionsTests.cs b/dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsTemplateEngineExtensionsTests.cs similarity index 91% rename from dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsTemplateEngineExtensionsTests.cs rename to dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsTemplateEngineExtensionsTests.cs index af2400659543..f7fbc6a679b5 100644 --- a/dotnet/src/IntegrationTests/Planners/HandlebarsPlanner/HandlebarsTemplateEngineExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Planners/Handlebars/HandlebarsTemplateEngineExtensionsTests.cs @@ -14,9 +14,7 @@ using Xunit; using Xunit.Abstractions; -#pragma warning disable IDE0130 // Namespace does not match folder structure namespace SemanticKernel.IntegrationTests.Planners.Handlebars; -#pragma warning restore IDE0130 public sealed class HandlebarsTemplateEngineExtensionsTests : IDisposable { @@ -198,23 +196,22 @@ public void ShouldRenderTemplateWithFunctionHelpers() Assert.Equal("Foo Bar", result); } - // TODO [@teresaqhoang]: Add this back in when parameter metadata types are better supported. Currently, parameter type is null when it should be string. - // [Fact] - // public void ShouldRenderTemplateWithFunctionHelpersWithPositionalArguments() - // { - // // Arrange - // var kernel = this.InitializeKernel(); - // var executionContext = kernel.CreateNewContext(); - // var template = "{{Foo-Combine \"Bar\" \"Baz\"}}"; // Use positional arguments instead of hashed arguments - // var variables = new Dictionary(); - // kernel.ImportFunctions(new Foo(), "Foo"); - - // // Act - // var result = HandlebarsTemplateEngineExtensions.Render(kernel, executionContext, template, variables); - - // // Assert - // Assert.Equal("BazBar", result); - // } + [Fact] + public void ShouldRenderTemplateWithFunctionHelpersWithPositionalArguments() + { + // Arrange + var kernel = this.InitializeKernel(); + var executionContext = kernel.CreateNewContext(); + var template = "{{Foo-Combine \"Bar\" \"Baz\"}}"; // Use positional arguments instead of hashed arguments + var variables = new Dictionary(); + kernel.ImportPluginFromObject(new Foo(), "Foo"); + + // Act + var result = HandlebarsTemplateEngineExtensions.Render(kernel, executionContext, template, variables); + + // Assert + Assert.Equal("BazBar", result); + } [Fact] public void ShouldRenderTemplateWithFunctionHelpersWitHashArguments() diff --git a/dotnet/src/IntegrationTests/Planners/Handlebars/SKParameterMetadataExtensionsTests.cs b/dotnet/src/IntegrationTests/Planners/Handlebars/SKParameterMetadataExtensionsTests.cs new file mode 100644 index 000000000000..079486645aad --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/Handlebars/SKParameterMetadataExtensionsTests.cs @@ -0,0 +1,314 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planning.Handlebars; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Planners.Handlebars; + +public class SKParameterMetadataExtensionsTests +{ + [Fact] + public void ReturnsTrueForPrimitiveOrStringTypes() + { + // Arrange + var primitiveTypes = new Type[] { typeof(int), typeof(double), typeof(bool), typeof(char) }; + var stringType = typeof(string); + + // Act and Assert + foreach (var type in primitiveTypes) + { + Assert.True(SKParameterMetadataExtensions.IsPrimitiveOrStringType(type)); + } + + Assert.True(SKParameterMetadataExtensions.IsPrimitiveOrStringType(stringType)); + } + + [Fact] + public void ReturnsFalseForNonPrimitiveOrStringTypes() + { + // Arrange + var nonPrimitiveTypes = new Type[] { typeof(object), typeof(DateTime), typeof(List), typeof(HandlebarsParameterTypeMetadata) }; + + // Act and Assert + foreach (var type in nonPrimitiveTypes) + { + Assert.False(SKParameterMetadataExtensions.IsPrimitiveOrStringType(type)); + } + } + + [Fact] + public void ReturnsEmptySetForPrimitiveOrStringType() + { + // Arrange + var primitiveType = typeof(int); + + // Act + var result = primitiveType.ToHandlebarsParameterTypeMetadata(); + + // Assert + Assert.Empty(result); + } + + [Fact] + public void ReturnsSetWithOneElementForSimpleClassType() + { + // Arrange + var simpleClassType = typeof(SimpleClass); + + // Act + var result = simpleClassType.ToHandlebarsParameterTypeMetadata(); + + // Assert + Assert.Single(result); + Assert.Equal("SimpleClass", result.First().Name); + Assert.True(result.First().IsComplex); + Assert.Equal(2, result.First().Properties.Count); + Assert.Equal("Id", result.First().Properties[0].Name); + Assert.Equal(typeof(int), result.First().Properties[0].ParameterType); + Assert.Equal("Name", result.First().Properties[1].Name); + Assert.Equal(typeof(string), result.First().Properties[1].ParameterType); + } + + [Fact] + public void ReturnsSetWithMultipleElementsForNestedClassType() + { + // Arrange + var nestedClassType = typeof(NestedClass); + + // Act + var result = nestedClassType.ToHandlebarsParameterTypeMetadata(); + + // Assert + Assert.Equal(3, result.Count); + Assert.Contains(result, r => r.Name == "NestedClass"); + Assert.Contains(result, r => r.Name == "SimpleClass"); + Assert.Contains(result, r => r.Name == "AnotherClass"); + + var nestedClass = result.First(r => r.Name == "NestedClass"); + Assert.True(nestedClass.IsComplex); + Assert.Equal(3, nestedClass.Properties.Count); + Assert.Equal("Id", nestedClass.Properties[0].Name); + Assert.Equal(typeof(int), nestedClass.Properties[0].ParameterType); + Assert.Equal("Simple", nestedClass.Properties[1].Name); + Assert.Equal(typeof(SimpleClass), nestedClass.Properties[1].ParameterType); + Assert.Equal("Another", nestedClass.Properties[2].Name); + Assert.Equal(typeof(AnotherClass), nestedClass.Properties[2].ParameterType); + + var simpleClass = result.First(r => r.Name == "SimpleClass"); + Assert.True(simpleClass.IsComplex); + Assert.Equal(2, simpleClass.Properties.Count); + Assert.Equal("Id", simpleClass.Properties[0].Name); + Assert.Equal(typeof(int), simpleClass.Properties[0].ParameterType); + Assert.Equal("Name", simpleClass.Properties[1].Name); + Assert.Equal(typeof(string), simpleClass.Properties[1].ParameterType); + + var anotherClass = result.First(r => r.Name == "AnotherClass"); + Assert.True(anotherClass.IsComplex); + Assert.Single(anotherClass.Properties); + Assert.Equal("Value", anotherClass.Properties[0].Name); + Assert.Equal(typeof(double), anotherClass.Properties[0].ParameterType); + + // Should not contain primitive types + Assert.DoesNotContain(result, r => r.Name == "Id"); + Assert.DoesNotContain(result, r => !r.IsComplex); + + // Should not contain empty complex types + Assert.DoesNotContain(result, r => r.IsComplex && r.Properties.Count == 0); + } + + [Fact] + public void ReturnsSetWithOneElementForTaskOfSimpleClassType() + { + // Arrange + var taskOfSimpleClassType = typeof(Task); + + // Act + var result = taskOfSimpleClassType.ToHandlebarsParameterTypeMetadata(); + + // Assert + Assert.Single(result); + Assert.Equal("SimpleClass", result.First().Name); + Assert.True(result.First().IsComplex); + Assert.Equal(2, result.First().Properties.Count); + Assert.Equal("Id", result.First().Properties[0].Name); + Assert.Equal(typeof(int), result.First().Properties[0].ParameterType); + Assert.Equal("Name", result.First().Properties[1].Name); + Assert.Equal(typeof(string), result.First().Properties[1].ParameterType); + } + + [Fact] + public void ReturnsEmptySetForTaskOfPrimitiveOrStringType() + { + // Arrange + var taskOfPrimitiveType = typeof(Task); + var taskOfStringType = typeof(Task); + + // Act + var result1 = taskOfPrimitiveType.ToHandlebarsParameterTypeMetadata(); + var result2 = taskOfStringType.ToHandlebarsParameterTypeMetadata(); + + // Assert + Assert.Empty(result1); + Assert.Empty(result2); + } + + [Fact] + public void ReturnsTrueForPrimitiveOrStringSchemaTypes() + { + // Arrange + var primitiveSchemaTypes = new string[] { "string", "number", "integer", "boolean" }; + + // Act and Assert + foreach (var type in primitiveSchemaTypes) + { + Assert.True(SKParameterMetadataExtensions.IsPrimitiveOrStringType(type)); + } + } + + [Fact] + public void ReturnsFalseForNonPrimitiveOrStringSchemaTypes() + { + // Arrange + var nonPrimitiveSchemaTypes = new string[] { "object", "array", "any", "null" }; + + // Act and Assert + foreach (var type in nonPrimitiveSchemaTypes) + { + Assert.False(SKParameterMetadataExtensions.IsPrimitiveOrStringType(type)); + } + } + + [Fact] + public void ReturnsParameterWithParameterTypeForPrimitiveOrStringSchemaType() + { + // Arrange + var schemaTypeMap = new Dictionary + { + {"string", typeof(string)}, + {"integer", typeof(long)}, + {"number", typeof(double)}, + {"boolean", typeof(bool)}, + {"null", typeof(object)} + }; + + foreach (var pair in schemaTypeMap) + { + var schema = SKJsonSchema.Parse($"{{\"type\": \"{pair.Key}\"}}"); + var parameter = new SKParameterMetadata("test") { Schema = schema }; + + // Act + var result = parameter.ParseJsonSchema(); + + // Assert + Assert.Equal(pair.Value, result.ParameterType); + Assert.Null(result.Schema); + } + } + + [Fact] + public void ReturnsParameterWithSchemaForNonPrimitiveOrStringSchemaType() + { + // Arrange + var schema = SKJsonSchema.Parse("{\"type\": \"object\", \"properties\": {\"name\": {\"type\": \"string\"}}}"); + var parameter = new SKParameterMetadata("test") { Schema = schema }; + + // Act + var result = parameter.ParseJsonSchema(); + + // Assert + Assert.Null(result.ParameterType); + Assert.Equal(schema, result.Schema); + } + + [Fact] + public void ReturnsIndentedJsonStringForJsonElement() + { + // Arrange + var jsonProperties = SKJsonSchema.Parse("{\"name\": \"Alice\", \"age\": 25}").RootElement; + + // Act + var result = jsonProperties.ToJsonString(); + + // Ensure that the line endings are consistent across different dotnet versions + result = result.Replace("\r\n", "\n", StringComparison.InvariantCulture); + + // Assert + var expected = "{\n \"name\": \"Alice\",\n \"age\": 25\n}"; + Assert.Equal(expected, result); + } + + [Fact] + public void ReturnsParameterNameAndSchemaType() + { + // Arrange + var schema = SKJsonSchema.Parse("{\"type\": \"object\", \"properties\": {\"name\": {\"type\": \"string\"}}}"); + var parameter = new SKParameterMetadata("test") { Schema = schema }; + + // Act + var result = parameter.GetSchemaTypeName(); + + // Assert + Assert.Equal("test-object", result); + } + + [Fact] + public void ConvertsReturnParameterMetadataToParameterMetadata() + { + // Arrange + var schema = SKJsonSchema.Parse("{\"type\": \"object\", \"properties\": {\"name\": {\"type\": \"string\"}}}"); + var returnParameter = new SKReturnParameterMetadata() { Description = "test", ParameterType = typeof(object), Schema = schema }; + + // Act + var functionName = "Foo"; + var result = returnParameter.ToSKParameterMetadata(functionName); + + // Assert + Assert.Equal("FooReturns", result.Name); + Assert.Equal("test", result.Description); + Assert.Equal(typeof(object), result.ParameterType); + Assert.Equal(schema, result.Schema); + } + + [Fact] + public void ConvertsParameterMetadataToReturnParameterMetadata() + { + // Arrange + var schema = SKJsonSchema.Parse("{\"type\": \"object\", \"properties\": {\"name\": {\"type\": \"string\"}}}"); + var parameter = new SKParameterMetadata("test") { Description = "test", ParameterType = typeof(object), Schema = schema }; + + // Act + var result = parameter.ToSKReturnParameterMetadata(); + + // Assert + Assert.Equal("test", result.Description); + Assert.Equal(typeof(object), result.ParameterType); + Assert.Equal(schema, result.Schema); + } + + #region Simple helper classes + + private sealed class SimpleClass + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + } + + private sealed class AnotherClass + { + public double Value { get; set; } + } + + private static class NestedClass + { + public static int Id { get; set; } + public static SimpleClass Simple { get; set; } = new SimpleClass(); + public static AnotherClass Another { get; set; } = new AnotherClass(); + } + + #endregion +} diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/skPrompt.handlebars b/dotnet/src/Planners/Planners.Core/Handlebars/CreatePlanPrompt.handlebars similarity index 50% rename from dotnet/src/Planners/Planners.Core/Handlebars/skPrompt.handlebars rename to dotnet/src/Planners/Planners.Core/Handlebars/CreatePlanPrompt.handlebars index 92afb8ab36b6..7da5b7fcb83e 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/skPrompt.handlebars +++ b/dotnet/src/Planners/Planners.Core/Handlebars/CreatePlanPrompt.handlebars @@ -2,19 +2,23 @@ Explain how to achieve the user's goal with the available helpers with a Handlebars template. ## Example -If the user wanted you to generate 10 random numbers and use them in another helper, you could answer with the following.{{/message}} +If the user wanted you to {{#if allowLoops}}generate 10 random numbers and use them in another helper{{else}}return the sum of 3 numbers{{/if}}, you could answer with the following.{{/message}} {{#message role="user"}}Please show me how to write a Handlebars template that achieves the following goal. -## Goal -I want you to generate 10 random numbers and send them to another helper.{{/message}} +{{#if allowLoops}}## Goal +I want you to generate 10 random numbers and send them to another helper. +{{else}}## Goal +What's the sum of 5+10+15? +{{~/if}}{{/message}} {{#message role="assistant"}}Here's the Handlebars template that achieves the goal: ```handlebars -{{!-- Step 1: initialize the count --}} +{{#if allowLoops}} +\{{!-- Step 1: Initialize the count --}} \{{set "count" 10 }} -{{!-- Step 2: loop using the count --}} +\{{!-- Step 2: Loop using the count --}} \{{#each (range 1 @@ -27,14 +31,16 @@ I want you to generate 10 random numbers and send them to another helper.{{/mess "index" this }} - {{!-- Step 3: create random number --}} + \{{!-- Step 3: Create random number --}} \{{set "randomNumber" - (example.random - seed=index + (Example{{reservedNameDelimiter}}Random + seed=(get + "index" + ) ) }} - {{!-- Step 4: call example helper with random number and print the result to the screen --}} + \{{!-- Step 4: Call example helper with random number and print the result to the screen --}} \{{json (Example{{reservedNameDelimiter}}Helper input=(get @@ -43,6 +49,42 @@ I want you to generate 10 random numbers and send them to another helper.{{/mess ) }} \{{/each}} +{{else}} +\{{!-- Step 1: Initialize the variables --}} +\{{set + "num1" + 5 +}} +\{{set + "num2" + 10 +}} +\{{set + "num3" + 15 +}} +\{{!-- Step 2: Call the Example-AddNums helper with the variables and store the result --}} +\{{set + "sum" + (Example{{reservedNameDelimiter}}AddNums + num1=(get + "num1" + ) + num2=(get + "num2" + ) + num3=(get + "num3" + ) + ) +}} +\{{!-- Step 3: Print the result using the json helper --}} +\{{json + (get + "sum" + ) +}} +{{/if}} ```{{/message}} {{#message role="system"}}Now let's try the real thing.{{/message}} {{#message role="user"}}Please show me how to write a Handlebars template that achieves the following goal with the available helpers. @@ -52,72 +94,87 @@ I want you to generate 10 random numbers and send them to another helper.{{/mess ## Out-of-the-box helpers The following helpers are available to you: -- {{{{raw}}}}`{{#if}}{{/if}}`{{{{/raw}}}} -- {{{{raw}}}}`{{#unless}}{{/unless}}}`{{{{/raw}}}} -- {{{{raw}}}}`{{#each}}{{/each}}`{{{{/raw}}}} -- {{{{raw}}}}`{{#with}}{{/with}}`{{{{/raw}}}} +- `\{{#if}}\{{/if}}` +- `\{{#unless}}\{{/unless}}`{{#if allowLoops}} +- `\{{#each}}\{{/each}}`{{/if}} +- `\{{#with}}\{{/with}}` +{{#if allowLoops}} ## Loop helpers If you need to loop through a list of values with `\{{#each}}`, you can use the following helpers: -- {{{{raw}}}}`{{range}}`{{{{/raw}}}} – Generates a sequence of integral numbers within a specified range, inclusive of last value. -- {{{{raw}}}}`{{array}}`{{{{/raw}}}} – Generates an array of values from the given values. +- `\{{range}}` – Generates a sequence of integral numbers within a specified range, inclusive of last value. +- `\{{array}}` – Generates an array of values from the given values. -IMPORTANT: `range` and `array` are the _only_ supported data structures. Others like `hash` are not supported. Also, you cannot use any methods or properties on the built-in data structures, such as `array.push` or `range.length`. +IMPORTANT: `range` and `array` are the only supported data structures. Others like `hash` are not supported. Also, you cannot use any methods or properties on the built-in data structures, such as `array.push` or `range.length`. -# Math helpers +## Math helpers If you need to do basic operations, you can use these two helpers with numerical values: -- {{{{raw}}}}`{{add}}`{{{{/raw}}}} – Adds two values together. -- {{{{raw}}}}`{{subtract}}`{{{{/raw}}}} – Subtracts the second value from the first. +- `\{{add}}` – Adds two values together. +- `\{{subtract}}` – Subtracts the second value from the first. +{{/if}} ## Comparison helpers -If you need to compare two values, you can use the following helpers: -- {{{{raw}}}}`{{equal}}`{{{{/raw}}}} -- {{{{raw}}}}`{{lessThan}}`{{{{/raw}}}} -- {{{{raw}}}}`{{greaterThan}}`{{{{/raw}}}} -- {{{{raw}}}}`{{lessThanOrEqual}}`{{{{/raw}}}} -- {{{{raw}}}}`{{greaterThanOrEqual}}`{{{{/raw}}}} - -To use the math and comparison helpers, you must pass in two positional values. For example, to check if the variable `var` is less than the number `1`, you would use the following helper like so: `\{{#if (lessThan var 1)}}\{{/if}}`. +If you need to compare two values, you can use the `\{{equal}}` helper. +To use the {{#if allowLoops}}math and {{/if}}comparison helpers, you must pass in two positional values. For example, to check if the variable `var` is equal to number `1`, you would use the following helper like so: `\{{#if (equal var 1)}}\{{/if}}`. ## Variable helpers If you need to create or retrieve a variable, you can use the following helpers: -- {{{{raw}}}}`{{set}}`{{{{/raw}}}} – Creates a variable with the given name and value. It does not print anything to the template, so you must use `\{{json}}` to print the value. -- {{{{raw}}}}`{{get}}`{{{{/raw}}}} – Retrieves the value of a variable with the given name. -- {{{{raw}}}}`{{json}}`{{{{/raw}}}} – Generates a JSON string from the given value; no need to use on strings. -- {{{{raw}}}}`{{concat}}`{{{{/raw}}}} – Concatenates the given values into a string. +- `\{{set}}` – Creates a variable with the given name and value. It does not print anything to the template, so you must use `\{{json}}` to print the value. +- `\{{get}}` – Retrieves the value of a variable with the given name. +- `\{{json}}` – Generates a JSON string from the given value; no need to use on strings. +- `\{{concat}}` – Concatenates the given values into a string. -{{#if complexTypeDefinitions}} +{{#if (or complexTypeDefinitions complexSchemaDefinitions)}} ## Complex types -Before we get to custom helpers, let's learn about complex objects. Some helpers require arguments that are complex objects. The JSON schemas for these complex objects are defined below. If a helper requires a argument with a complex type, use the `set` helper to store the value before using it or one of its properties in another step. +Some helpers require arguments that are complex objects. The JSON schemas for these complex objects are defined below: {{#each complexTypeDefinitions}} ### {{Name}}: -```json { -{{#each Properties}} - "{{Name}}": {{Type.Name}}, -{{/each}} + "type": "object", + "properties": { + {{#each Properties}} + "{{Name}}": { + "type": "{{ParameterType.Name}}", + }, + {{/each}} + } } -``` {{/each}} -{{/if}} +{{#each complexSchemaDefinitions}} +### {{@key}}: +{{this}} +{{/each}} +{{/if}} ## Custom helpers -Lastly, you also have the following Handlebars helpers that you can use to accomplish my goal. +Lastly, you also have the following Handlebars helpers that you can use: {{#each functions}} ### `{{doubleOpen}}{{PluginName}}{{../reservedNameDelimiter}}{{Name}}{{doubleClose}}` Description: {{Description}} Inputs: {{#each Parameters}} - - {{Name}}: {{#if Type}}{{Type.Name}} - {{/if}}{{Description}} {{#if IsRequired}}(required){{else}}(optional){{/if}} + - {{Name}}: + {{~#if ParameterType}} {{ParameterType.Name}} - + {{~else}} + {{~#if Schema}} {{getSchemaTypeName this}} -{{/if}} + {{~/if}} + {{~#if Description}} {{Description}}{{/if}} + {{~#if IsRequired}} (required){{else}} (optional){{/if}} {{/each}} -{{!-- TODO (@teresaqhoang): support return type {{ReturnType.Name}} {{../reservedNameDelimiter}} {{ReturnDescription}}--}} -Output: string - The result of the helper. +Output: +{{~#if ReturnParameter}} + {{~#if ReturnParameter.ParameterType}} {{ReturnParameter.ParameterType.Name}} + {{~else}} + {{~#if ReturnParameter.Schema}} {{getSchemaReturnTypeName ReturnParameter}} + {{else}} string{{/if}} + {{~/if}} + {{~#if ReturnParameter.Description}} - {{ReturnParameter.Description}}{{/if}} +{{/if}} {{/each}} - IMPORTANT: You can only use the helpers that are listed above. Do not use any other helpers that are not listed here. For example, do not use `\{{log}}` or any `\{{Example}}` helpers, as they are not supported.{{/message}} {{#message role="system"}} ## Tips and tricks @@ -127,7 +184,7 @@ IMPORTANT: You can only use the helpers that are listed above. Do not use any ot - Do not make up values. Use the helpers to generate the data you need or extract it from the goal. - Keep data well-defined. Each variable should have a unique name. Create and assign each variable only once. - Be extremely careful about types. For example, if you pass an array to a helper that expects a number, the template will error out. -- Avoid using loops. Try a solution without loops before you deploy a loop. +{{#if allowLoops}}- Avoid using loops. Try a solution without before you deploy a loop.{{/if}} - There is no need to check your results in the template. - Do not nest sub-expressions or helpers because it will cause the template to error out. - Each step should contain only one helper call. diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerExtensions.cs b/dotnet/src/Planners/Planners.Core/Handlebars/Extensions/HandlebarsPlannerExtensions.cs similarity index 96% rename from dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerExtensions.cs rename to dotnet/src/Planners/Planners.Core/Handlebars/Extensions/HandlebarsPlannerExtensions.cs index e5c0d91ac42f..23e4db9c7a79 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerExtensions.cs +++ b/dotnet/src/Planners/Planners.Core/Handlebars/Extensions/HandlebarsPlannerExtensions.cs @@ -4,7 +4,9 @@ using System.IO; using System.Reflection; +#pragma warning disable IDE0130 // Namespace does not match folder structure namespace Microsoft.SemanticKernel.Planning.Handlebars; +#pragma warning restore IDE0130 /// /// Extension methods for the interface. diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/Extensions/SKParameterMetadataExtensions.cs b/dotnet/src/Planners/Planners.Core/Handlebars/Extensions/SKParameterMetadataExtensions.cs new file mode 100644 index 000000000000..23a14114dde7 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Handlebars/Extensions/SKParameterMetadataExtensions.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Threading.Tasks; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planning.Handlebars; +#pragma warning restore IDE0130 + +internal static class SKParameterMetadataExtensions +{ + /// + /// Checks if type is primitive or string + /// + public static bool IsPrimitiveOrStringType(Type type) => type.IsPrimitive || type == typeof(string); + + /// + /// Checks if stringified type is primitive or string + /// + public static bool IsPrimitiveOrStringType(string type) => + type == "string" || type == "number" || type == "integer" || type == "boolean"; + + /// + /// Converts non-primitive types to a data class definition and returns a hash set of complex type metadata. + /// Complex types will become a data class. + /// If there are nested complex types, the nested complex type will also be returned. + /// Example: + /// Complex type: + /// class ComplexType: + /// propertyA: int + /// propertyB: str + /// propertyC: PropertyC + /// + public static HashSet ToHandlebarsParameterTypeMetadata(this Type type) + { + var parameterTypes = new HashSet(); + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Task<>)) + { + // Async return type - need to extract the actual return type + var actualReturnType = type.GenericTypeArguments[0]; // Actual Return Type + var returnTypeProperties = actualReturnType.GetProperties(); + + if (!IsPrimitiveOrStringType(actualReturnType) && returnTypeProperties.Length is not 0) + { + parameterTypes.Add(new HandlebarsParameterTypeMetadata() + { + Name = actualReturnType.Name, + IsComplex = true, + Properties = returnTypeProperties.Select(p => new SKParameterMetadata(p.Name) { ParameterType = p.PropertyType }).ToList() + }); + + parameterTypes.AddNestedComplexTypes(returnTypeProperties); + } + } + else if (type.IsClass && type != typeof(string)) + { + // Class + var properties = type.GetProperties(); + + parameterTypes.Add(new HandlebarsParameterTypeMetadata() + { + Name = type.Name, + IsComplex = properties.Length is not 0, + Properties = properties.Select(p => new SKParameterMetadata(p.Name) { ParameterType = p.PropertyType }).ToList() + }); + + parameterTypes.AddNestedComplexTypes(properties); + } + + return parameterTypes; + } + + private static void AddNestedComplexTypes(this HashSet parameterTypes, PropertyInfo[] properties) + { + // Add nested complex types + foreach (var property in properties) + { + parameterTypes.UnionWith(property.PropertyType.ToHandlebarsParameterTypeMetadata()); + } + } + + private static Type GetTypeFromSchema(string schemaType) + { + var typeMap = new Dictionary + { + {"string", typeof(string)}, + {"integer", typeof(long)}, + {"number", typeof(double)}, + {"boolean", typeof(bool)}, + {"object", typeof(object)}, + {"array", typeof(object[])}, + // If type is null, default to object + {"null", typeof(object)} + }; + + return typeMap[schemaType]; + } + + public static SKParameterMetadata ParseJsonSchema(this SKParameterMetadata parameter) + { + var schema = parameter.Schema!; + var type = schema.RootElement.GetProperty("type").GetString() ?? "object"; + if (IsPrimitiveOrStringType(type) || type == "null") + { + return new(parameter) + { + ParameterType = GetTypeFromSchema(type), + Schema = null + }; + } + + return parameter; + } + + public static string ToJsonString(this JsonElement jsonProperties) + { + var options = new JsonSerializerOptions() + { + WriteIndented = true, + }; + + return JsonSerializer.Serialize(jsonProperties, options); + } + + public static string GetSchemaTypeName(this SKParameterMetadata parameter) + { + var schemaType = parameter.Schema is not null && parameter.Schema.RootElement.TryGetProperty("type", out var typeElement) ? typeElement.ToString() : "object"; + return $"{parameter.Name}-{schemaType}"; + } + + public static SKParameterMetadata ToSKParameterMetadata(this SKReturnParameterMetadata parameter, string functionName) => new($"{functionName}Returns") + { + Description = parameter.Description, + ParameterType = parameter.ParameterType, + Schema = parameter.Schema + }; + + public static SKReturnParameterMetadata ToSKReturnParameterMetadata(this SKParameterMetadata parameter) => new() + { + Description = parameter.Description, + ParameterType = parameter.ParameterType, + Schema = parameter.Schema + }; +} diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsParameterTypeView.cs b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsParameterTypeView.cs deleted file mode 100644 index 05fb4f3f7f7b..000000000000 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsParameterTypeView.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json.Serialization; - -namespace Microsoft.SemanticKernel.Planning.Handlebars; - -internal sealed class HandlebarsParameterTypeView -{ - [JsonPropertyName("name")] - public string Name { get; set; } = string.Empty; - - [JsonPropertyName("isComplexType")] - public bool IsComplexType { get; set; } = false; - - /// - /// If this is a complex type, this will contain the properties of the complex type. - /// - [JsonPropertyName("properties")] - public List Properties { get; set; } = new(); -} - -internal static class ParameterTypeExtensions -{ - /// - /// Converts a type to a data class definition. - /// Primitive types will stay a primitive type. - /// Complex types will become a data class. - /// If there are nested complex types, the nested complex type will also be returned. - /// Example: - /// Primitive type: - /// System.String -> string - /// Complex type: - /// class ComplexType: - /// propertyA: int - /// propertyB: str - /// propertyC: PropertyC - /// - public static HashSet ToParameterTypes(this Type type) - { - var parameterTypes = new HashSet(); - - if (type.IsPrimitive || type == typeof(string)) - { - // Primitive types and string - parameterTypes.Add(new HandlebarsParameterTypeView() - { - Name = type.Name, - }); - } - else if (type.IsEnum) - { - // Enum - parameterTypes.Add(new HandlebarsParameterTypeView() - { - Name = type.Name, - }); - } - else if (type.IsClass) - { - // Class - var properties = type.GetProperties(); - - parameterTypes.Add(new HandlebarsParameterTypeView() - { - Name = type.Name, - IsComplexType = true, - Properties = properties.Select(p => new SKParameterMetadata(p.Name) { ParameterType = p.PropertyType }).ToList() - }); - - // Add nested complex types - foreach (var property in properties) - { - var propertyParameterTypes = property.PropertyType.ToParameterTypes(); - foreach (var propertyParameterType in propertyParameterTypes) - { - if (propertyParameterType.IsComplexType) - { - parameterTypes.Add(propertyParameterType); - } - } - } - } - - return parameterTypes; - } -} diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlan.cs b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlan.cs index 40264fcc7ac7..9d2394c67700 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlan.cs +++ b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlan.cs @@ -4,6 +4,7 @@ using System.Threading; using Microsoft.SemanticKernel.Orchestration; +#pragma warning disable IDE0130 // Namespace does not match folder structure namespace Microsoft.SemanticKernel.Planning.Handlebars; /// @@ -11,21 +12,38 @@ namespace Microsoft.SemanticKernel.Planning.Handlebars; /// public sealed class HandlebarsPlan { + /// + /// The kernel instance. + /// private readonly Kernel _kernel; + + /// + /// The handlebars template representing the plan. + /// private readonly string _template; + /// + /// Gets the prompt template used to generate the plan. + /// + public string Prompt { get; } + /// /// Initializes a new instance of the class. /// - /// The kernel. - /// The Handlebars template. - public HandlebarsPlan(Kernel kernel, string template) + /// Kernel instance. + /// A Handlebars template representing the generated plan. + /// Prompt template used to generate the plan. + public HandlebarsPlan(Kernel kernel, string generatedPlan, string createPlanPromptTemplate) { this._kernel = kernel; - this._template = template; + this._template = generatedPlan; + this.Prompt = createPlanPromptTemplate; } - /// + /// + /// Print the generated plan, aka handlebars template that was the create plan chat completion result. + /// + /// Handlebars template representing the plan. public override string ToString() { return this._template; diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlanner.cs b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlanner.cs index d9ef41d84dd3..b6f8aa6988e8 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlanner.cs +++ b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlanner.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.ChatCompletion; +#pragma warning disable IDE0130 // Namespace does not match folder structure namespace Microsoft.SemanticKernel.Planning.Handlebars; /// @@ -17,11 +18,6 @@ namespace Microsoft.SemanticKernel.Planning.Handlebars; /// public sealed class HandlebarsPlanner { - /// - /// The key for the available kernel functions. - /// - public const string AvailableKernelFunctionsKey = "AVAILABLE_KERNEL_FUNCTIONS"; - /// /// Gets the stopwatch used for measuring planning time. /// @@ -32,8 +28,6 @@ public sealed class HandlebarsPlanner private readonly HandlebarsPlannerConfig _config; - private readonly HashSet _parameterTypes = new(); - /// /// Initializes a new instance of the class. /// @@ -65,23 +59,21 @@ public Task CreatePlanAsync(string goal, CancellationToken cance private async Task CreatePlanCoreAsync(string goal, CancellationToken cancellationToken = default) { - var availableFunctions = this.GetAvailableFunctionsManual(cancellationToken); - var handlebarsTemplate = this.GetHandlebarsTemplate(this._kernel, goal, availableFunctions); + var availableFunctions = this.GetAvailableFunctionsManual(out var complexParameterTypes, out var complexParameterSchemas, cancellationToken); + var createPlanPrompt = this.GetHandlebarsTemplate(this._kernel, goal, availableFunctions, complexParameterTypes, complexParameterSchemas); var chatCompletion = this._kernel.GetService(); // Extract the chat history from the rendered prompt string pattern = @"<(user~|system~|assistant~)>(.*?)<\/\1>"; - MatchCollection matches = Regex.Matches(handlebarsTemplate, pattern, RegexOptions.Singleline); + MatchCollection matches = Regex.Matches(createPlanPrompt, pattern, RegexOptions.Singleline); // Add the chat history to the chat - ChatHistory chatMessages = this.GetChatHistoryFromPrompt(handlebarsTemplate, chatCompletion); + ChatHistory chatMessages = this.GetChatHistoryFromPrompt(createPlanPrompt, chatCompletion); // Get the chat completion results - var completionResults = await chatCompletion.GetChatCompletionsAsync(chatMessages, cancellationToken: cancellationToken).ConfigureAwait(false); - var completionMessage = await completionResults[0].GetChatMessageAsync(cancellationToken).ConfigureAwait(false); - + var completionResults = await chatCompletion.GenerateMessageAsync(chatMessages, cancellationToken: cancellationToken).ConfigureAwait(false); var resultContext = this._kernel.CreateNewContext(); - resultContext.Variables.Update(completionMessage.Content); + resultContext.Variables.Update(completionResults); if (resultContext.Variables.Input.IndexOf("Additional helpers may be required", StringComparison.OrdinalIgnoreCase) >= 0) { @@ -95,26 +87,91 @@ private async Task CreatePlanCoreAsync(string goal, Cancellation throw new SKException("Could not find the plan in the results"); } - var template = match.Groups[2].Value.Trim(); // match.Success ? match.Groups[2].Value.Trim() : resultContext.Result; + var planTemplate = match.Groups[2].Value.Trim(); - template = template.Replace("compare.equal", "equal"); - template = template.Replace("compare.lessThan", "lessThan"); - template = template.Replace("compare.greaterThan", "greaterThan"); - template = template.Replace("compare.lessThanOrEqual", "lessThanOrEqual"); - template = template.Replace("compare.greaterThanOrEqual", "greaterThanOrEqual"); - template = template.Replace("compare.greaterThanOrEqual", "greaterThanOrEqual"); + planTemplate = planTemplate.Replace("compare.equal", "equal"); + planTemplate = planTemplate.Replace("compare.lessThan", "lessThan"); + planTemplate = planTemplate.Replace("compare.greaterThan", "greaterThan"); + planTemplate = planTemplate.Replace("compare.lessThanOrEqual", "lessThanOrEqual"); + planTemplate = planTemplate.Replace("compare.greaterThanOrEqual", "greaterThanOrEqual"); + planTemplate = planTemplate.Replace("compare.greaterThanOrEqual", "greaterThanOrEqual"); - template = MinifyHandlebarsTemplate(template); - return new HandlebarsPlan(this._kernel, template); + planTemplate = MinifyHandlebarsTemplate(planTemplate); + return new HandlebarsPlan(this._kernel, planTemplate, createPlanPrompt); } - private List GetAvailableFunctionsManual(CancellationToken cancellationToken = default) + private List GetAvailableFunctionsManual( + out HashSet complexParameterTypes, + out Dictionary complexParameterSchemas, + CancellationToken cancellationToken = default) { - return this._kernel.Plugins.GetFunctionsMetadata() + complexParameterTypes = new(); + complexParameterSchemas = new(); + var availableFunctions = this._kernel.Plugins.GetFunctionsMetadata() .Where(s => !this._config.ExcludedPlugins.Contains(s.PluginName, StringComparer.OrdinalIgnoreCase) && !this._config.ExcludedFunctions.Contains(s.Name, StringComparer.OrdinalIgnoreCase) && !s.Name.Contains("Planner_Excluded")) .ToList(); + + var functionsMetadata = new List(); + foreach (var skFunction in availableFunctions) + { + // Extract any complex parameter types for isolated render in prompt template + var parametersMetadata = new List(); + foreach (var parameter in skFunction.Parameters) + { + var paramToAdd = this.SetComplexTypeDefinition(parameter, complexParameterTypes, complexParameterSchemas); + parametersMetadata.Add(paramToAdd); + } + + var returnParameter = skFunction.ReturnParameter.ToSKParameterMetadata(skFunction.Name); + returnParameter = this.SetComplexTypeDefinition(returnParameter, complexParameterTypes, complexParameterSchemas); + + // Need to override function metadata in case parameter metadata changed (e.g., converted primitive types from schema objects) + var functionMetadata = new SKFunctionMetadata(skFunction.Name) + { + PluginName = skFunction.PluginName, + Description = skFunction.Description, + Parameters = parametersMetadata, + ReturnParameter = returnParameter.ToSKReturnParameterMetadata() + }; + functionsMetadata.Add(functionMetadata); + } + + return functionsMetadata; + } + + // Extract any complex types or schemas for isolated render in prompt template + private SKParameterMetadata SetComplexTypeDefinition( + SKParameterMetadata parameter, + HashSet complexParameterTypes, + Dictionary complexParameterSchemas) + { + // TODO (@teresaqhoang): Handle case when schema and ParameterType can exist i.e., when ParameterType = RestApiResponse + if (parameter.ParameterType is not null) + { + // Async return type - need to extract the actual return type and override ParameterType property + var type = parameter.ParameterType; + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Task<>)) + { + parameter = new(parameter) { ParameterType = type.GenericTypeArguments[0] }; // Actual Return Type + } + + complexParameterTypes.UnionWith(parameter.ParameterType.ToHandlebarsParameterTypeMetadata()); + } + else if (parameter.Schema is not null) + { + // Parse the schema to extract any primitive types and set in ParameterType property instead + var parsedParameter = parameter.ParseJsonSchema(); + if (parsedParameter.Schema is not null) + { + complexParameterSchemas[parameter.GetSchemaTypeName()] = parameter.Schema.RootElement.ToJsonString(); + } + + parameter = parsedParameter; + } + + return parameter; } private ChatHistory GetChatHistoryFromPrompt(string prompt, IChatCompletion chatCompletion) @@ -147,15 +204,21 @@ private ChatHistory GetChatHistoryFromPrompt(string prompt, IChatCompletion chat return chatMessages; } - private string GetHandlebarsTemplate(Kernel kernel, string goal, List availableFunctions) + private string GetHandlebarsTemplate( + Kernel kernel, string goal, + List availableFunctions, + HashSet complexParameterTypes, + Dictionary complexParameterSchemas) { - var plannerTemplate = this.ReadPrompt("skPrompt.handlebars", this._config.AllowLoops ? null : "NoLoops"); + var plannerTemplate = this.ReadPrompt("CreatePlanPrompt.handlebars"); var variables = new Dictionary() { { "functions", availableFunctions}, { "goal", goal }, { "reservedNameDelimiter", HandlebarsTemplateEngineExtensions.ReservedNameDelimiter}, - { "complexTypeDefinitions", this._parameterTypes.Count > 0 && this._parameterTypes.Any(p => p.IsComplexType) ? this._parameterTypes.Where(p => p.IsComplexType) : null}, + { "allowLoops", this._config.AllowLoops }, + { "complexTypeDefinitions", complexParameterTypes.Count > 0 && complexParameterTypes.Any(p => p.IsComplex) ? complexParameterTypes.Where(p => p.IsComplex) : null}, + { "complexSchemaDefinitions", complexParameterSchemas.Count > 0 ? complexParameterSchemas : null}, { "lastPlan", this._config.LastPlan }, { "lastError", this._config.LastError } }; diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerConfig.cs b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerConfig.cs index 80e0090546a5..65b09ad7283c 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerConfig.cs +++ b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsPlannerConfig.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable IDE0130 // Namespace does not match folder structure namespace Microsoft.SemanticKernel.Planning.Handlebars; /// diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsTemplateEngineExtensions.cs b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsTemplateEngineExtensions.cs index c96a96dbcfb3..c32178660cba 100644 --- a/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsTemplateEngineExtensions.cs +++ b/dotnet/src/Planners/Planners.Core/Handlebars/HandlebarsTemplateEngineExtensions.cs @@ -10,6 +10,7 @@ using HandlebarsDotNet.Compiler; using Microsoft.SemanticKernel.Orchestration; +#pragma warning disable IDE0130 // Namespace does not match folder structure namespace Microsoft.SemanticKernel.Planning.Handlebars; /// @@ -66,11 +67,11 @@ private static void RegisterFunctionAsHelper( Kernel kernel, SKContext executionContext, IHandlebars handlebarsInstance, - SKFunctionMetadata functionView, + SKFunctionMetadata functionMetadata, Dictionary variables, CancellationToken cancellationToken = default) { - string fullyResolvedFunctionName = functionView.PluginName + ReservedNameDelimiter + functionView.Name; + string fullyResolvedFunctionName = functionMetadata.PluginName + ReservedNameDelimiter + functionMetadata.Name; handlebarsInstance.RegisterHelper(fullyResolvedFunctionName, (in HelperOptions options, in Context context, in Arguments arguments) => { @@ -79,75 +80,23 @@ private static void RegisterFunctionAsHelper( { if (arguments[0].GetType() == typeof(HashParameterDictionary)) { - // Process hash arguments - var handlebarArgs = arguments[0] as IDictionary; - - // Prepare the input parameters for the function - foreach (var param in functionView.Parameters) - { - var fullyQualifiedParamName = functionView.Name + ReservedNameDelimiter + param.Name; - var value = handlebarArgs != null && (handlebarArgs.TryGetValue(param.Name, out var val) || handlebarArgs.TryGetValue(fullyQualifiedParamName, out val)) ? val : null; - - if (value != null && (handlebarArgs?.ContainsKey(param.Name) == true || handlebarArgs?.ContainsKey(fullyQualifiedParamName) == true)) - { - variables[param.Name] = value; - } - else if (param.IsRequired) - { - throw new SKException($"Parameter {param.Name} is required for function {functionView.Name}."); - } - } + ProcessHashArguments(functionMetadata, variables, arguments[0] as IDictionary); } else { - // Process positional arguments - var requiredParameters = functionView.Parameters.Where(p => p.IsRequired).ToList(); - if (arguments.Length >= requiredParameters.Count && arguments.Length <= functionView.Parameters.Count) - { - var argIndex = 0; - foreach (var arg in arguments) - { - var param = functionView.Parameters[argIndex]; - if (IsExpectedParameterType(param, arg)) - { - variables[param.Name] = arguments[argIndex]; - argIndex++; - } - else - { - throw new SKException($"Invalid parameter type for function {functionView.Name}. Parameter {param.Name} expects type {param.ParameterType ?? (object?)param.Schema} but received {arguments[argIndex].GetType()}."); - } - } - } - else - { - throw new SKException($"Invalid parameter count for function {functionView.Name}. {arguments.Length} were specified but {functionView.Parameters.Count} are required."); - } + ProcessPositionalArguments(functionMetadata, variables, arguments); } } - - foreach (var v in variables) + else if (functionMetadata.Parameters.Any(p => p.IsRequired)) { - var varString = v.Value?.ToString() ?? ""; - if (executionContext.Variables.TryGetValue(v.Key, out var argVal)) - { - executionContext.Variables[v.Key] = varString; - } - else - { - executionContext.Variables.Add(v.Key, varString); - } + throw new SKException($"Invalid parameter count for function {functionMetadata.Name}. {arguments.Length} were specified but {functionMetadata.Parameters.Count} are required."); } - // TODO (@teresaqhoang): Add model results to execution context + test possible deadlock scenario - KernelFunction function = kernel.Plugins.GetFunction(functionView.PluginName, functionView.Name); + InitializeContextVariables(variables, executionContext); + KernelFunction function = kernel.Plugins.GetFunction(functionMetadata.PluginName, functionMetadata.Name); -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits - FunctionResult result = kernel.RunAsync(executionContext.Variables, cancellationToken, function).GetAwaiter().GetResult(); -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - - // Write the result to the template - return result.GetValue(); + // Invoke the function and write the result to the template + return InvokeSKFunction(kernel, function, executionContext, cancellationToken); }); } @@ -156,6 +105,34 @@ private static void RegisterSystemHelpers( Dictionary variables ) { + // Not exposed as a helper to the model, used for initial prompt rendering only. + handlebarsInstance.RegisterHelper("or", (in HelperOptions options, in Context context, in Arguments arguments) => + { + var isAtLeastOneTruthy = false; + foreach (var arg in arguments) + { + if (arg is not null) + { + isAtLeastOneTruthy = true; + } + } + + return isAtLeastOneTruthy; + }); + + handlebarsInstance.RegisterHelper("getSchemaTypeName", (in HelperOptions options, in Context context, in Arguments arguments) => + { + SKParameterMetadata parameter = (SKParameterMetadata)arguments[0]; + return parameter.GetSchemaTypeName(); + }); + + handlebarsInstance.RegisterHelper("getSchemaReturnTypeName", (in HelperOptions options, in Context context, in Arguments arguments) => + { + SKReturnParameterMetadata parameter = (SKReturnParameterMetadata)arguments[0]; + var functionName = arguments[1].ToString(); + return parameter.ToSKParameterMetadata(functionName).GetSchemaTypeName(); + }); + handlebarsInstance.RegisterHelper("array", (in HelperOptions options, in Context context, in Arguments arguments) => { // Convert all the arguments to an array @@ -211,7 +188,7 @@ private static void RegisterSystemHelpers( object? left = arguments[0]; object? right = arguments[1]; - return left == right || (left != null && left.Equals(right)); + return left == right || (left is not null && left.Equals(right)); }); handlebarsInstance.RegisterHelper("lessThan", (in HelperOptions options, in Context context, in Arguments arguments) => @@ -351,13 +328,16 @@ private static double CastToNumber(object number) } } - /* - * Type check will pass if: - * Types are an exact match. - * Handlebar argument is any kind of numeric type if function parameter requires a numeric type. - * Handlebar argument type is an object (this covers complex types). - * Function parameter is a generic type. - */ + /// + /// Checks if handlebars argument is a valid type for the function parameter. + /// Must satisfy one of the following: + /// Types are an exact match. + /// Handlebar argument is any kind of numeric type if function parameter requires a numeric type. + /// Handlebar argument type is an object (this covers complex types). + /// Function parameter is a generic type. + /// + /// Function parameter type + /// Handlebar argument private static bool IsExpectedParameterType(SKParameterMetadata parameterType, object argument) { if (parameterType.ParameterType == argument.GetType() || @@ -374,4 +354,113 @@ private static bool IsExpectedParameterType(SKParameterMetadata parameterType, o parameterIsNumeric && (IsNumericType(argument?.GetType()) || TryParseAnyNumber(argument?.ToString())); } + + /// + /// Processes the hash arguments passed to a Handlebars helper function. + /// + /// SKFunctionMetadata for the function being invoked. + /// Dictionary of variables passed to the Handlebars template engine. + /// Dictionary of arguments passed to the Handlebars helper function. + /// Thrown when a required parameter is missing. + private static void ProcessHashArguments(SKFunctionMetadata functionMetadata, Dictionary variables, IDictionary? handlebarArgs) + { + // Prepare the input parameters for the function + foreach (var param in functionMetadata.Parameters) + { + var fullyQualifiedParamName = functionMetadata.Name + ReservedNameDelimiter + param.Name; + var value = handlebarArgs is not null && (handlebarArgs.TryGetValue(param.Name, out var val) || handlebarArgs.TryGetValue(fullyQualifiedParamName, out val)) ? val : null; + + if (value is not null && (handlebarArgs?.ContainsKey(param.Name) == true || handlebarArgs?.ContainsKey(fullyQualifiedParamName) == true)) + { + variables[param.Name] = value; + } + else if (param.IsRequired) + { + throw new SKException($"Parameter {param.Name} is required for function {functionMetadata.Name}."); + } + } + } + + /// + /// Processes the positional arguments passed to a Handlebars helper function. + /// + /// SKFunctionMetadata for the function being invoked. + /// Dictionary of variables passed to the Handlebars template engine. + /// Dictionary of arguments passed to the Handlebars helper function. + /// Thrown when a required parameter is missing. + private static void ProcessPositionalArguments(SKFunctionMetadata functionMetadata, Dictionary variables, Arguments handlebarArgs) + { + var requiredParameters = functionMetadata.Parameters.Where(p => p.IsRequired).ToList(); + if (handlebarArgs.Length >= requiredParameters.Count && handlebarArgs.Length <= functionMetadata.Parameters.Count) + { + var argIndex = 0; + foreach (var arg in handlebarArgs) + { + var param = functionMetadata.Parameters[argIndex]; + if (IsExpectedParameterType(param, arg)) + { + variables[param.Name] = handlebarArgs[argIndex]; + argIndex++; + } + else + { + throw new SKException($"Invalid parameter type for function {functionMetadata.Name}. Parameter {param.Name} expects type {param.ParameterType ?? (object?)param.Schema} but received {handlebarArgs[argIndex].GetType()}."); + } + } + } + else + { + throw new SKException($"Invalid parameter count for function {functionMetadata.Name}. {handlebarArgs.Length} were specified but {functionMetadata.Parameters.Count} are required."); + } + } + + /// + /// Initializes the variables in the SK function context with the variables maintained by the Handlebars template engine. + /// + /// Dictionary of variables passed to the Handlebars template engine. + /// The execution context of the SK function. + private static void InitializeContextVariables(Dictionary variables, SKContext executionContext) + { + foreach (var v in variables) + { + var value = v.Value ?? ""; + var varString = !SKParameterMetadataExtensions.IsPrimitiveOrStringType(value.GetType()) ? JsonSerializer.Serialize(value) : value.ToString(); + if (executionContext.Variables.TryGetValue(v.Key, out var argVal)) + { + executionContext.Variables[v.Key] = varString; + } + else + { + executionContext.Variables.Add(v.Key, varString); + } + } + } + + /// + /// Invokes an SK function and returns a typed result, if specified. + /// + private static object? InvokeSKFunction( + Kernel kernel, + KernelFunction function, + SKContext executionContext, + CancellationToken cancellationToken = default) + { +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits + FunctionResult result = function.InvokeAsync(kernel, executionContext, cancellationToken: cancellationToken).GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + + // If return type is complex, serialize the object so it can be deserialized with expected class properties. + // i.e., class properties can be different if JsonPropertyName = 'id' and class property is 'Id'. + var returnType = function.GetMetadata().ReturnParameter.ParameterType; + var resultAsObject = result.GetValue(); + + if (returnType is not null && !SKParameterMetadataExtensions.IsPrimitiveOrStringType(returnType)) + { + var serializedResult = JsonSerializer.Serialize(resultAsObject); + resultAsObject = JsonSerializer.Deserialize(serializedResult, returnType); + } + + // TODO (@teresaqhoang): Add model results to execution context + test possible deadlock scenario + return resultAsObject; + } } diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/Models/HandlebarsParameterTypeMetadata.cs b/dotnet/src/Planners/Planners.Core/Handlebars/Models/HandlebarsParameterTypeMetadata.cs new file mode 100644 index 000000000000..34112a16e163 --- /dev/null +++ b/dotnet/src/Planners/Planners.Core/Handlebars/Models/HandlebarsParameterTypeMetadata.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +#pragma warning disable IDE0130 // Namespace does not match folder structure +namespace Microsoft.SemanticKernel.Planning.Handlebars; +#pragma warning restore IDE0130 + +internal class HandlebarsParameterTypeMetadata +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("isComplexType")] + public bool IsComplex { get; set; } = false; + + /// + /// If this is a complex type, this will contain the properties of the complex type. + /// + [JsonPropertyName("properties")] + public List Properties { get; set; } = new(); + + // Override the Equals method to compare the property values + public override bool Equals(object obj) + { + // Check to make sure the object is the expected type + if (obj is not HandlebarsParameterTypeMetadata other) + { + return false; + } + + // Compare the Name and IsComplex properties + if (this.Name != other.Name || this.IsComplex != other.IsComplex) + { + return false; + } + + // Compare the Properties lists using a helper method + return ArePropertiesEqual(this.Properties, other.Properties); + } + + // A helper method to compare two lists of SKParameterMetadata + private static bool ArePropertiesEqual(List list1, List list2) + { + // Check if the lists are null or have different lengths + if (list1 == null || list2 == null || list1.Count != list2.Count) + { + return false; + } + + // Compare the elements of the lists by comparing the Name and ParameterType properties + for (int i = 0; i < list1.Count; i++) + { + if (!list1[i].Name.Equals(list2[i].Name, System.StringComparison.Ordinal) || !list1[i].ParameterType!.Equals(list2[i].ParameterType)) + { + return false; + } + } + + // If all elements are equal, return true + return true; + } + + // Override the GetHashCode method to generate a hash code based on the property values + public override int GetHashCode() + { + HashCode hash = default; + hash.Add(this.Name); + hash.Add(this.IsComplex); + foreach (var item in this.Properties) + { + // Combine the Name and ParameterType properties into one hash code + hash.Add( + HashCode.Combine(item.Name, item.ParameterType) + ); + } + + return hash.ToHashCode(); + } +} diff --git a/dotnet/src/Planners/Planners.Core/Handlebars/NoLoops/skPrompt.handlebars b/dotnet/src/Planners/Planners.Core/Handlebars/NoLoops/skPrompt.handlebars deleted file mode 100644 index 95dd5ed9acdb..000000000000 --- a/dotnet/src/Planners/Planners.Core/Handlebars/NoLoops/skPrompt.handlebars +++ /dev/null @@ -1,130 +0,0 @@ -{{#message role="system"}}## Instructions -Explain how to achieve the user's goal with the available helpers with a Handlebars template. - -## Example -If the user wanted you to generate 10 random numbers and use them in another helper, you could answer with the following.{{/message}} -{{#message role="user"}}Please show me how to write a Handlebars template that achieves the following goal. - -## Goal -I want you to generate 10 random numbers and send them to another helper.{{/message}} -{{#message role="assistant"}}Here's the Handlebars template that achieves the goal: -```handlebars -{{!-- Step 1: initialize the variables --}} -\{{set - "num1" - 5 -}} -\{{set - "num2" - 10 -}} -\{{set - "num3" - 15 -}} -{{!-- Step 2: call the Example-AddThreeNumbers helper with the variables and store the result --}} -\{{set - "sum" - (Example{{reservedNameDelimiter}}AddThreeNumbers - num1=(get - "num1" - ) - num2=(get - "num2" - ) - num3=(get - "num3" - ) - ) -}} -{{!-- Step 3: output the result using the json helper --}} -\{{json - (get - "sum" - ) -}} -```{{/message}} -{{#message role="system"}}Now let's try the real thing.{{/message}} -{{#message role="user"}}Please show me how to write a Handlebars template that achieves the following goal with the available helpers. - -## Goal -{{goal}} - -## Out-of-the-box helpers -The following helpers are available to you: -- {{{{raw}}}}`{{#if}}{{/if}}`{{{{/raw}}}} -- {{{{raw}}}}`{{#unless}}{{/unless}}}`{{{{/raw}}}} -- {{{{raw}}}}`{{#with}}{{/with}}`{{{{/raw}}}} - -## Comparison helpers -If you need to compare two values, you can use the following helpers: -- {{{{raw}}}}`{{equal}}`{{{{/raw}}}} -- {{{{raw}}}}`{{lessThan}}`{{{{/raw}}}} -- {{{{raw}}}}`{{greaterThan}}`{{{{/raw}}}} -- {{{{raw}}}}`{{lessThanOrEqual}}`{{{{/raw}}}} -- {{{{raw}}}}`{{greaterThanOrEqual}}`{{{{/raw}}}} - -To use the comparison helpers, you must pass in two positional values. For example, to check if the variable `var` is less than the number `1`, you would use the following helper like so: `\{{#if (lessThan var 1)}}\{{/if}}`. - -## Variable helpers -If you need to create or retrieve a variable, you can use the following helpers: -- {{{{raw}}}}`{{set}}`{{{{/raw}}}} – Creates a variable with the given name and value. It does not print anything to the template, so you must use `\{{json}}` to print the value. -- {{{{raw}}}}`{{get}}`{{{{/raw}}}} – Retrieves the value of a variable with the given name. -- {{{{raw}}}}`{{json}}`{{{{/raw}}}} – Generates a JSON string from the given value; no need to use on strings. -- {{{{raw}}}}`{{concat}}`{{{{/raw}}}} – Concatenates the given values into a string. - -{{#if complexTypeDefinitions}} -## Complex types -Before we get to custom helpers, let's learn about complex objects. Some helpers require arguments that are complex objects. The JSON schemas for these complex objects are defined below. If a helper requires a argument with a complex type, use the `set` helper to store the value before using it or one of its properties in another step. - -{{#each complexTypeDefinitions}} -### {{Name}}: -```json -{ -{{#each Properties}} - "{{Name}}": {{Type.Name}}, -{{/each}} -} -``` - -{{/each}} -{{/if}} - -## Custom helpers -Lastly, you also have the following Handlebars helpers that you can use to accomplish my goal. - -{{#each functions}} -### `{{doubleOpen}}{{PluginName}}{{../reservedNameDelimiter}}{{Name}}{{doubleClose}}` -Description: {{Description}} -Inputs: - {{#each Parameters}} - - {{Name}}: {{#if Type}}{{Type.Name}} - {{/if}}{{Description}} {{#if IsRequired}}(required){{else}}(optional){{/if}} - {{/each}} -{{!-- TODO (@teresaqhoang): support return type {{ReturnType.Name}} {{../reservedNameDelimiter}} {{ReturnDescription}}--}} -Output: string - The result of the helper. - -{{/each}} - -IMPORTANT: You can only use the helpers that are listed above. Do not use any other helpers that are not listed here. For example, do not use `\{{hash}}` or any `\{{Example}}` helpers, as they are not supported.{{/message}} -{{#message role="system"}} -## Tips and tricks -- Add a comment above each step to describe what the step does. -- Use the `\{{set}}` and `\{{get}}` helpers to save and retrieve the results of another helper so you can use it later in the template without wasting resources. -- There are no initial variables available to you. You must create them yourself using the `\{{set}}` helper and then access them using `\{{get}}`. -- Do not make up values. Use the helpers to generate the data you need or extract it from the goal. -- Keep data well-defined. Each variable should have a unique name. Create and assign each variable only once. -- Be extremely careful about types. For example, if you pass a boolean to a helper that expects a number, the template will error out. -- This template has no built-in support for data structures. Do not try to use helpers like `array` because they will cause the template to fail. -- There is no need to check your results in the template. -- Do not nest sub-expressions or helpers because it will cause the template to error out. -- Each step should contain only one helper call. - -## Start -Now take a deep breath and accomplish the task: -1. Be as efficient as possible while keeping the template readable and well-defined. -2. Do not use helpers that were not provided to you, and be especially careful to not assume or make up any helpers or operations that were not explicitly defined already. -3. If none of the available helpers can achieve the goal, respond with "Additional helpers may be required". -4. The first steps should always be to initialize any variables you need. -5. The template should use the \{{json}} helper at least once to output the result of the final step. -6. Don't forget to use the tips and tricks otherwise the template will not work. -7. Don't close the ``` handlebars block until you're done with all the steps.{{/message}} \ No newline at end of file diff --git a/dotnet/src/Planners/Planners.Core/Planners.Core.csproj b/dotnet/src/Planners/Planners.Core/Planners.Core.csproj index 9a144801c757..37490233ef8d 100644 --- a/dotnet/src/Planners/Planners.Core/Planners.Core.csproj +++ b/dotnet/src/Planners/Planners.Core/Planners.Core.csproj @@ -52,10 +52,7 @@ Always - - Always - - + Always