Merge branch 'perf/tokenizer'

IngeniumSE · Nov 14, 2024 · f4742a3 · f4742a3
2 parents e854c53 + 18b9239
commit f4742a3
Show file tree

Hide file tree

Showing 46 changed files with 1,680 additions and 433 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -230,3 +230,13 @@ indent_size = 2
 end_of_line = lf
 [*.{cmd,bat}]
 end_of_line = crlf
+
+# Verify settings
+[*.{received,verified}.{json,txt,xml}]
+charset = "utf-8-bom"
+end_of_line = lf
+indent_size = unset
+indent_style = unset
+insert_final_newline = false
+tab_width = unset
+trim_trailing_whitespace = false
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+*.verified.txt text eol=lf working-tree-encoding=UTF-8
+*.verified.xml text eol=lf working-tree-encoding=UTF-8
+*.verified.json text eol=lf working-tree-encoding=UTF-8
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,12 @@
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
 
+# Verify received Files
+*.received.*
+
+# Profiler diagnostics
+*.diagsession
+
 # User-specific files
 *.rsuser
 *.suo
@@ -443,4 +449,4 @@ $RECYCLE.BIN/
 
 ## Ingenium Defaults
 artefacts/*
-**/appsettings.env.json
+**/appsettings.env.json
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -1,29 +1,29 @@
 <Project>
   <PropertyGroup>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
-
-		<!-- Microsoft.Extensions.* Ver -->
-		<DotNet8Version>8.0.0</DotNet8Version>
+    <!-- Microsoft.Extensions.* Ver -->
+    <DotNet8Version>8.0.0</DotNet8Version>
   </PropertyGroup>
-
   <ItemGroup>
-		<PackageVersion Include="Ben.Demystifier" Version="0.4.1" />
-		<PackageVersion Include="BenchmarkDotNet" Version="0.13.7" />
+    <PackageVersion Include="Ben.Demystifier" Version="0.4.1" />
+    <PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
     <PackageVersion Include="FluentValidation" Version="11.6.0" />
-		<PackageVersion Include="MinVer" Version="4.3.0" />
-		<PackageVersion Include="Microsoft.Extensions.Configuration" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Options" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(DotNet8Version)" />
-		<PackageVersion Include="Microsoft.Extensions.Http" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="9.0.0" />
+    <PackageVersion Include="MinVer" Version="4.3.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Options" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(DotNet8Version)" />
+    <PackageVersion Include="Microsoft.Extensions.Http" Version="$(DotNet8Version)" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="$(DotNet8Version)" />
-		<PackageVersion Include="System.Net.Http.Json" Version="$(DotNet8Version)" />
-		<PackageVersion Include="System.Text.Json" Version="8.0.4" />
-		<PackageVersion Include="xunit" Version="2.6.5" />
-		<PackageVersion Include="xunit.runner.visualstudio" Version="2.5.6" />
+    <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="$(DotNet8Version)" />
+    <PackageVersion Include="System.Net.Http.Json" Version="$(DotNet8Version)" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.4" />
+    <PackageVersion Include="Verify.Xunit" Version="28.2.1" />
+    <PackageVersion Include="xunit" Version="2.9.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
   </ItemGroup>
-</Project>
+</Project>
diff --git a/FuManchu.sln b/FuManchu.sln
@@ -41,6 +41,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FuManchu", "libs\FuManchu\F
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FuManchu.Tests", "tests\FuManchu.Tests\FuManchu.Tests.csproj", "{5657C855-7F41-4BBD-8F28-6158C9F4412B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "apps", "apps", "{7DF84E8F-934C-4AF3-9AA1-5465BF597E57}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FuManchu.Benckmarking", "apps\FuManchu.Benckmarking\FuManchu.Benckmarking.csproj", "{EAB25E3C-BB47-4D8B-B6E6-2770BA922448}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -63,6 +67,10 @@ Global
 		{5657C855-7F41-4BBD-8F28-6158C9F4412B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5657C855-7F41-4BBD-8F28-6158C9F4412B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5657C855-7F41-4BBD-8F28-6158C9F4412B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EAB25E3C-BB47-4D8B-B6E6-2770BA922448}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EAB25E3C-BB47-4D8B-B6E6-2770BA922448}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EAB25E3C-BB47-4D8B-B6E6-2770BA922448}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EAB25E3C-BB47-4D8B-B6E6-2770BA922448}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -74,6 +82,7 @@ Global
 		{97639CDE-C698-4FD7-995F-8AE44F1BF8E0} = {158819D8-F88C-4F77-890F-D0C2E4D56C08}
 		{83D92240-965A-4E84-8686-48EAAA153FE6} = {E48489AB-B880-4C98-92D3-4E171FB1735C}
 		{5657C855-7F41-4BBD-8F28-6158C9F4412B} = {37ABC77B-D881-471D-AF7A-1A5E8118D534}
+		{EAB25E3C-BB47-4D8B-B6E6-2770BA922448} = {7DF84E8F-934C-4AF3-9AA1-5465BF597E57}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {EFA92856-B041-4D18-B733-EC53980E9492}

diff --git a/apps/FuManchu.Benckmarking/FuManchu.Benckmarking.csproj b/apps/FuManchu.Benckmarking/FuManchu.Benckmarking.csproj
@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+	<PropertyGroup>
+		<OutputType>Exe</OutputType>
+		<TargetFramework>net8.0</TargetFramework>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<Nullable>enable</Nullable>
+		<Configurations>Debug;Release</Configurations>
+		<ImplicitPackageReferences>false</ImplicitPackageReferences>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="BenchmarkDotNet" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\..\libs\FuManchu\FuManchu.csproj" />
+	</ItemGroup>
+
+</Project>
diff --git a/apps/FuManchu.Benckmarking/Program.cs b/apps/FuManchu.Benckmarking/Program.cs
@@ -0,0 +1,37 @@
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+
+using FuManchu.Text;
+using FuManchu.Tokenizer;
+
+BenchmarkSwitcher.FromAssembly(typeof(Benchmarking).Assembly).Run(args);
+
+//Benchmarking b = new();
+//for (int i = 0; i < 10_000; i++)
+//{
+//	b.Test();
+//}
+
+[MemoryDiagnoser, ShortRunJob]
+public class Benchmarking
+{
+	[Benchmark]
+	public void Test()
+	{
+		const string Template = "{{#if Variable}}Hello World{{/if}}";
+
+		Baseline(Template);
+	}
+
+	void Baseline(string template)
+	{
+		using var reader = new StringReader(template);
+		using var source = new SeekableTextReader(reader);
+		using var tokenizer = new HandlebarsTokenizer(source);
+
+		while (tokenizer.NextSymbol() != null) { }
+	}
+}
diff --git a/libs/FuManchu/FuManchu.csproj b/libs/FuManchu/FuManchu.csproj
@@ -19,9 +19,11 @@
 		<PackageProjectUrl>https://github.com/IngeniumSE/FuManchu</PackageProjectUrl>
 		<Copyright>Copyright 2024 Ingenium Software Engineering</Copyright>
 		<ImplicitPackageReferences>false</ImplicitPackageReferences>
+		<Configurations>Debug;Release</Configurations>
 	</PropertyGroup>
 
 	<ItemGroup>
+		<PackageReference Include="Microsoft.Extensions.ObjectPool" />
 		<PackageReference Include="MinVer" />
 	</ItemGroup>