chore: Benchmark cap CPU profile to 100%

Cysharp · Sep 4, 2024 · 2ddb4df · 2ddb4df
1 parent 39e4d19
commit 2ddb4df
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 41 deletions.
diff --git a/perf/BenchmarkApp/PerformanceTest.Client/Program.cs b/perf/BenchmarkApp/PerformanceTest.Client/Program.cs
@@ -317,12 +317,12 @@ static IReadOnlyList<PerformanceResult> RemoveOutlinerByIQR(IReadOnlyList<Perfor
             var mean = data.Select(x => x.Latency.Mean).OrderBy(x => x).ToArray();
 
             // get outliner for rps
-            var lowerBoundRps = GetLowerBound(rps);
-            var upperBoundRps = GetUpperBound(rps);
+            var lowerBoundRps = OutlinerHelper.GetLowerBound(rps);
+            var upperBoundRps = OutlinerHelper.GetUpperBound(rps);
 
             // get outliner for mean
-            var lowerBoundMean = GetLowerBound(mean);
-            var upperBoundMean = GetUpperBound(mean);
+            var lowerBoundMean = OutlinerHelper.GetLowerBound(mean);
+            var upperBoundMean = OutlinerHelper.GetUpperBound(mean);
 
             // compute tuple in range
             var filteredData = data
@@ -332,33 +332,6 @@ static IReadOnlyList<PerformanceResult> RemoveOutlinerByIQR(IReadOnlyList<Perfor
 
             return filteredData;
         }
-
-        static double GetLowerBound(IReadOnlyList<double> sortedData)
-        {
-            var Q1 = GetPercentile(sortedData, 25);
-            var Q3 = GetPercentile(sortedData, 75);
-            var IQR = Q3 - Q1;
-            return Q1 - 1.5 * IQR;
-        }
-
-        static double GetUpperBound(IReadOnlyList<double> sortedData)
-        {
-            var Q1 = GetPercentile(sortedData, 25);
-            var Q3 = GetPercentile(sortedData, 75);
-            var IQR = Q3 - Q1;
-            return Q3 + 1.5 * IQR;
-        }
-
-        static double GetPercentile(IReadOnlyList<double> sortedData, double percentile)
-        {
-            var N = sortedData.Count;
-            var n = (N - 1) * percentile / 100.0 + 1;
-            if (n == 1) return sortedData[0];
-            if (n == N) return sortedData[N - 1];
-            var k = (int)Math.Floor(n) - 1;
-            var d = n - Math.Floor(n);
-            return sortedData[k] + d * (sortedData[k + 1] - sortedData[k]);
-        }
     }
 }
 

diff --git a/perf/BenchmarkApp/PerformanceTest.Server/ProfileService.cs b/perf/BenchmarkApp/PerformanceTest.Server/ProfileService.cs
@@ -26,6 +26,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             var result = hardwarehardwarePerformanceReporter.GetResultAndClear();
             await datadog.PutServerBenchmarkMetricsAsync(ApplicationInformation.Current, result);
         }
+
+        hardwarehardwarePerformanceReporter.Stop();
     }
 }
 

diff --git a/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/HardwarePerformanceReporter.cs b/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/HardwarePerformanceReporter.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using System.Diagnostics;
 
 namespace PerformanceTest.Shared.Reporting;
@@ -8,8 +9,8 @@ public class HardwarePerformanceReporter
     private readonly TimeProvider timeProvider;
     private readonly Process currentProcess;
     private readonly int cpuCores;
-    private readonly List<double> cpuUsages;
-    private readonly List<double> memoryUsages;
+    private readonly ConcurrentBag<double> cpuUsages;
+    private readonly ConcurrentBag<double> memoryUsages;
     private CancellationTokenSource cancellationTokenSource;
     private bool running;
 
@@ -22,15 +23,13 @@ public HardwarePerformanceReporter(TimeSpan samplingInterval)
         this.timeProvider = SystemTimeProvider.TimeProvider;
         currentProcess = Process.GetCurrentProcess();
         cpuCores = Environment.ProcessorCount;
-        cpuUsages = new List<double>(1000);
-        memoryUsages = new List<double>(1000);
+        cpuUsages = new ConcurrentBag<double>();
+        memoryUsages = new ConcurrentBag<double>();
         cancellationTokenSource = new CancellationTokenSource();
     }
 
     public void Start()
     {
-        if (running) return;
-
         running = true;
         var prevMemory = currentProcess.WorkingSet64;
         Task.Run(async () =>
@@ -45,8 +44,8 @@ public void Start()
                 if (cancellationTokenSource.IsCancellationRequested) break;
 
                 // end
-                var duration = timeProvider.GetElapsedTime(start);
                 TimeSpan endCpuTime = currentProcess.TotalProcessorTime;
+                var duration = timeProvider.GetElapsedTime(start);
 
                 // CPU usage
                 var cpuUsedMs = (endCpuTime - startCpuTime).TotalMilliseconds;
@@ -66,13 +65,13 @@ public void Stop()
         running = false;
         cancellationTokenSource.Cancel();
         cancellationTokenSource.Dispose();
-        cancellationTokenSource = new CancellationTokenSource();
     }
 
     public HardwarePerformanceResult GetResult()
     {
-        var maxCpuUsage = cpuUsages.Count > 0 ? cpuUsages.Max() : 0d;
-        var avgCpuUsage = cpuUsages.Count > 0 ? cpuUsages.Average() : 0d;
+        var filteredCpuUsages = OutlinerHelper.RemoveOutlinerByIQR(cpuUsages.ToArray(), 100.0);
+        var maxCpuUsage = cpuUsages.Count > 0 ? filteredCpuUsages.Max() : 0d;
+        var avgCpuUsage = cpuUsages.Count > 0 ? filteredCpuUsages.Average() : 0d;
         var maxMemoryUsage = memoryUsages.Count > 0 ? memoryUsages.Max() / 1024 / 1024 : 0d;
         var avgMemoryUsage = memoryUsages.Count > 0 ? memoryUsages.Average() / 1024 / 1024 : 0d;
 

diff --git a/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/OutlinerHelper.cs b/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/OutlinerHelper.cs
@@ -0,0 +1,55 @@
+namespace PerformanceTest.Shared.Reporting;
+
+public static partial class OutlinerHelper
+{
+    public static IReadOnlyList<double> RemoveOutlinerByIQR(IReadOnlyList<double> data, double upperLimit)
+    {
+        if (data.Count == 0)
+            return data;
+
+        // sort rps, latency.mean
+        var sirted = data.OrderBy(x => x).ToArray();
+
+        // get outliner for rps
+        var lowerBound = GetLowerBound(sirted);
+        var upperBound = GetUpperBound(sirted);
+
+        // compute tuple in range
+        var filteredData = data
+            .Where(x => x >= lowerBound && x <= upperBound)
+            .Where(x => x <= upperLimit)
+            .ToArray();
+
+        return filteredData;
+    }
+
+    internal static double GetLowerBound(IReadOnlyList<double> sortedData)
+    {
+        var Q1 = GetPercentile(sortedData, 25);
+        var Q3 = GetPercentile(sortedData, 75);
+        var IQR = Q3 - Q1;
+        return Q1 - 1.5 * IQR;
+    }
+
+    internal static double GetUpperBound(IReadOnlyList<double> sortedData)
+    {
+        if (sortedData.Count == 0)
+            return sortedData[0];
+
+        var Q1 = GetPercentile(sortedData, 25);
+        var Q3 = GetPercentile(sortedData, 75);
+        var IQR = Q3 - Q1;
+        return Q3 + 1.5 * IQR;
+    }
+
+    private static double GetPercentile(IReadOnlyList<double> sortedData, double percentile)
+    {
+        var N = sortedData.Count;
+        var n = (N - 1) * percentile / 100.0 + 1;
+        if (n == 1) return sortedData[0];
+        if (n == N) return sortedData[N - 1];
+        var k = (int)Math.Floor(n) - 1;
+        var d = n - Math.Floor(n);
+        return sortedData[k] + d * (sortedData[k + 1] - sortedData[k]);
+    }
+}
-Original file line number
+Diff line change
@@ Expand Up @@
                 var result = hardwarehardwarePerformanceReporter.GetResultAndClear();
                 await datadog.PutServerBenchmarkMetricsAsync(ApplicationInformation.Current, result);
             }
+            hardwarehardwarePerformanceReporter.Stop();
         }
     }
@@ Expand Down @@