Update to latest Azure AI packages

Packages with vulnerabilities are bad.
golf1052 · Dec 3, 2024 · d756b18 · d756b18
1 parent 95143dc
commit d756b18
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 42 deletions.
diff --git a/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs b/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs
@@ -20,7 +20,6 @@
 using LinqToTwitter.OAuth;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models;
 using Newtonsoft.Json;
 using SeattleCarsInBikeLanes.Database;
 using SeattleCarsInBikeLanes.Database.Models;

diff --git a/SeattleCarsInBikeLanes/Controllers/UploadController.cs b/SeattleCarsInBikeLanes/Controllers/UploadController.cs
@@ -1,5 +1,7 @@
 using System.Diagnostics;
 using System.Text;
+using Azure.AI.ContentSafety;
+using Azure.AI.Vision.ImageAnalysis;
 using Azure.Maps.Search;
 using Azure.Maps.Search.Models;
 using Azure.Storage.Blobs;
@@ -10,8 +12,6 @@
 using LinqToTwitter;
 using LinqToTwitter.OAuth;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision;
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models;
 using Microsoft.Azure.Cosmos.Spatial;
 using SeattleCarsInBikeLanes.Providers;
 using SeattleCarsInBikeLanes.Storage.Models;
@@ -29,23 +29,26 @@ public class UploadController : ControllerBase
             new Position(-122.436522, 47.495082),
             new Position(-122.235787, 47.735525));
         private readonly ILogger<UploadController> logger;
-        private readonly ComputerVisionClient computerVisionClient;
+        private readonly ImageAnalysisClient imageAnalysisClient;
+        private readonly ContentSafetyClient contentSafetyClient;
         private readonly MapsSearchClient mapsSearchClient;
         private readonly BlobContainerClient blobContainerClient;
         private readonly MastodonClientProvider mastodonClientProvider;
         private readonly SlackbotProvider slackbotProvider;
         private readonly HelperMethods helperMethods;
 
         public UploadController(ILogger<UploadController> logger,
-            ComputerVisionClient computerVisionClient,
+            ImageAnalysisClient imageAnalysisClient,
+            ContentSafetyClient contentSafetyClient,
             MapsSearchClient mapsSearchClient,
             BlobContainerClient blobContainerClient,
             MastodonClientProvider mastodonClientProvider,
             SlackbotProvider slackbotProvider,
             HelperMethods helperMethods)
         {
             this.logger = logger;
-            this.computerVisionClient = computerVisionClient;
+            this.imageAnalysisClient = imageAnalysisClient;
+            this.contentSafetyClient = contentSafetyClient;
             this.mapsSearchClient = mapsSearchClient;
             this.blobContainerClient = blobContainerClient;
             this.mastodonClientProvider = mastodonClientProvider;
@@ -165,30 +168,51 @@ private async Task<InitialPhotoUploadWithSasUriMetadata> ProcessInitialUpload(IF
                 tempFile = tempFile2;
                 using var fileStream2 = System.IO.File.OpenRead(tempFile);
 
-                List<VisualFeatureTypes?> visualFeatureTypes = new List<VisualFeatureTypes?>()
-                {
-                    VisualFeatureTypes.Tags,
-                    VisualFeatureTypes.Adult
-                };
+                Task<Azure.Response<AnalyzeImageResult>> contentSafetyAnalyzeImageTask = contentSafetyClient.AnalyzeImageAsync(BinaryData.FromStream(fileStream2));
+                fileStream2.Seek(0, SeekOrigin.Begin);
+                Task<Azure.Response<ImageAnalysisResult>> analyzeImageTask = imageAnalysisClient.AnalyzeAsync(BinaryData.FromStream(fileStream2), VisualFeatures.Tags);
 
-                Task<ImageAnalysis> imageAnalysisResultsTask = computerVisionClient.AnalyzeImageInStreamAsync(fileStream2, visualFeatureTypes);
                 Task<ReverseSearchCrossStreetAddressResultItem?>? crossStreetItemTask = null;
                 if (photoLocation != null)
                 {
                     crossStreetItemTask = helperMethods.ReverseSearchCrossStreet(photoLocation, mapsSearchClient);
                 }
 
                 // No idea why this is not detected as null free since we're doing the filter
-                await Task.WhenAll(new List<Task?>() { imageAnalysisResultsTask, crossStreetItemTask }.Where(t => t != null)!);
+                await Task.WhenAll(new List<Task?>() { crossStreetItemTask, contentSafetyAnalyzeImageTask, analyzeImageTask }.Where(t => t != null)!);
 
                 fileStream2.Dispose();
 
-                ImageAnalysis imageAnalysisResults = imageAnalysisResultsTask.Result;
-                if (imageAnalysisResults.Adult.IsAdultContent || imageAnalysisResults.Adult.IsGoryContent || imageAnalysisResults.Adult.IsRacyContent)
+                AnalyzeImageResult contentSafetyAnalyzeImageResult = contentSafetyAnalyzeImageTask.Result.Value;
+                ImageCategoriesAnalysis? hateCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Hate);
+                ImageCategoriesAnalysis? selfHarmCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.SelfHarm);
+                ImageCategoriesAnalysis? sexualCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Sexual);
+                ImageCategoriesAnalysis? violenceCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Violence);
+                if ((hateCategory != null && hateCategory.Severity > 2) ||
+                    (selfHarmCategory != null && selfHarmCategory.Severity > 2) ||
+                    (sexualCategory != null && sexualCategory.Severity > 2) ||
+                    (violenceCategory != null && violenceCategory.Severity > 2))
                 {
+                    logger.LogWarning($"Photo does not pass content check. Analysis results: " +
+                    $"Hate: {hateCategory?.Severity}, Self Harm: {selfHarmCategory?.Severity}, Sexual: {sexualCategory?.Severity}, Violence: {violenceCategory?.Severity}");
                     throw new BikeLaneException("Error: Photo does not pass content check.");
                 }
 
+                ImageAnalysisResult analyzeImageResult = analyzeImageTask.Result.Value;
+                static List<ImageTag> AzureTagToImageTag(IReadOnlyList<DetectedTag> tags)
+                {
+                    List<ImageTag> imageTags = new List<ImageTag>();
+                    foreach (var tag in tags)
+                    {
+                        imageTags.Add(new ImageTag()
+                        {
+                            Name = tag.Name,
+                            Confidence = tag.Confidence
+                        });
+                    }
+                    return imageTags;
+                }
+
                 ReverseSearchCrossStreetAddressResultItem? crossStreetItem = null;
                 string? crossStreet = null;
                 if (crossStreetItemTask != null)
@@ -215,25 +239,25 @@ private async Task<InitialPhotoUploadWithSasUriMetadata> ProcessInitialUpload(IF
                         photoLocation.Latitude.ToString("#.#####"),
                         photoLocation.Longitude.ToString("#.#####"),
                         crossStreet,
-                        imageAnalysisResults.Tags.ToList());
+                        AzureTagToImageTag(analyzeImageResult.Tags.Values));
                 }
                 else
                 {
                     metadata = new InitialPhotoUploadMetadata(randomFileName,
                         submissionId,
                         index,
-                        imageAnalysisResults.Tags.ToList());
+                        AzureTagToImageTag(analyzeImageResult.Tags.Values));
                 }
 
                 BlobClient photoBlobClient = blobContainerClient.GetBlobClient($"{InitialUploadPrefix}{randomFileName}.jpeg");
                 using var fileStream3 = System.IO.File.OpenRead(tempFile);
                 Task uploadPhotoTask = photoBlobClient.UploadAsync(fileStream3);
-                fileStream3.Dispose();
 
                 BlobClient metadataBlobClient = blobContainerClient.GetBlobClient($"{InitialUploadPrefix}{randomFileName}.json");
                 Task uploadMetadataTask = metadataBlobClient.UploadAsync(new BinaryData(metadata));
 
                 await Task.WhenAll(uploadPhotoTask, uploadMetadataTask);
+                fileStream3.Dispose();
 
                 Uri sasUri = await photoBlobClient.GenerateUserDelegationReadOnlySasUri(DateTimeOffset.UtcNow.AddMinutes(10));
                 return InitialPhotoUploadWithSasUriMetadata.FromMetadata(sasUri.ToString(), metadata);

diff --git a/SeattleCarsInBikeLanes/Program.cs b/SeattleCarsInBikeLanes/Program.cs
@@ -1,5 +1,8 @@
 using System.Security.Claims;
 using System.Security.Cryptography;
+using Azure;
+using Azure.AI.ContentSafety;
+using Azure.AI.Vision.ImageAnalysis;
 using Azure.Identity;
 using Azure.Maps.Search;
 using Azure.Security.KeyVault.Secrets;
@@ -13,7 +16,6 @@
 using Imgur.API.Models;
 using LinqToTwitter;
 using LinqToTwitter.OAuth;
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision;
 using Microsoft.Azure.Cosmos;
 using Microsoft.Azure.Cosmos.Spatial;
 using SeattleCarsInBikeLanes.Database;
@@ -155,12 +157,16 @@ public static void Main(string[] args)
             services.AddSingleton(c =>
             {
                 SecretClient client = c.GetRequiredService<SecretClient>();
-                KeyVaultSecret computerVisionTokenSecret = client.GetSecret("computervision");
-                string computerVisionToken = computerVisionTokenSecret.Value;
-                return new ComputerVisionClient(new ApiKeyServiceClientCredentials(computerVisionToken), c.GetRequiredService<HttpClient>(), false)
-                {
-                    Endpoint = "https://seattlecarsinbikelanesvision.cognitiveservices.azure.com/"
-                };
+                KeyVaultSecret imageAnalysisTokenSecret = client.GetSecret("computervision");
+                string imageAnalysisToken = imageAnalysisTokenSecret.Value;
+                return new ImageAnalysisClient(new Uri("https://seattlecarsinbikelanesvision.cognitiveservices.azure.com/"), new AzureKeyCredential(imageAnalysisToken));
+            });
+            services.AddSingleton(c =>
+            {
+                SecretClient client = c.GetRequiredService<SecretClient>();
+                KeyVaultSecret contentSafetyTokenSecret = client.GetSecret("contentsafety");
+                string contentSafetyToken = contentSafetyTokenSecret.Value;
+                return new ContentSafetyClient(new Uri("https://carsinbikelanes-content-safety.cognitiveservices.azure.com/"), new AzureKeyCredential(contentSafetyToken));
             });
             services.AddSingleton(c =>
             {

diff --git a/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj b/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj
@@ -15,23 +15,24 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.12.1" />
+    <PackageReference Include="Azure.AI.ContentSafety" Version="1.0.0" />
+    <PackageReference Include="Azure.AI.Vision.ImageAnalysis" Version="1.0.0" />
+    <PackageReference Include="Azure.Identity" Version="1.13.1" />
     <PackageReference Include="Azure.Maps.Search" Version="1.0.0-beta.4" />
-    <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.22.1" />
+    <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.7.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
     <PackageReference Include="golf1052.atproto.net" Version="0.4.0" />
     <PackageReference Include="golf1052.Mastodon" Version="0.7.1" />
     <PackageReference Include="golf1052.ThreadsAPI" Version="0.2.0" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.67" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
     <PackageReference Include="idunno.Authentication.Basic" Version="2.4.0" />
     <PackageReference Include="Imgur.API" Version="5.0.0" />
     <PackageReference Include="linqtotwitter" Version="6.15.0" />
-    <PackageReference Include="Magick.NET-Q16-AnyCPU" Version="14.0.0" />
+    <PackageReference Include="Magick.NET-Q16-AnyCPU" Version="14.2.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
-    <PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.ComputerVision" Version="7.0.1" />
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.43.1" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.46.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.ServiceModel.Syndication" Version="8.0.0" />
+    <PackageReference Include="System.ServiceModel.Syndication" Version="9.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs
@@ -1,6 +1,4 @@
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models;
-
-namespace SeattleCarsInBikeLanes.Storage.Models
+namespace SeattleCarsInBikeLanes.Storage.Models
 {
     public abstract class AbstractPhotoUploadMetadata
     {
@@ -68,4 +66,10 @@ public AbstractPhotoUploadMetadata(string photoId,
             Tags = tags;
         }
     }
+
+    public class ImageTag
+    {
+        public string Name { get; set; } = default!;
+        public float Confidence { get; set; } = default!;
+    }
 }
diff --git a/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs
@@ -1,6 +1,4 @@
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models;
-
-namespace SeattleCarsInBikeLanes.Storage.Models
+namespace SeattleCarsInBikeLanes.Storage.Models
 {
     public class FinalizedPhotoUploadMetadata : AbstractPhotoUploadMetadata
     {

diff --git a/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs
@@ -1,6 +1,4 @@
-using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models;
-
-namespace SeattleCarsInBikeLanes.Storage.Models
+namespace SeattleCarsInBikeLanes.Storage.Models
 {
     public class InitialPhotoUploadMetadata : AbstractPhotoUploadMetadata
     {