From d756b18c915459a338adae265ddf00d0f18ee143 Mon Sep 17 00:00:00 2001 From: Sanders Lauture Date: Tue, 3 Dec 2024 01:33:43 -0800 Subject: [PATCH] Update to latest Azure AI packages Packages with vulnerabilities are bad. --- .../Controllers/AdminPageController.cs | 1 - .../Controllers/UploadController.cs | 58 +++++++++++++------ SeattleCarsInBikeLanes/Program.cs | 20 ++++--- .../SeattleCarsInBikeLanes.csproj | 17 +++--- .../Models/AbstractPhotoUploadMetadata.cs | 10 +++- .../Models/FinalizedPhotoUploadMetadata.cs | 4 +- .../Models/InitialPhotoUploadMetadata.cs | 4 +- 7 files changed, 72 insertions(+), 42 deletions(-) diff --git a/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs b/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs index 42df51f..ddf2159 100644 --- a/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs +++ b/SeattleCarsInBikeLanes/Controllers/AdminPageController.cs @@ -20,7 +20,6 @@ using LinqToTwitter.OAuth; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; using Newtonsoft.Json; using SeattleCarsInBikeLanes.Database; using SeattleCarsInBikeLanes.Database.Models; diff --git a/SeattleCarsInBikeLanes/Controllers/UploadController.cs b/SeattleCarsInBikeLanes/Controllers/UploadController.cs index d777571..54efc9c 100644 --- a/SeattleCarsInBikeLanes/Controllers/UploadController.cs +++ b/SeattleCarsInBikeLanes/Controllers/UploadController.cs @@ -1,5 +1,7 @@ using System.Diagnostics; using System.Text; +using Azure.AI.ContentSafety; +using Azure.AI.Vision.ImageAnalysis; using Azure.Maps.Search; using Azure.Maps.Search.Models; using Azure.Storage.Blobs; @@ -10,8 +12,6 @@ using LinqToTwitter; using LinqToTwitter.OAuth; using Microsoft.AspNetCore.Mvc; -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision; -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; using Microsoft.Azure.Cosmos.Spatial; using SeattleCarsInBikeLanes.Providers; using SeattleCarsInBikeLanes.Storage.Models; @@ -29,7 +29,8 @@ public class UploadController : ControllerBase new Position(-122.436522, 47.495082), new Position(-122.235787, 47.735525)); private readonly ILogger logger; - private readonly ComputerVisionClient computerVisionClient; + private readonly ImageAnalysisClient imageAnalysisClient; + private readonly ContentSafetyClient contentSafetyClient; private readonly MapsSearchClient mapsSearchClient; private readonly BlobContainerClient blobContainerClient; private readonly MastodonClientProvider mastodonClientProvider; @@ -37,7 +38,8 @@ public class UploadController : ControllerBase private readonly HelperMethods helperMethods; public UploadController(ILogger logger, - ComputerVisionClient computerVisionClient, + ImageAnalysisClient imageAnalysisClient, + ContentSafetyClient contentSafetyClient, MapsSearchClient mapsSearchClient, BlobContainerClient blobContainerClient, MastodonClientProvider mastodonClientProvider, @@ -45,7 +47,8 @@ public UploadController(ILogger logger, HelperMethods helperMethods) { this.logger = logger; - this.computerVisionClient = computerVisionClient; + this.imageAnalysisClient = imageAnalysisClient; + this.contentSafetyClient = contentSafetyClient; this.mapsSearchClient = mapsSearchClient; this.blobContainerClient = blobContainerClient; this.mastodonClientProvider = mastodonClientProvider; @@ -165,13 +168,10 @@ private async Task ProcessInitialUpload(IF tempFile = tempFile2; using var fileStream2 = System.IO.File.OpenRead(tempFile); - List visualFeatureTypes = new List() - { - VisualFeatureTypes.Tags, - VisualFeatureTypes.Adult - }; + Task> contentSafetyAnalyzeImageTask = contentSafetyClient.AnalyzeImageAsync(BinaryData.FromStream(fileStream2)); + fileStream2.Seek(0, SeekOrigin.Begin); + Task> analyzeImageTask = imageAnalysisClient.AnalyzeAsync(BinaryData.FromStream(fileStream2), VisualFeatures.Tags); - Task imageAnalysisResultsTask = computerVisionClient.AnalyzeImageInStreamAsync(fileStream2, visualFeatureTypes); Task? crossStreetItemTask = null; if (photoLocation != null) { @@ -179,16 +179,40 @@ private async Task ProcessInitialUpload(IF } // No idea why this is not detected as null free since we're doing the filter - await Task.WhenAll(new List() { imageAnalysisResultsTask, crossStreetItemTask }.Where(t => t != null)!); + await Task.WhenAll(new List() { crossStreetItemTask, contentSafetyAnalyzeImageTask, analyzeImageTask }.Where(t => t != null)!); fileStream2.Dispose(); - ImageAnalysis imageAnalysisResults = imageAnalysisResultsTask.Result; - if (imageAnalysisResults.Adult.IsAdultContent || imageAnalysisResults.Adult.IsGoryContent || imageAnalysisResults.Adult.IsRacyContent) + AnalyzeImageResult contentSafetyAnalyzeImageResult = contentSafetyAnalyzeImageTask.Result.Value; + ImageCategoriesAnalysis? hateCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Hate); + ImageCategoriesAnalysis? selfHarmCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.SelfHarm); + ImageCategoriesAnalysis? sexualCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Sexual); + ImageCategoriesAnalysis? violenceCategory = contentSafetyAnalyzeImageResult.CategoriesAnalysis.FirstOrDefault(c => c.Category == ImageCategory.Violence); + if ((hateCategory != null && hateCategory.Severity > 2) || + (selfHarmCategory != null && selfHarmCategory.Severity > 2) || + (sexualCategory != null && sexualCategory.Severity > 2) || + (violenceCategory != null && violenceCategory.Severity > 2)) { + logger.LogWarning($"Photo does not pass content check. Analysis results: " + + $"Hate: {hateCategory?.Severity}, Self Harm: {selfHarmCategory?.Severity}, Sexual: {sexualCategory?.Severity}, Violence: {violenceCategory?.Severity}"); throw new BikeLaneException("Error: Photo does not pass content check."); } + ImageAnalysisResult analyzeImageResult = analyzeImageTask.Result.Value; + static List AzureTagToImageTag(IReadOnlyList tags) + { + List imageTags = new List(); + foreach (var tag in tags) + { + imageTags.Add(new ImageTag() + { + Name = tag.Name, + Confidence = tag.Confidence + }); + } + return imageTags; + } + ReverseSearchCrossStreetAddressResultItem? crossStreetItem = null; string? crossStreet = null; if (crossStreetItemTask != null) @@ -215,25 +239,25 @@ private async Task ProcessInitialUpload(IF photoLocation.Latitude.ToString("#.#####"), photoLocation.Longitude.ToString("#.#####"), crossStreet, - imageAnalysisResults.Tags.ToList()); + AzureTagToImageTag(analyzeImageResult.Tags.Values)); } else { metadata = new InitialPhotoUploadMetadata(randomFileName, submissionId, index, - imageAnalysisResults.Tags.ToList()); + AzureTagToImageTag(analyzeImageResult.Tags.Values)); } BlobClient photoBlobClient = blobContainerClient.GetBlobClient($"{InitialUploadPrefix}{randomFileName}.jpeg"); using var fileStream3 = System.IO.File.OpenRead(tempFile); Task uploadPhotoTask = photoBlobClient.UploadAsync(fileStream3); - fileStream3.Dispose(); BlobClient metadataBlobClient = blobContainerClient.GetBlobClient($"{InitialUploadPrefix}{randomFileName}.json"); Task uploadMetadataTask = metadataBlobClient.UploadAsync(new BinaryData(metadata)); await Task.WhenAll(uploadPhotoTask, uploadMetadataTask); + fileStream3.Dispose(); Uri sasUri = await photoBlobClient.GenerateUserDelegationReadOnlySasUri(DateTimeOffset.UtcNow.AddMinutes(10)); return InitialPhotoUploadWithSasUriMetadata.FromMetadata(sasUri.ToString(), metadata); diff --git a/SeattleCarsInBikeLanes/Program.cs b/SeattleCarsInBikeLanes/Program.cs index b496e6d..6dbecb2 100644 --- a/SeattleCarsInBikeLanes/Program.cs +++ b/SeattleCarsInBikeLanes/Program.cs @@ -1,5 +1,8 @@ using System.Security.Claims; using System.Security.Cryptography; +using Azure; +using Azure.AI.ContentSafety; +using Azure.AI.Vision.ImageAnalysis; using Azure.Identity; using Azure.Maps.Search; using Azure.Security.KeyVault.Secrets; @@ -13,7 +16,6 @@ using Imgur.API.Models; using LinqToTwitter; using LinqToTwitter.OAuth; -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision; using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Spatial; using SeattleCarsInBikeLanes.Database; @@ -155,12 +157,16 @@ public static void Main(string[] args) services.AddSingleton(c => { SecretClient client = c.GetRequiredService(); - KeyVaultSecret computerVisionTokenSecret = client.GetSecret("computervision"); - string computerVisionToken = computerVisionTokenSecret.Value; - return new ComputerVisionClient(new ApiKeyServiceClientCredentials(computerVisionToken), c.GetRequiredService(), false) - { - Endpoint = "https://seattlecarsinbikelanesvision.cognitiveservices.azure.com/" - }; + KeyVaultSecret imageAnalysisTokenSecret = client.GetSecret("computervision"); + string imageAnalysisToken = imageAnalysisTokenSecret.Value; + return new ImageAnalysisClient(new Uri("https://seattlecarsinbikelanesvision.cognitiveservices.azure.com/"), new AzureKeyCredential(imageAnalysisToken)); + }); + services.AddSingleton(c => + { + SecretClient client = c.GetRequiredService(); + KeyVaultSecret contentSafetyTokenSecret = client.GetSecret("contentsafety"); + string contentSafetyToken = contentSafetyTokenSecret.Value; + return new ContentSafetyClient(new Uri("https://carsinbikelanes-content-safety.cognitiveservices.azure.com/"), new AzureKeyCredential(contentSafetyToken)); }); services.AddSingleton(c => { diff --git a/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj b/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj index e839aa1..9c215c5 100644 --- a/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj +++ b/SeattleCarsInBikeLanes/SeattleCarsInBikeLanes.csproj @@ -15,23 +15,24 @@ - + + + - - + + - + - + - - + - + diff --git a/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs index b0f120f..2489232 100644 --- a/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs +++ b/SeattleCarsInBikeLanes/Storage/Models/AbstractPhotoUploadMetadata.cs @@ -1,6 +1,4 @@ -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; - -namespace SeattleCarsInBikeLanes.Storage.Models +namespace SeattleCarsInBikeLanes.Storage.Models { public abstract class AbstractPhotoUploadMetadata { @@ -68,4 +66,10 @@ public AbstractPhotoUploadMetadata(string photoId, Tags = tags; } } + + public class ImageTag + { + public string Name { get; set; } = default!; + public float Confidence { get; set; } = default!; + } } diff --git a/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs index 73a67cf..9ec9227 100644 --- a/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs +++ b/SeattleCarsInBikeLanes/Storage/Models/FinalizedPhotoUploadMetadata.cs @@ -1,6 +1,4 @@ -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; - -namespace SeattleCarsInBikeLanes.Storage.Models +namespace SeattleCarsInBikeLanes.Storage.Models { public class FinalizedPhotoUploadMetadata : AbstractPhotoUploadMetadata { diff --git a/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs b/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs index 702f6e6..9078c5e 100644 --- a/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs +++ b/SeattleCarsInBikeLanes/Storage/Models/InitialPhotoUploadMetadata.cs @@ -1,6 +1,4 @@ -using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; - -namespace SeattleCarsInBikeLanes.Storage.Models +namespace SeattleCarsInBikeLanes.Storage.Models { public class InitialPhotoUploadMetadata : AbstractPhotoUploadMetadata {