diff --git a/Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj b/Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj index 10adb42..f6d8359 100644 --- a/Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj +++ b/Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj @@ -1,28 +1,24 @@ - - + - net6.0 - + net9.0 false - - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - - PreserveNewest @@ -67,5 +63,4 @@ PreserveNewest - - + \ No newline at end of file diff --git a/Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs b/Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs index 8123fc5..6075735 100644 --- a/Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs +++ b/Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs @@ -1,7 +1,6 @@ using Caf.Midden.Cli.Services; using Caf.Midden.Core.Models.v0_2; using Caf.Midden.Core.Services.Metadata; -using NuGet.Frameworks; using System; using System.Collections.Generic; using System.Linq; diff --git a/Caf.Midden.Cli/Caf.Midden.Cli.csproj b/Caf.Midden.Cli/Caf.Midden.Cli.csproj index cc3cc05..b22bc66 100644 --- a/Caf.Midden.Cli/Caf.Midden.Cli.csproj +++ b/Caf.Midden.Cli/Caf.Midden.Cli.csproj @@ -1,36 +1,31 @@  - Exe - net6.0 + net9.0 true enable - 0.3.1.0 - 0.3.1.0 + 0.4.0 + 0.4.0 MiddenCli - 0.3.1 + 0.4-dev.0 https://github.com/cafincubator/midden MiddenCli true - - - - - + + + + + - - PreserveNewest - - - + \ No newline at end of file diff --git a/Caf.Midden.Cli/Services/AzureDataLakeCrawler.cs b/Caf.Midden.Cli/Services/AzureDataLakeCrawler.cs index ff2eb10..6cb7840 100644 --- a/Caf.Midden.Cli/Services/AzureDataLakeCrawler.cs +++ b/Caf.Midden.Cli/Services/AzureDataLakeCrawler.cs @@ -99,26 +99,33 @@ public List GetMetadatas( foreach (var fileName in fileNames) { - // Get file contents as json string - DataLakeFileClient fileClient = - fileSystemClient.GetFileClient(fileName); + try + { + // Get file contents as json string + DataLakeFileClient fileClient = + fileSystemClient.GetFileClient(fileName); - Response fileContents = fileClient.Read(); + Response fileContents = fileClient.Read(); - string json; - using (MemoryStream ms = new MemoryStream()) - { - fileContents.Value.Content.CopyTo(ms); - json = Encoding.UTF8.GetString(ms.ToArray()); - } + string json; + using (MemoryStream ms = new MemoryStream()) + { + fileContents.Value.Content.CopyTo(ms); + json = Encoding.UTF8.GetString(ms.ToArray()); + } - // Parse json string and add relative path to Dataset - Metadata metadata = parser.Parse(json); + // Parse json string and add relative path to Dataset + Metadata metadata = parser.Parse(json); - string filePath = fileClient.Path.Replace(MIDDEN_FILE_EXTENSION, ""); - metadata.Dataset.DatasetPath = filePath; + string filePath = fileClient.Path.Replace(MIDDEN_FILE_EXTENSION, ""); + metadata.Dataset.DatasetPath = filePath; - metadatas.Add(metadata); + metadatas.Add(metadata); + } + catch (Exception ex) + { + Console.WriteLine($"Error parsing file: {fileName}, reason: {ex}"); + } } return metadatas; diff --git a/Caf.Midden.Core.Tests/Caf.Midden.Core.Tests.csproj b/Caf.Midden.Core.Tests/Caf.Midden.Core.Tests.csproj index 05dd565..69119f6 100644 --- a/Caf.Midden.Core.Tests/Caf.Midden.Core.Tests.csproj +++ b/Caf.Midden.Core.Tests/Caf.Midden.Core.Tests.csproj @@ -1,29 +1,24 @@ - - + - net6.0 - + net9.0 false - - - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - - PreserveNewest @@ -50,5 +45,4 @@ PreserveNewest - - + \ No newline at end of file diff --git a/Caf.Midden.Core/Caf.Midden.Core.csproj b/Caf.Midden.Core/Caf.Midden.Core.csproj index 5f9c84a..1b03f16 100644 --- a/Caf.Midden.Core/Caf.Midden.Core.csproj +++ b/Caf.Midden.Core/Caf.Midden.Core.csproj @@ -1,11 +1,8 @@  - - net6.0 + net9.0 - - + - - + \ No newline at end of file diff --git a/Caf.Midden.Wasm/Caf.Midden.Wasm.csproj b/Caf.Midden.Wasm/Caf.Midden.Wasm.csproj index 72e8b24..5a1ba67 100644 --- a/Caf.Midden.Wasm/Caf.Midden.Wasm.csproj +++ b/Caf.Midden.Wasm/Caf.Midden.Wasm.csproj @@ -1,29 +1,27 @@ - - + - net6.0 + net9.0 service-worker-assets.js - 0.3 + 0.4.0 + 0.4.0 - - - - - - - - + + + + + + + + + - - - PreserveNewest @@ -59,11 +57,9 @@ PreserveNewest - PreserveNewest - diff --git a/Caf.Midden.Wasm/Pages/CatalogTags.razor b/Caf.Midden.Wasm/Pages/CatalogTags.razor index 503a5bb..05eebdc 100644 --- a/Caf.Midden.Wasm/Pages/CatalogTags.razor +++ b/Caf.Midden.Wasm/Pages/CatalogTags.razor @@ -2,12 +2,9 @@ @inject Services.StateContainer State
+ -

Coming soon...

-
- -@code { - -} \ No newline at end of file + + \ No newline at end of file diff --git a/Caf.Midden.Wasm/Pages/Index.razor b/Caf.Midden.Wasm/Pages/Index.razor index 4eab5b3..63ae089 100644 --- a/Caf.Midden.Wasm/Pages/Index.razor +++ b/Caf.Midden.Wasm/Pages/Index.razor @@ -1,4 +1,5 @@ @page "/" + @inject Services.StateContainer State
@@ -74,19 +75,20 @@ + + + Recent Datasets <small><a href="catalog/datasets"><Icon Type="double-right" Theme="outline" /></a></small> + + + - Recent Projects <small><a href="catalog/projects"><Icon Type="double-right" Theme="outline" /></a></small> + Recent Projects <small><a href="catalog/projects"><Icon Type="double-right" Theme="outline" /></a></small> + - Recent Datasets <small><a href="catalog/datasets"><Icon Type="double-right" Theme="outline" /></a></small> - - - - Stats for Nerds - @@ -170,8 +172,7 @@
- -@code{ +@code { private string DebugMsg { get; set; } private async Task AppConfig_StateChanged( @@ -185,6 +186,8 @@ } } + + //protected override void OnInitialized() //{ // State.StateChanged += async (source, property) @@ -196,4 +199,7 @@ // => await AppConfig_StateChanged(source, property); //} -} \ No newline at end of file +} + + + diff --git a/Caf.Midden.Wasm/Pages/Index.razor.cs b/Caf.Midden.Wasm/Pages/Index.razor.cs index c688e6a..f6564d1 100644 --- a/Caf.Midden.Wasm/Pages/Index.razor.cs +++ b/Caf.Midden.Wasm/Pages/Index.razor.cs @@ -31,6 +31,8 @@ public partial class Index : IDisposable IChartComponent MetadataPerZone = new Column(); public object[] MetadataPerZoneData { get; set; } + + ColumnConfig MetadataPerZoneConfig = new ColumnConfig { Title = new AntDesign.Charts.Title @@ -166,6 +168,7 @@ private void SetSimpleStats() this.TotalContacts = UniqueContacts.Count; } + private void CreateDatasetsPerZone() { List objs = new List(); @@ -189,6 +192,8 @@ private void CreateDatasetsPerZone() MetadataPerZone.ChangeData(MetadataPerZoneData); } + + private void CreateProjectsPerStatus() { List objs = new List(); diff --git a/Caf.Midden.Wasm/Services/StateContainer.cs b/Caf.Midden.Wasm/Services/StateContainer.cs index a9cb12e..6c5f8f9 100644 --- a/Caf.Midden.Wasm/Services/StateContainer.cs +++ b/Caf.Midden.Wasm/Services/StateContainer.cs @@ -62,10 +62,16 @@ public void UpdateCatalog( public StateContainer() { - this.AssemblyVersion = - Assembly.GetExecutingAssembly() + var informationalVersion = + Assembly.GetEntryAssembly() .GetCustomAttribute() .InformationalVersion; + + // Added because a GUID was appended to the end for unknown reasons + if (informationalVersion.Contains("+")) + informationalVersion = informationalVersion.Split("+")[0]; + + this.AssemblyVersion = informationalVersion; } public event Action StateChanged; diff --git a/Caf.Midden.Wasm/Shared/CatalogVariableViewer.razor.cs b/Caf.Midden.Wasm/Shared/CatalogVariableViewer.razor.cs index 5406f83..ea61930 100644 --- a/Caf.Midden.Wasm/Shared/CatalogVariableViewer.razor.cs +++ b/Caf.Midden.Wasm/Shared/CatalogVariableViewer.razor.cs @@ -154,16 +154,11 @@ private void SearchHandler() { ViewModel.FilteredCatalogVariables = ViewModel.CatalogVariables .Where(c => - (c.DatasetName.ToLower().Contains( - ViewModel.SearchTerm.ToLower())) || - (c.Name.ToLower().Contains( - ViewModel.SearchTerm.ToLower())) || - (c.Description.ToLower().Contains( - ViewModel.SearchTerm.ToLower())) || - (c.Units.ToLower().Contains( - ViewModel.SearchTerm.ToLower())) || - (c.Tags.Any(t => t.ToLower().Contains( - ViewModel.SearchTerm.ToLower())))) + (c.DatasetName != null && c.DatasetName.ToLower().Contains(ViewModel.SearchTerm.ToLower())) || + (c.Name != null && c.Name.ToLower().Contains(ViewModel.SearchTerm.ToLower())) || + (c.Description != null && c.Description.ToLower().Contains(ViewModel.SearchTerm.ToLower())) || + (c.Units != null && c.Units.ToLower().Contains(ViewModel.SearchTerm.ToLower())) || + (c.Tags != null && c.Tags.Any(t => t != null && t.ToLower().Contains(ViewModel.SearchTerm.ToLower())))) .ToList(); } } diff --git a/Caf.Midden.Wasm/Shared/FilteredCatalogMetadataViewer.razor b/Caf.Midden.Wasm/Shared/FilteredCatalogMetadataViewer.razor index e816bd8..2fa8011 100644 --- a/Caf.Midden.Wasm/Shared/FilteredCatalogMetadataViewer.razor +++ b/Caf.Midden.Wasm/Shared/FilteredCatalogMetadataViewer.razor @@ -27,24 +27,30 @@ @foreach (var metadata in FilteredMetadata) { - + + - + + + @metadata.Dataset.Name + + - + @metadata.Dataset.Zone + @metadata.Dataset.Project @@ -78,4 +84,5 @@ } else { -} \ No newline at end of file +} + diff --git a/Caf.Midden.Wasm/Shared/FilteredCatalogProjectViewer.razor b/Caf.Midden.Wasm/Shared/FilteredCatalogProjectViewer.razor index ab80f46..57c12a0 100644 --- a/Caf.Midden.Wasm/Shared/FilteredCatalogProjectViewer.razor +++ b/Caf.Midden.Wasm/Shared/FilteredCatalogProjectViewer.razor @@ -27,17 +27,21 @@ @foreach (var project in ViewModel.FilteredCatalogProjects) { - + - - + + @project.Name - @if(!string.IsNullOrEmpty(project.ProjectStatus)) - { - @project.ProjectStatus - } + + + @if (!string.IsNullOrEmpty(project.ProjectStatus)) + { + @project.ProjectStatus + } + + @project.DatasetCount @@ -54,4 +58,4 @@ } -} \ No newline at end of file +} diff --git a/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor b/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor new file mode 100644 index 0000000..cba212f --- /dev/null +++ b/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor @@ -0,0 +1,56 @@ +@inject Caf.Midden.Wasm.Services.StateContainer State + +@if (this.ShowHeader) +{ + Tags +} + +@if (State.Catalog != null && this.ShowSearch) +{ + + + + + + + +} + +@if (FilteredDatasetTags != null && FilteredVariableTags != null) +{ + + + +
Dataset Tags
+ + + + + @item.Key @item.Value + + + + + + + +
Variable Tags
+ + + + @item.Key @item.Value + + + + + + +} +else { + +} \ No newline at end of file diff --git a/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor.cs b/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor.cs new file mode 100644 index 0000000..eb9cce9 --- /dev/null +++ b/Caf.Midden.Wasm/Shared/FilteredCatalogTagViewer.razor.cs @@ -0,0 +1,114 @@ +using AntDesign; +using Caf.Midden.Core.Models.v0_2; +using Caf.Midden.Wasm.Shared.Modals; +using Markdig; +using Microsoft.AspNetCore.Components; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Caf.Midden.Wasm.Shared +{ + public partial class FilteredCatalogTagViewer : IDisposable + { + [Parameter] + public bool ShowSearch { get; set; } = true; + + [Parameter] + public bool ShowHeader { get; set; } = true; + + EmbeddedProperty Property(int span, int offset) => new() { Span = span, Offset = offset }; + + + Dictionary BaseDatasetTags { get; set; } + Dictionary BaseVariableTags { get; set; } + Dictionary FilteredDatasetTags { get; set; } + Dictionary FilteredVariableTags { get; set; } + + public string SearchTerm { get; set; } + + protected override void OnInitialized() + { + State.StateChanged += async (source, property) + => await StateChanged(source, property); + + if (State?.Catalog != null) + { + SetBaseTags(); + InitializeFilteredTags(); + } + } + + private async Task StateChanged( + ComponentBase source, + string property) + { + if (source != this) + { + if (property == "UpdateCatalog") + { + SetBaseTags(); + InitializeFilteredTags(); + } + + await InvokeAsync(StateHasChanged); + } + } + + private void SetBaseTags() + { + List DatasetTags = new List(); + List VariableTags = new List(); + + foreach (Metadata meta in State.Catalog.Metadatas) + { + // Get tags from all datasets and variables + DatasetTags = DatasetTags.Concat(meta.Dataset.Tags).ToList(); + VariableTags = VariableTags.Concat(meta.Dataset.Variables + .SelectMany(v => v.Tags)).ToList(); + } + + this.BaseDatasetTags = DatasetTags.GroupBy(s => s) + .ToDictionary(g => g.Key, g => g.Count()) + .OrderByDescending(d => d.Value) + .ToDictionary(d => d.Key, d => d.Value); + this.BaseVariableTags = VariableTags.GroupBy(s => s) + .ToDictionary(g => g.Key, g => g.Count()) + .OrderByDescending(d => d.Value) + .ToDictionary(d => d.Key, d => d.Value); + } + private void InitializeFilteredTags() + { + this.FilteredDatasetTags = this.BaseDatasetTags; + this.FilteredVariableTags = this.BaseVariableTags; + } + private void SearchHandler() + { + if (string.IsNullOrWhiteSpace(SearchTerm)) + { + InitializeFilteredTags(); + } + else + { + FilteredDatasetTags = this.BaseDatasetTags + .Where(t => t.Key.ToLower().Contains(SearchTerm.ToLower())) + .ToDictionary() + .OrderByDescending(d => d.Value) + .ToDictionary(); + + FilteredVariableTags = this.BaseVariableTags + .Where(t => t.Key.ToLower().Contains(SearchTerm.ToLower())) + .ToDictionary() + .OrderByDescending(d => d.Value) + .ToDictionary(); + } + } + + public void Dispose() + { + State.StateChanged -= async (source, property) + => await StateChanged(source, property); + } + } +} diff --git a/Caf.Midden.Wasm/Shared/MainLayout.razor b/Caf.Midden.Wasm/Shared/MainLayout.razor index cbb3286..3d69097 100644 --- a/Caf.Midden.Wasm/Shared/MainLayout.razor +++ b/Caf.Midden.Wasm/Shared/MainLayout.razor @@ -26,7 +26,7 @@ CollapsedWidth="0"> @@ -45,6 +45,10 @@ Projects + + + Tags + diff --git a/Caf.Midden.Wasm/Shared/MetadataDetails.razor b/Caf.Midden.Wasm/Shared/MetadataDetails.razor index 95fb791..c7d833f 100644 --- a/Caf.Midden.Wasm/Shared/MetadataDetails.razor +++ b/Caf.Midden.Wasm/Shared/MetadataDetails.razor @@ -1,4 +1,5 @@ -@inject Caf.Midden.Wasm.Services.StateContainer State +@using Caf.Midden.Wasm.Shared.MetadataLineage +@inject Caf.Midden.Wasm.Services.StateContainer State @if (this.Metadata != null) { @@ -59,18 +60,14 @@ - -
- Parent Datasets -
- - - - - - + + + Lineage Diagram + + + + + @@ -112,7 +109,14 @@ Variables + + + + Download + + + @if (Metadata.Dataset.Variables?.Count > 0) { diff --git a/Caf.Midden.Wasm/Shared/MetadataDetails.razor.cs b/Caf.Midden.Wasm/Shared/MetadataDetails.razor.cs index 661a686..a12cd29 100644 --- a/Caf.Midden.Wasm/Shared/MetadataDetails.razor.cs +++ b/Caf.Midden.Wasm/Shared/MetadataDetails.razor.cs @@ -6,6 +6,9 @@ using System.Threading.Tasks; using AntDesign; using Markdig; +using Microsoft.JSInterop; +using System.Text.Json; +using Caf.Midden.Wasm.Shared.MetadataLineage; namespace Caf.Midden.Wasm.Shared { @@ -14,6 +17,9 @@ public partial class MetadataDetails : ComponentBase [Parameter] public Metadata Metadata { get; set; } + [Inject] + private IJSRuntime JSRuntime { get; set; } // Inject the IJSRuntime interface + public bool VarsHaveMethods { get; set; } public bool VarsHaveQCApplied { get; set; } @@ -113,5 +119,70 @@ private void SetFilters(Configuration appConfig) } this.FilterVariableType = variableTypes.ToArray(); } + + private async Task DownloadVariables() + { + if (Metadata?.Dataset?.Variables == null || string.IsNullOrEmpty(Metadata.Dataset.Name)) + { + Console.WriteLine("No variables found to export or dataset name is missing."); + return; + } + + string datasetName = Metadata.Dataset.Name.Replace(" ", "_"); // Replace spaces with underscores + string filename = $"{datasetName}_metadata.csv"; + + var csvData = ConvertToCSV(Metadata.Dataset.Variables); + Console.WriteLine($"Downloading file: {filename}"); + + await JSRuntime.InvokeVoidAsync("downloadCSV", filename, csvData); + } + + + + private string ConvertToCSV(IEnumerable data) + { + if (!data.Any()) return string.Empty; + + var properties = typeof(T).GetProperties(); + var header = string.Join(",", properties.Select(p => $"\"{p.Name}\"")); // Ensure headers are enclosed in quotes + + var rows = data.Select(row => + string.Join(",", properties.Select(p => + { + var value = p.GetValue(row); + if (value == null) return "\"\""; // Handle null values + + if (value is IEnumerable collection) + { + // Convert lists to a single string, replacing "|" with ";" + string listAsString = string.Join("; ", collection.Select(v => v.ToString() + .Replace("\n", " ") // Remove newlines + .Replace("\r", " ") // Remove carriage returns + .Replace("\"", "\"\""))); // Escape existing double quotes + + return $"\"{listAsString}\""; // Enclose in double quotes + } + + var strValue = value.ToString().Trim() + .Replace("\r\n", " ") // Remove Windows-style newlines + .Replace("\n", " ") // Remove Unix-style newlines + .Replace("\r", " ") // Remove Mac-style newlines + .Replace(" ", " ") // Remove large spaces + .Replace("\"", "\"\""); // Escape double quotes + + return $"\"{strValue}\""; // Enclose all values in double quotes + })) + ); + + return $"{header}\n{string.Join("\n", rows)}"; + } + + + + + + + + } } diff --git a/Caf.Midden.Wasm/Shared/MetadataEditor.razor b/Caf.Midden.Wasm/Shared/MetadataEditor.razor index 2debec9..f044eba 100644 --- a/Caf.Midden.Wasm/Shared/MetadataEditor.razor +++ b/Caf.Midden.Wasm/Shared/MetadataEditor.razor @@ -24,7 +24,7 @@ Layout="@FormLayout.Vertical" ValidateMode="@FormValidateMode.Rules" @ref="form"> - + - @@ -180,6 +179,8 @@ ScrollX="1400" PageSize="50"> + + @if(VariableQuickEditRef == variable) diff --git a/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageDiagram.razor b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageDiagram.razor new file mode 100644 index 0000000..0df3369 --- /dev/null +++ b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageDiagram.razor @@ -0,0 +1,158 @@ +@using Blazor.Diagrams +@using Blazor.Diagrams.Core.Anchors +@using Blazor.Diagrams.Core.Extensions +@using Blazor.Diagrams.Core.Models +@using Blazor.Diagrams.Core.PathGenerators; +@using Blazor.Diagrams.Core.Routers; +@using Blazor.Diagrams.Options; +@using Blazor.Diagrams.Components; +@using Caf.Midden.Wasm.Services +@using Caf.Midden.Wasm.Shared.ViewModels; + +@inject Caf.Midden.Wasm.Services.StateContainer State + +
+ + + + +
+
+
+ +@code { + [Parameter] + public Dataset dataset { get; set; } + + private MetadataLineageDiagramViewModel viewModel; + private const int COLUMN_WIDTH = 400; + private const int ROW_HEIGHT = 200; + private const double ZOOM_INCREMENT = 0.2; + + private bool _InitialResize = false; + private int _NumberCols = 1; + private int _NumberRows = 1; + private List> _CellTracker = new List>(); // Column> + + public BlazorDiagram Diagram { get; set; } = null!; + + protected override void OnInitialized() + { + viewModel = + new MetadataLineageDiagramViewModel(dataset, State.Catalog); + + var datasetNode = viewModel.GetDatasetNode(); + + Diagram = new BlazorDiagram(); + Diagram.Options.Zoom.Enabled = true; + Diagram.RegisterComponent(); + Diagram.ContainerChanged += OnContainerChanged; + + CreateNodes(datasetNode, 0, 0); + } + + private void ZoomIn() + { + var currentZoom = Diagram.Zoom; + var newZoom = currentZoom + ZOOM_INCREMENT; + if(newZoom > 0) + { + Diagram.SetZoom(newZoom); + } + } + + private void ZoomOut() + { + var currentZoom = Diagram.Zoom; + var newZoom = currentZoom - ZOOM_INCREMENT; + if (newZoom > 0) + { + Diagram.SetZoom(newZoom); + } + } + + private void OnContainerChanged() + { + if(Diagram.Container != null && _InitialResize == false) + { + Diagram.SetZoom(0.7); + + // Centers base node -- derived from this: https://github.com/Blazor-Diagrams/Blazor.Diagrams/issues/269#issuecomment-1322025484 + int margin = 0; + + //var width = COLUMN_WIDTH * Diagram.Zoom; + var width = Diagram.Zoom; + var scaledMargin = margin * Diagram.Zoom; + var deltaX = (Diagram.Container.Width / 2) - (width / 2) - scaledMargin; + Diagram.UpdatePan(deltaX, 0); + + _InitialResize = true; + } + } + + private void CreateNodes(DatasetNode datasetNode, int column, int row, NodeModel thisNode = null) + { + // Create a new node if one wasn't passed through recursion + if(thisNode == null) + { + thisNode = Diagram.Nodes.Add( + new MetadataLineageNode( + position: new Blazor.Diagrams.Core.Geometry.Point((column * COLUMN_WIDTH), (row * ROW_HEIGHT))) + { + Name = $"{datasetNode.Name}", + Project = $"{datasetNode.Project}", + Zone = $"{datasetNode.Zone}", + Url = $"{datasetNode.Url}" + }); + + // Register this on the cell tracker + _CellTracker.Add(new List() + { + true + }); + } + + var leftPortThisNode = thisNode.AddPort(PortAlignment.Left); + + // If there are parents, then we need to move over a column + if(datasetNode.Parents.Count > 0) + { + _CellTracker.Add(new List()); + column += 1; + } + + foreach (var parent in datasetNode.Parents.OrderByDescending(p => p.Parents.Count).ToList()) + { + var diagramCol = column * -1; + var diagramRow = _CellTracker[column].Count; + + var parentNode = Diagram.Nodes.Add(new MetadataLineageNode(position: new Blazor.Diagrams.Core.Geometry.Point((diagramCol * COLUMN_WIDTH), (diagramRow * ROW_HEIGHT))) + { + Name = $"{parent.Name}", + Project = $"{parent.Project}", + Zone = $"{parent.Zone}", + Url = $"{parent.Url}" + }); + var rightPortParentNode = parentNode.AddPort(PortAlignment.Right); + var parentAnchor = new SinglePortAnchor(rightPortParentNode); + var childAnchor = new SinglePortAnchor(leftPortThisNode); + var parentToChildLink = Diagram.Links.Add(new LinkModel(parentAnchor, childAnchor) + { + PathGenerator = new SmoothPathGenerator(), + Router = new NormalRouter(), + TargetMarker = LinkMarker.Arrow + }); + + // Adds a row to current column + _CellTracker[column].Add(true); + + if(parent.Parents.Count > 0) + { + _CellTracker.Add(new List()); + CreateNodes(parent, column, _CellTracker[column].Count, parentNode); + } + } + } +} diff --git a/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageNode.cs b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageNode.cs new file mode 100644 index 0000000..6b75620 --- /dev/null +++ b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageNode.cs @@ -0,0 +1,14 @@ +using Blazor.Diagrams.Core.Models; + +namespace Caf.Midden.Wasm.Shared.MetadataLineage +{ + public class MetadataLineageNode : NodeModel + { + public MetadataLineageNode(Blazor.Diagrams.Core.Geometry.Point? position = null): base(position) { } + + public string Name { get; set; } + public string Project { get; set; } + public string Zone { get; set; } + public string Url { get; set; } + } +} diff --git a/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor new file mode 100644 index 0000000..444969c --- /dev/null +++ b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor @@ -0,0 +1,46 @@ +@using Blazor.Diagrams.Components.Renderers; +@using Caf.Midden.Wasm.Shared.MetadataLineage; + +
+ @foreach (var port in Node.Ports) + { + // In case you have any ports to show + // IMPORTANT: You are always in charge of rendering ports + + } + + + + @if (!String.IsNullOrEmpty(Node.Url)) + { + + } + + + @if(!String.IsNullOrEmpty(Node.Zone)) + { +

+ + + @Node.Zone + +

+ } + @if(!String.IsNullOrEmpty(Node.Project)) + { +

+ + + @Node.Project + +

+ } + + + + +
+ +@code { + [Parameter] public MetadataLineageNode Node { get; set; } = null!; +} diff --git a/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor.css b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor.css new file mode 100644 index 0000000..dfb9aa9 --- /dev/null +++ b/Caf.Midden.Wasm/Shared/MetadataLineage/MetadataLineageWidget.razor.css @@ -0,0 +1,20 @@ +::deep .diagram-port { + position: absolute; + width: 1px; + height: 1px; + background-color: darkgray; + transform: translate(-50%, -50%); + top: 50%; +} + + ::deep .diagram-port.left { + left: 0; + } + + ::deep .diagram-port.right { + left: 100%; + } + +div .dataset-node { + box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19); +} \ No newline at end of file diff --git a/Caf.Midden.Wasm/Shared/ViewModels/MetadataLineageDiagramViewModel.cs b/Caf.Midden.Wasm/Shared/ViewModels/MetadataLineageDiagramViewModel.cs new file mode 100644 index 0000000..73641de --- /dev/null +++ b/Caf.Midden.Wasm/Shared/ViewModels/MetadataLineageDiagramViewModel.cs @@ -0,0 +1,172 @@ +using Caf.Midden.Core.Models.v0_2; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; + +namespace Caf.Midden.Wasm.Shared.ViewModels +{ + public class MetadataLineageDiagramViewModel(Dataset dataset, Catalog catalog) + { + private readonly Dataset _Dataset = dataset; + private readonly Catalog _Catalog = catalog; + + private const string MIDDEN_DATASET_FLAG = "[Midden]"; + private const int MAX_NUMBER_CONNECTIONS = 20; + + // Tracks the number of connections (NOT number of parent). This is to prevent infinite loops. + private int _NumberConnections = 0; + private DatasetNode _Node; + public DatasetNode GetDatasetNode() + { + if (_Node == null) + { + _NumberConnections = 0; + _Node = InitializeDatasetNode(_Dataset); + } + + return _Node; + } + private DatasetNode InitializeDatasetNode(Dataset dataset) + { + var node = new DatasetNode( + dataset.Name, + dataset.Project, + dataset.Zone, + GetRelativeUrlForDataset(dataset)); + + foreach (var parentDatasetString in dataset.ParentDatasets) + { + var parentName = ""; + var parentUrl = ""; + var parentZone = ""; + var parentProject = ""; + + if (parentDatasetString.StartsWith(MIDDEN_DATASET_FLAG)) + { + string potentialUrl = parentDatasetString.Replace(MIDDEN_DATASET_FLAG, ""); + if (Uri.IsWellFormedUriString(potentialUrl, UriKind.Absolute)) + { + // It's a Midden dataset with a valid url, so set URL and look up dataset in catalog + parentUrl = potentialUrl; + + var parentDataset = GetDatasetFromCatalogByUrl(parentUrl); + if (parentDataset != null) + { + _NumberConnections += 1; + + // Check if we've already reached max connections + // If so, stop recursive + if (_NumberConnections > MAX_NUMBER_CONNECTIONS) + { + return node; + } + + // Recursive time! Call Initialize on this one + var parentNode = InitializeDatasetNode(parentDataset); + + node.AddParent(parentNode); + } + else + { + // Failed to find the dataset in the catalog, so just set name as url + parentName = parentUrl; + + var parentNode = new DatasetNode(parentName, parentProject, parentZone, parentUrl); + _NumberConnections += 1; + node.AddParent(parentNode); + } + + } + else + { + // It's not a valid url, so set the name to it and don't link anything + parentName = potentialUrl; + + var parentNode = new DatasetNode(parentName, parentProject, parentZone, parentUrl); + _NumberConnections += 1; + node.AddParent(parentNode); + } + } + else + { + parentName = parentDatasetString; + + if (Uri.IsWellFormedUriString(parentDatasetString, UriKind.Absolute)) + { + parentUrl = parentDatasetString; + } + + var parentNode = new DatasetNode(parentName, parentProject, parentZone, parentUrl); + _NumberConnections += 1; + node.AddParent(parentNode); + } + } + + return node; + } + + private string GetRelativeUrlForDataset(Dataset dataset) + { + var result = $"/catalog/datasets/{dataset.Zone}/{dataset.Project}/{dataset.Name}"; + + return result; + } + private Dataset GetDatasetFromCatalogByUrl(string url) + { + // Parse the url, assume: {base url}/catalog/dataset/{zone}/{project}/{dataset} + // NOTE: Url structure may change, so this is on weak footing + + Uri uri = new Uri(url); + string pathOnly = uri.AbsolutePath; + string[] segments = pathOnly.Split('/'); + + if(segments.Length != 6) + { + // Not in Midden format + return null; + } + + string zone = segments[3]; + string project = segments[4]; + string name = segments[5]; + + Dataset dataset = _Catalog.Metadatas.Where( + m => m.Dataset.Zone == zone && + m.Dataset.Project == project && + m.Dataset.Name == name).FirstOrDefault().Dataset; + + return dataset; + } + } + + public class DatasetNode + { + private List _Parents = new List(); + //private List _Children = new List(); + + + public string Zone { get; set; } + public string Project { get; set; } + public string Name { get; set; } + public string Url { get; set; } + + public DatasetNode(string name, string project, string zone, string url) + { + this.Name = name; + this.Url = url; + this.Zone = zone; + this.Project = project; + } + + public IReadOnlyCollection Parents + { + get { return _Parents.AsReadOnly(); } + } + + public void AddParent(DatasetNode node) + { + _Parents.Add(node); + } + } +} diff --git a/Caf.Midden.Wasm/wwwroot/catalog.json b/Caf.Midden.Wasm/wwwroot/catalog.json index bf68a57..02d6ad2 100644 --- a/Caf.Midden.Wasm/wwwroot/catalog.json +++ b/Caf.Midden.Wasm/wwwroot/catalog.json @@ -3582,6 +3582,98 @@ ], "derivedWorks": [] } + }, + { + "schemaVersion": "v0.2", + "creationDate": "2024-12-11T01:00:00.757Z", + "modifiedDate": "2024-12-11T01:00:00.757Z", + "dataset": { + "zone": "Raw", + "project": "TestProject", + "name": "RawDataset", + "description": "This is raw data and a parent to all", + "tags": [], + "contacts": [ + { + "name": "User1" + } + ], + "methods": [], + "variables": [], + "derivedWorks": [], + "parentDatasets": [] + } + }, + { + "schemaVersion": "v0.2", + "creationDate": "2024-12-11T01:00:00.757Z", + "modifiedDate": "2024-12-11T01:00:41.318Z", + "dataset": { + "zone": "Raw", + "project": "AnotherTestProject", + "name": "AnotherRawDataset", + "description": "This is raw data outside of TestProject", + "tags": [], + "contacts": [ + { + "name": "User1" + } + ], + "methods": [], + "variables": [], + "derivedWorks": [], + "parentDatasets": [] + } + }, + { + "schemaVersion": "v0.2", + "creationDate": "2024-12-11T01:00:00.757Z", + "modifiedDate": "2024-12-11T01:03:11.813Z", + "dataset": { + "zone": "Work", + "project": "TestProject", + "name": "WorkingDataFromTwoRaw", + "description": "This dataset is created from two raw datasets", + "tags": [], + "contacts": [ + { + "name": "User2" + } + ], + "methods": [], + "variables": [], + "derivedWorks": [], + "parentDatasets": [ + "[Midden]https://localhost:5001/catalog/datasets/Raw/TestProject/RawDataset", + "[Midden]https://localhost:5001/catalog/datasets/Raw/AnotherTestProject/AnotherRawDataset" + ] + } + }, + { + "schemaVersion": "v0.2", + "creationDate": "2024-12-11T01:00:00.757Z", + "modifiedDate": "2024-12-11T01:07:06.382Z", + "dataset": { + "zone": "Production", + "project": "TestProject", + "name": "ProductionFromWorkingAndExternal", + "description": "This dataset is created from working dataset and an external reference dataset", + "tags": [], + "contacts": [ + { + "name": "User2" + } + ], + "methods": [], + "variables": [], + "derivedWorks": [], + "parentDatasets": [ + "https://github.com/cafltar/Midden_AzureFunctionRunCollator/blob/main/README.md", + "Armendariz, Gerardo; Coffin, Alisa W.; Archer, David; Arthur, Dan; Bean, Alycia; Browning, Dawn; Carlson, Bryan; Clark, Pat; Flynn, Colton; Goslee, Sarah; Hall, Veronica; Holifield Collins, Chandra; Hsieh, Hsun-Yi; Johnson, Jane M. F.; Kaplan, Nicole; Kautz, Mark; Kettler, Tim; King, Kevin; Moglen, Glenn; Schmer, Marty; Sclater, Vivienne; Spiegal, Sheri; Stark, Patrick; Stinner, Jedediah; Sudduth, Ken; Teet, Stephen; Wagner, Steve; Yasarer, Lindsey (2021). The Long-Term Agroecosystem Research (LTAR) Network Standard GIS Data Layers, 2020 version. Ag Data Commons. https://doi.org/10.15482/USDA.ADC/1521161", + "[Midden]https://localhost:5001/catalog/datasets/Work/TestProject/WorkingDataFromTwoRaw", + "[Midden]https://localhost:5001/catalog/datasets/Work/TestProject/WorkingDataFromTwoRaw" + ] + } } ] } \ No newline at end of file diff --git a/Caf.Midden.Wasm/wwwroot/css/custom.css b/Caf.Midden.Wasm/wwwroot/css/custom.css index d427ecc..d01a6bb 100644 --- a/Caf.Midden.Wasm/wwwroot/css/custom.css +++ b/Caf.Midden.Wasm/wwwroot/css/custom.css @@ -2,7 +2,8 @@ height: 40px; padding: 5px; line-height: 0px; - background: #40A9FF; + background: #1890ff; + //background: #40A9FF; position: relative; z-index: 10; -webkit-box-shadow: 0px 0px 5px 3px rgba(41,41,41,.25); @@ -170,4 +171,38 @@ -webkit-line-clamp: 3; -webkit-box-orient: vertical; max-height: 100px; -} \ No newline at end of file +} + +.dataset-card.ant-card-bordered { + border: 1px solid #44a5ff; +} + +.dataset-card .ant-card-head { + background-color: #44a5ff; /* Dark gray for dataset heads */ + color: white; /* White text for contrast */ + /*padding: 10px;*/ /* Add spacing */ +} + +.dataset-card .ant-card-head-title { + color: rgb(0,0,0,0.85); /* Ensure title text is white */ + /*font-weight: bold; Make text bold */ +} + +.dataset-card .ant-card-head-extra { + color: rgb(0,0,0,0.85); /* Style extra content if present */ + background-color: darkgray; /* Dark gray for dataset heads */ +} + +.magnifying-button { + background-color: #abb2b9; /* Dark gray background */ + color: black; /* White icon color */ + border: none; /* Remove border */ + border-radius: 5px; /* Rounded corners */ + cursor: pointer; /* Add pointer cursor */ +} + +.magnifying-button:hover { + background-color: #333; /* Slightly darker gray on hover */ + color:white; + font-weight:bold; +} diff --git a/Caf.Midden.Wasm/wwwroot/index.html b/Caf.Midden.Wasm/wwwroot/index.html index 999ca97..d0d1c73 100644 --- a/Caf.Midden.Wasm/wwwroot/index.html +++ b/Caf.Midden.Wasm/wwwroot/index.html @@ -17,7 +17,9 @@ - + + + Midden: Research Data Catalog document.body.removeChild(link); } } + + window.downloadCSV = (filename, content) => { + console.log("JavaScript function downloadCSV called!"); + + if (!content) { + console.error("No content provided for CSV download."); + return; + } + + console.log("Content preview:", content.substring(0, 200) + "..."); + + try { + // Encode content as UTF-8 with BOM to fix character encoding issues + const BOM = "\uFEFF"; // Byte Order Mark (UTF-8) + const blob = new Blob([BOM + content], { type: 'text/csv;charset=utf-8;' }); + const url = URL.createObjectURL(blob); + + // Create a hidden element for download + const link = document.createElement("a"); + link.setAttribute("href", url); + link.setAttribute("download", filename); + link.style.visibility = "hidden"; + + console.log("Appending download link to document..."); + document.body.appendChild(link); + + // Trigger the download + console.log("Triggering download..."); + link.click(); + + // Cleanup + document.body.removeChild(link); + URL.revokeObjectURL(url); + console.log("Download should now be triggered!"); + } catch (error) { + console.error("Error during CSV download:", error); + } + }; + + + + @@ -78,6 +122,9 @@

Midden: Research Data Catalog

+ + +