Merge pull request #155 from CafIncubator/develop

Major changes

- Upgraded from .NET6 to .NET9
- Added Tag page
- Added data lineage graphs
- Some UI style changes
- Can export variable table as CSV
- Numerous bug fixes and small changes

Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj

@@ -1,28 +1,24 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
-
+    <TargetFramework>net9.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
-
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="xunit" Version="2.4.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageReference Include="System.Text.Json" Version="9.0.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="3.1.1">
+    <PackageReference Include="coverlet.collector" Version="6.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Caf.Midden.Cli\Caf.Midden.Cli.csproj" />
   </ItemGroup>
-
   <ItemGroup>
     <None Update="Assets\CliConfigurationSecrets\AzureDataLakeProjectTest.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -67,5 +63,4 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
-
-</Project>
+</Project>

Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs

@@ -1,7 +1,6 @@
 using Caf.Midden.Cli.Services;
 using Caf.Midden.Core.Models.v0_2;
 using Caf.Midden.Core.Services.Metadata;
-using NuGet.Frameworks;
 using System;
 using System.Collections.Generic;
 using System.Linq;

Caf.Midden.Cli/Caf.Midden.Cli.csproj

@@ -1,36 +1,31 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <PackAsTool>true</PackAsTool>
     <Nullable>enable</Nullable>
-    <AssemblyVersion>0.3.1.0</AssemblyVersion>
-    <FileVersion>0.3.1.0</FileVersion>
+    <AssemblyVersion>0.4.0</AssemblyVersion>
+    <FileVersion>0.4.0</FileVersion>
     <PackageId>MiddenCli</PackageId>
-    <Version>0.3.1</Version>
+    <Version>0.4-dev.0</Version>
     <RepositoryUrl>https://github.com/cafincubator/midden</RepositoryUrl>
     <AssemblyName>MiddenCli</AssemblyName>
     <PublishSingleFile>true</PublishSingleFile>
   </PropertyGroup>
-
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.5.0" />
-    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.8.0" />
-    <PackageReference Include="Azure.Storage.Files.Shares" Version="12.8.0" />
-    <PackageReference Include="Google.Apis.Drive.v3" Version="1.55.0.2566" />
+    <PackageReference Include="Azure.Identity" Version="1.13.1" />
+    <PackageReference Include="Azure.Storage.Files.DataLake" Version="12.21.0" />
+    <PackageReference Include="Azure.Storage.Files.Shares" Version="12.21.0" />
+    <PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3608" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20574.7" />
+    <PackageReference Include="System.Text.Json" Version="9.0.0" />
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Caf.Midden.Core\Caf.Midden.Core.csproj" />
   </ItemGroup>
-
   <ItemGroup>
     <None Update="configuration.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
-
-
-</Project>
+</Project>

Caf.Midden.Cli/Services/AzureDataLakeCrawler.cs

@@ -99,26 +99,33 @@ public List<Metadata> GetMetadatas(
 
             foreach (var fileName in fileNames)
             {
-                // Get file contents as json string
-                DataLakeFileClient fileClient = 
-                    fileSystemClient.GetFileClient(fileName);
+                try
+                {
+                    // Get file contents as json string
+                    DataLakeFileClient fileClient =
+                        fileSystemClient.GetFileClient(fileName);
 
-                Response<FileDownloadInfo> fileContents = fileClient.Read();
+                    Response<FileDownloadInfo> fileContents = fileClient.Read();
 
-                string json;
-                using (MemoryStream ms = new MemoryStream())
-                {
-                    fileContents.Value.Content.CopyTo(ms);
-                    json = Encoding.UTF8.GetString(ms.ToArray());
-                }
+                    string json;
+                    using (MemoryStream ms = new MemoryStream())
+                    {
+                        fileContents.Value.Content.CopyTo(ms);
+                        json = Encoding.UTF8.GetString(ms.ToArray());
+                    }
 
-                // Parse json string and add relative path to Dataset
-                Metadata metadata = parser.Parse(json);
+                    // Parse json string and add relative path to Dataset
+                    Metadata metadata = parser.Parse(json);
 
-                string filePath = fileClient.Path.Replace(MIDDEN_FILE_EXTENSION, "");
-                metadata.Dataset.DatasetPath = filePath;
+                    string filePath = fileClient.Path.Replace(MIDDEN_FILE_EXTENSION, "");
+                    metadata.Dataset.DatasetPath = filePath;
 
-                metadatas.Add(metadata);
+                    metadatas.Add(metadata);
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"Error parsing file: {fileName}, reason: {ex}");
+                }
             }
 
             return metadatas;

Caf.Midden.Core.Tests/Caf.Midden.Core.Tests.csproj

@@ -1,29 +1,24 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
-
+    <TargetFramework>net9.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
-
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="xunit" Version="2.4.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageReference Include="Moq" Version="4.20.72" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="coverlet.collector" Version="3.1.1">
+    <PackageReference Include="coverlet.collector" Version="6.0.2">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Caf.Midden.Core\Caf.Midden.Core.csproj" />
   </ItemGroup>
-
   <ItemGroup>
     <None Update="Assets\ConfigFiles\app-config_v0_1_0-alpha4.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -50,5 +45,4 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
-
-</Project>
+</Project>

Caf.Midden.Core/Caf.Midden.Core.csproj

@@ -1,11 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
   </PropertyGroup>
-
   <ItemGroup>
-    <PackageReference Include="CsvHelper" Version="27.2.1" />
+    <PackageReference Include="CsvHelper" Version="33.0.1" />
   </ItemGroup>
-
-</Project>
+</Project>

Caf.Midden.Wasm/Caf.Midden.Wasm.csproj

@@ -1,29 +1,27 @@
-<Project Sdk="Microsoft.NET.Sdk.BlazorWebAssembly">
-
+<Project Sdk="Microsoft.NET.Sdk.BlazorWebAssembly">
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <ServiceWorkerAssetsManifest>service-worker-assets.js</ServiceWorkerAssetsManifest>
-    <Version>0.3</Version>
+    <Version>0.4.0</Version>
+    <InformationalVersion>0.4.0</InformationalVersion>
   </PropertyGroup>
-
   <ItemGroup>
-    <PackageReference Include="AntDesign" Version="0.12.2" />
-    <PackageReference Include="AntDesign.Charts" Version="0.2.3" />
-    <PackageReference Include="Markdig" Version="0.27.0" />
-    <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="6.0.1" />
-    <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="6.0.1" PrivateAssets="all" />
-    <PackageReference Include="PSC.Blazor.Components.MarkdownEditor" Version="2.0.9" />
-    <PackageReference Include="System.Net.Http.Json" Version="6.0.0" />
+    <PackageReference Include="AntDesign" Version="1.0.1" />
+    <PackageReference Include="AntDesign.Charts" Version="0.5.6" />
+    <PackageReference Include="Markdig" Version="0.38.0" />
+    <PackageReference Include="PSC.Blazor.Components.MarkdownEditor" Version="8.0.5" />
+    <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="9.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="9.0.0" />
+    <PackageReference Include="System.Net.Http.Json" Version="9.0.0" />
+    <PackageReference Include="System.Text.Json" Version="9.0.0" />
+    <PackageReference Include="Z.Blazor.Diagrams" Version="3.0.3" />
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Caf.Midden.Core\Caf.Midden.Core.csproj" />
   </ItemGroup>
-
   <ItemGroup>
     <ServiceWorker Include="wwwroot\service-worker.js" PublishedContent="wwwroot\service-worker.published.js" />
   </ItemGroup>
-
   <ItemGroup>
     <Content Update="wwwroot\app-config.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -59,11 +57,9 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
-
   <ItemGroup>
     <None Update="Shared\MainLayout.razor.css">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
-
 </Project>

Caf.Midden.Wasm/Pages/CatalogTags.razor

@@ -2,12 +2,9 @@
 @inject Services.StateContainer State
 
 <div class="site-layout-background">
+    <CatalogLoader />
     <PageHeader Class="site-page-header"
                 Title="Tag Catalog" />
-    <p>Coming soon...</p>
-</div>
 
-
-@code {
-
-} 
+    <FilteredCatalogTagViewer />
+</div>

Caf.Midden.Wasm/Pages/Index.razor

@@ -1,4 +1,5 @@
 @page "/"
+
 @inject Services.StateContainer State
 
 <div class="site-layout-background">
@@ -74,19 +75,20 @@
         </AntDesign.Col>
     </AntDesign.Row>
 
+    <Divider />
+
+    <Title Level="2">Recent Datasets <small><a href="catalog/datasets"><Icon Type="double-right" Theme="outline" /></a></small></Title>
+    <FilteredCatalogMetadataViewer ShowSearch=false ShowHeader=false ShowRecentNumber="2" />
+
+
     <Divider />
 
-     <Title Level="2">Recent Projects <small><a href="catalog/projects"><Icon Type="double-right" Theme="outline" /></a></small></Title>
+    <Title Level="2">Recent Projects <small><a href="catalog/projects"><Icon Type="double-right" Theme="outline" /></a></small></Title>
     <FilteredCatalogProjectViewer ShowSearch=false ShowHeader=false ShowRecentNumber="2" />
+
     <Divider />
 
-     <Title Level="2">Recent Datasets <small><a href="catalog/datasets"><Icon Type="double-right" Theme="outline" /></a></small></Title>
-
-    <FilteredCatalogMetadataViewer ShowSearch=false ShowHeader=false ShowRecentNumber="2" />
-    <Divider />
-
     <Title Level="2">Stats for Nerds</Title>
-
     <AntDesign.Row Gutter="16" Wrap="true">
         <AntDesign.Col Xs="@Property(24,0)" Lg="@Property(12,0)" Style="padding: 5px;">
             <AntDesign.Card Bordered Title="Projects by Status" >
@@ -170,8 +172,7 @@
 
 </div>
 
-
-@code{
+@code {
     private string DebugMsg { get; set; }
 
     private async Task AppConfig_StateChanged(
@@ -185,6 +186,8 @@
         }
     }
 
+
+
     //protected override void OnInitialized()
     //{
     //    State.StateChanged += async (source, property)
@@ -196,4 +199,7 @@
     //    => await AppConfig_StateChanged(source, property);
     //}
     
-}
+}
+
+
+

Caf.Midden.Wasm/Pages/Index.razor.cs

@@ -31,6 +31,8 @@ public partial class Index : IDisposable
 
         IChartComponent MetadataPerZone = new Column();
         public object[] MetadataPerZoneData { get; set; }
+
+
         ColumnConfig MetadataPerZoneConfig = new ColumnConfig
         {
             Title = new AntDesign.Charts.Title
@@ -166,6 +168,7 @@ private void SetSimpleStats()
             this.TotalContacts = UniqueContacts.Count;
         }
 
+
         private void CreateDatasetsPerZone()
         {
             List<object> objs = new List<object>();
@@ -189,6 +192,8 @@ private void CreateDatasetsPerZone()
             MetadataPerZone.ChangeData(MetadataPerZoneData);
         }
 
+
+
         private void CreateProjectsPerStatus()
         {
             List<object> objs = new List<object>();

Caf.Midden.Wasm/Services/StateContainer.cs

@@ -62,10 +62,16 @@ public void UpdateCatalog(
 
         public StateContainer()
         {
-            this.AssemblyVersion = 
-                Assembly.GetExecutingAssembly()
+            var informationalVersion = 
+                Assembly.GetEntryAssembly()
                     .GetCustomAttribute<AssemblyInformationalVersionAttribute>()
                     .InformationalVersion;
+
+            // Added because a GUID was appended to the end for unknown reasons
+            if (informationalVersion.Contains("+"))
+                informationalVersion = informationalVersion.Split("+")[0];
+
+            this.AssemblyVersion = informationalVersion;
         }
 
         public event Action<ComponentBase, string> StateChanged;