Merge pull request #124 from CafIncubator/develop

Merge 0.2-dev.10 to main

Caf.Midden.Cli.Tests/.gitignore

@@ -1 +1 @@
-Assets/CliConfigurationSecrets
+Assets/CliConfigurationSecrets/*

Caf.Midden.Cli.Tests/Assets/MockDataStoreLocal/Production/DESCRIPTION.md

@@ -0,0 +1,6 @@
+---
+project: "ProductionProject"
+---
+# Production Title
+
+Some text and stuff

Caf.Midden.Cli.Tests/Assets/MockDataStoreLocal/Raw/DESCRIPTION.md

@@ -0,0 +1,6 @@
+---
+project: "RawProject"
+---
+# Raw Title
+
+Some text and stuff

Caf.Midden.Cli.Tests/Caf.Midden.Cli.Tests.csproj

@@ -24,6 +24,18 @@
   </ItemGroup>
 
   <ItemGroup>
+    <None Update="Assets\CliConfigurationSecrets\AzureDataLakeProjectTest.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\CliConfigurationSecrets\GoogleWorkspaceSharedDriveProjectTest.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\CliConfigurationSecrets\LocalFileSystemProjectTest.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\MockDataStoreLocal\Production\DESCRIPTION.md">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="Assets\CliConfigurationSecrets\GoogleDriveProjectTest.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -36,6 +48,9 @@
     <None Update="Assets\MockDataStoreLocal\Raw\CookEastMet_V40826.midden">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="Assets\MockDataStoreLocal\Raw\DESCRIPTION.md">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="Assets\MockDataStoreLocal\Raw\wepsV5_2.midden">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>

Caf.Midden.Cli.Tests/CrawlerIntegrationTests.cs

@@ -0,0 +1,91 @@
+using Caf.Midden.Cli.Models;
+using Caf.Midden.Cli.Services;
+using Caf.Midden.Core.Services;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace Caf.Midden.Cli.Tests
+{
+    public class CrawlerIntegrationTests
+    {
+        // NOTE: These tests require configuration files that are not included in the repository. These need to be generated for each clone.
+        [Fact]
+        public void GetProjects_Local()
+        {
+            string configPath = "Assets/CliConfigurationSecrets/LocalFileSystemProjectTest.json";
+            if(!File.Exists(configPath))
+                throw new FileNotFoundException(configPath);
+
+            // Gets config file, fails if not exist
+            ConfigurationService configService = new ConfigurationService();
+            CliConfiguration config = configService.GetConfiguration(configPath);
+
+            DataStore dataStore = config.DataStores[0];
+            LocalFileSystemCrawler sut = new LocalFileSystemCrawler(
+                dataStore.Path);
+
+            List<Core.Models.v0_2.Project> actual = sut.GetProjects(new ProjectReader(
+                new ProjectParser()));
+
+            Assert.NotNull(actual);
+            Assert.Single(actual);
+            Assert.Equal("ProductionProject", actual[0].Name);
+        }
+
+        [Fact]
+        public void GetProjects_AzureDataLake()
+        {
+            string configPath = "Assets/CliConfigurationSecrets/AzureDataLakeProjectTest.json";
+            if (!File.Exists(configPath))
+                throw new FileNotFoundException(configPath);
+
+            // Gets config file, fails if not exist
+            ConfigurationService configService = new ConfigurationService();
+            CliConfiguration config = configService.GetConfiguration(configPath);
+
+            DataStore dataStore = config.DataStores[0];
+            AzureDataLakeCrawler sut = new AzureDataLakeCrawler(
+                dataStore.AccountName,
+                dataStore.TenantId,
+                dataStore.ClientId,
+                dataStore.ClientSecret,
+                dataStore.AzureFileSystemName);
+
+            List<Core.Models.v0_2.Project> actual = sut.GetProjects(new ProjectReader(
+                new ProjectParser()));
+
+            Assert.NotNull(actual);
+            Assert.Single(actual);
+            Assert.Equal("TestProject", actual[0].Name);
+        }
+
+        [Fact]
+        public void GetProjects_GoogleWorkspaceSharedDrive()
+        {
+            string configPath = "Assets/CliConfigurationSecrets/GoogleWorkspaceSharedDriveProjectTest.json";
+            if (!File.Exists(configPath))
+                throw new FileNotFoundException(configPath);
+
+            // Gets config file, fails if not exist
+            ConfigurationService configService = new ConfigurationService();
+            CliConfiguration config = configService.GetConfiguration(configPath);
+
+            DataStore dataStore = config.DataStores[0];
+            GoogleWorkspaceSharedDriveCrawler sut = new GoogleWorkspaceSharedDriveCrawler(
+                dataStore.ClientId,
+                dataStore.ClientSecret,
+                dataStore.ApplicationName);
+
+            List<Core.Models.v0_2.Project> actual = sut.GetProjects(new ProjectReader(
+                new ProjectParser()));
+
+            Assert.NotNull(actual);
+            Assert.True(actual.Count() > 0);
+        }
+    }
+}

Caf.Midden.Cli.Tests/GoogleDriveCrawlerTests.cs

@@ -1,5 +1,7 @@
 using Caf.Midden.Cli.Models;
 using Caf.Midden.Cli.Services;
+using Caf.Midden.Core.Services;
+using Caf.Midden.Core.Services.Metadata;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -16,6 +18,7 @@ public class GoogleDriveCrawlerTests
 
         /// <summary>
         /// These tests require `GoogleDriveProjectTest.json` file to be defined in `Assets/CliConfigurationSecrets`, included in the project, with proper formatting (conforms to CliConfiguration json), and at least one .midden file in the Google Drive that the configuration file points to 
+        /// Also, requires a 'DESCRIPTION.md' file in the Google Drive with project = ProductionProject
         /// </summary>
         public GoogleDriveCrawlerTests()
         {
@@ -51,12 +54,14 @@ public void GetMetadatas_ValidInput_Expected()
         {
             if (_config != null)
             {
+                var parser = new MetadataParser(
+                    new MetadataConverter());
                 var sut = new GoogleDriveCrawler(
                 _config.DataStores[0].ClientId,
                 _config.DataStores[0].ClientSecret,
                 _config.DataStores[0].ApplicationName);
 
-                var actual = sut.GetMetadatas();
+                var actual = sut.GetMetadatas(parser);
 
                 Assert.NotNull(actual);
             }
@@ -65,5 +70,30 @@ public void GetMetadatas_ValidInput_Expected()
                 throw new NotImplementedException();
             }
         }
+
+        [Fact]
+        public void GetProjects_ValidInput_Expected()
+        {
+            if (_config != null)
+            {
+                var reader = new ProjectReader(
+                    new ProjectParser());
+
+                var sut = new GoogleDriveCrawler(
+                _config.DataStores[0].ClientId,
+                _config.DataStores[0].ClientSecret,
+                _config.DataStores[0].ApplicationName);
+
+                var actual = sut.GetProjects(reader);
+
+                Assert.NotNull(actual);
+                Assert.True(actual.Count > 0);
+                Assert.Equal("ProductionProject", actual[0].Name);
+            }
+            else
+            {
+                throw new NotImplementedException();
+            }
+        }
     }
 }

Caf.Midden.Cli.Tests/LocalFileSystemCrawlerTests.cs

@@ -15,7 +15,7 @@ public void GetFileNames_ValidInput_ReturnsExpected()
         {
             var sut = new LocalFileSystemCrawler(@"Assets\MockDataStoreLocal");
 
-            var actual = sut.GetFileNames();
+            var actual = sut.GetFileNames(".midden");
 
             Assert.Equal(5, actual.Count);
         }

Caf.Midden.Cli/Actions/Collate.cs

@@ -1,7 +1,9 @@
 using Caf.Midden.Cli.Common;
 using Caf.Midden.Cli.Models;
 using Caf.Midden.Cli.Services;
-using Caf.Midden.Core.Models.v0_1;
+using Caf.Midden.Core.Models.v0_2;
+using Caf.Midden.Core.Services;
+using Caf.Midden.Core.Services.Metadata;
 using System;
 using System.Collections.Generic;
 using System.CommandLine;
@@ -91,6 +93,7 @@ public void HandleCollate(
 
             // Start crawling all specified data stores
             List<Metadata> middenMetadatas = new List<Metadata>();
+            List<Project> mippenProjects = new List<Project>();
 
             foreach (string store in datastores)
             {
@@ -106,7 +109,7 @@ public void HandleCollate(
 
                 Console.WriteLine($"Crawling Data Store: {currStore.Name}");
 
-                ICrawl crawler = null;
+                ICrawl? crawler = null;
                 switch (currStore.Type)
                 {
                     case DataStoreTypes.LocalFileSystem:
@@ -208,24 +211,44 @@ currStore.Path is not null &&
                         break;
                 }
 
-                var metadatas = crawler?.GetMetadatas();
+                var metadatas = crawler?.GetMetadatas(
+                    new MetadataParser(
+                        new MetadataConverter()));
+
+                List<Project>? projects = new List<Project>();
+                if(currStore.ShouldCollateProjects is true)
+                {
+                    projects = crawler?.GetProjects(
+                        new ProjectReader(
+                            new ProjectParser()));
+                }
 
                 if(metadatas != null)
                 {
                     AppendDataStoreNameToPath(metadatas, currStore.Name);
 
                     if (middenMetadatas != null) middenMetadatas.AddRange(metadatas);
                 }
+
+                if(projects?.Count > 0)
+                {
+                    if (mippenProjects != null) mippenProjects.AddRange(projects);
+                }
             }
 
             Catalog catalog = new Catalog()
             {
                 CreationDate = DateTime.UtcNow,
-                Metadatas = middenMetadatas
+                Metadatas = middenMetadatas,
+                Projects = mippenProjects
             };
 
             Console.WriteLine($"Writing output to {outdir}");
-            File.WriteAllText(outdir, JsonSerializer.Serialize(catalog));
+            JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions()
+            {
+                DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
+            };
+            File.WriteAllText(outdir, JsonSerializer.Serialize(catalog, jsonSerializerOptions));
         }
 
         private void AppendDataStoreNameToPath(

Caf.Midden.Cli/BUILD.bat

@@ -1,5 +1,5 @@
-dotnet publish -c Release -o Publish/linux-x64 -p:PublishReadyToRun=false -p:PublishSingleFile=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r linux-x64 /p:DebugType=None /p:DebugSymbols=false
+dotnet publish -c Release -o Publish/linux-x64 -p:PublishReadyToRun=false -p:PublishSingleFile=true -p:UseAppHost=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r linux-x64 /p:DebugType=None /p:DebugSymbols=false
 
-dotnet publish -c Release -o Publish/win-x64 -p:PublishReadyToRun=true -p:PublishSingleFile=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r win-x64 /p:DebugType=None /p:DebugSymbols=false
+dotnet publish -c Release -o Publish/win-x64 -p:PublishReadyToRun=true -p:PublishSingleFile=true -p:UseAppHost=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r win-x64 /p:DebugType=None /p:DebugSymbols=false
 
-dotnet publish -c Release -o Publish/osx.10.11-x64 -p:PublishReadyToRun=false -p:PublishSingleFile=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r osx.10.13-x64 /p:DebugType=None /p:DebugSymbols=false
+dotnet publish -c Release -o Publish/osx.10.11-x64 -p:PublishReadyToRun=false -p:PublishSingleFile=true -p:UseAppHost=true --self-contained true -p:IncludeNativeLibrariesForSelfExtract=true -r osx.10.13-x64 /p:DebugType=None /p:DebugSymbols=false

Caf.Midden.Cli/Caf.Midden.Cli.csproj

@@ -5,12 +5,13 @@
     <TargetFramework>net6.0</TargetFramework>
     <PackAsTool>true</PackAsTool>
     <Nullable>enable</Nullable>
-    <AssemblyVersion>0.1.1.0</AssemblyVersion>
-    <FileVersion>0.1.1.0</FileVersion>
+    <AssemblyVersion>0.2.0.0</AssemblyVersion>
+    <FileVersion>0.2.0.0</FileVersion>
     <PackageId>MiddenCli</PackageId>
-    <Version>0.1.1</Version>
+    <Version>0.2-dev.3</Version>
     <RepositoryUrl>https://github.com/cafincubator/midden</RepositoryUrl>
     <AssemblyName>MiddenCli</AssemblyName>
+    <PublishSingleFile>true</PublishSingleFile>
   </PropertyGroup>
 
   <ItemGroup>

Caf.Midden.Cli/Common/ICrawl.cs

@@ -1,4 +1,6 @@
-using Caf.Midden.Core.Models.v0_1;
+using Caf.Midden.Core.Models.v0_2;
+using Caf.Midden.Core.Services;
+using Caf.Midden.Core.Services.Metadata;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -9,7 +11,8 @@ namespace Caf.Midden.Cli.Common
 {
     public interface ICrawl
     {
-        List<string> GetFileNames();
-        List<Metadata> GetMetadatas();
+        List<string> GetFileNames(string fileExtension);
+        List<Metadata> GetMetadatas(IMetadataParser parser);
+        List<Project> GetProjects(ProjectReader reader);
     }
 }

Caf.Midden.Cli/Models/DataStore.cs

@@ -8,7 +8,7 @@ namespace Caf.Midden.Cli.Models
 {
     public class DataStore
     {
-        public string Name { get; set; }
+        public string Name { get; set; } = "";
         public DataStoreTypes Type { get; set; }
         public string? Path { get; set; }
         public string? TenantId { get; set; }
@@ -19,5 +19,6 @@ public class DataStore
         public string? ApplicationName { get; set; }
         public string? SharedAccessSignature { get; set; }
         public string? Uri { get; set; }
+        public bool? ShouldCollateProjects { get; set; }
     }
 }