Initial draft of a validation testing framework

This is work in progress, but it tries to provide an API for the implementation of 6.1 mandatory tests for validation.

.github/workflows/gradle.yml

@@ -7,7 +7,6 @@ on:
     tags:
       - v*.**
   pull_request:
-    branches: [ "main" ]
 
 jobs:
   build:
@@ -49,7 +48,10 @@ jobs:
       run: ./gradlew spotlessCheck -x spotlessApply
 
     - name: Build ${{ env.version }}
-      run: ./gradlew build koverXmlReport
+      run: |
+        git submodule init
+        git submodule update --remote
+        ./gradlew build koverXmlReport
       env:
         VERSION: ${{ env.version }}
 

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "csaf"]
+	path = csaf
+	url = https://github.com/oasis-tcs/csaf
+	branch = master

build-logic/src/main/kotlin/buildlogic.kotlin-common-conventions.gradle.kts

@@ -40,7 +40,7 @@ kover {
     reports {
         filters {
             excludes {
-                annotatedBy("io.github.csaf.sbom.retrieval.KoverIgnore")
+                annotatedBy("io.github.csaf.sbom.schema.KoverIgnore")
                 packages("io.github.csaf.sbom.schema.generated")
             }
         }

csaf-import/src/main/kotlin/io/github/csaf/sbom/retrieval/Main.kt

@@ -16,6 +16,7 @@
  */
 package io.github.csaf.sbom.retrieval
 
+import io.github.csaf.sbom.schema.KoverIgnore
 import kotlinx.coroutines.runBlocking
 
 @KoverIgnore("Entry point for demo purposes only")

csaf-import/src/main/kotlin/io/github/csaf/sbom/retrieval/KoverIgnore.kt → csaf-schema/src/main/kotlin/io/github/csaf/sbom/schema/KoverIgnore.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  *
  */
-package io.github.csaf.sbom.retrieval
+package io.github.csaf.sbom.schema
 
 @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION)
 annotation class KoverIgnore(@Suppress("unused") val reason: String)

csaf-validation/build.gradle.kts

@@ -1,5 +1,10 @@
 plugins {
     id("buildlogic.kotlin-library-conventions")
+    application
+}
+
+application {
+    mainClass = "io.github.csaf.sbom.validation.MainKt"
 }
 
 mavenPublishing {

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Main.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2024, The Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.github.csaf.sbom.validation
+
+import io.github.csaf.sbom.schema.KoverIgnore
+import io.github.csaf.sbom.schema.generated.Csaf
+import io.github.csaf.sbom.validation.tests.informativeTests
+import io.github.csaf.sbom.validation.tests.mandatoryTests
+import io.github.csaf.sbom.validation.tests.optionalTests
+import java.time.Duration
+import java.time.Instant
+import kotlin.io.path.Path
+import kotlin.io.path.readText
+import kotlinx.serialization.json.Json
+
+@KoverIgnore("Entry point for demo purposes only")
+fun main(args: Array<String>) {
+    val path = Path(args[0])
+    val doc = Json.decodeFromString<Csaf>(path.readText())
+
+    println("Analyzing file ${path}...\n")
+
+    val globalStart = Instant.now()
+
+    val allTests =
+        mapOf(
+            mandatoryTests to "mandatory",
+            optionalTests to "optional",
+            informativeTests to "informative",
+        )
+
+    for (entry in allTests) {
+        println("== ${entry.value.uppercase()} TESTS ==")
+
+        for (test in entry.key) {
+            val start = Instant.now()
+            val result = test.test(doc)
+            println(
+                "Test ${test::class.simpleName}: $result. It took ${
+                    Duration.between(start, Instant.now()).toMillis()
+                } ms"
+            )
+        }
+
+        println("")
+    }
+
+    println(
+        "Executing all tests took ${Duration.between(globalStart, Instant.now()).toMillis()} ms"
+    )
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Requirement.kt

@@ -51,19 +51,7 @@ fun allOf(vararg requirements: Requirement): Requirement {
 private class AllOf(private val list: List<Requirement>) : Requirement {
     override fun check(ctx: ValidationContext): ValidationResult {
         val results = list.map { it.check(ctx) }
-        return if (results.any { it is ValidationFailed }) {
-            ValidationFailed(
-                results.flatMap {
-                    if (it is ValidationFailed) {
-                        it.errors
-                    } else {
-                        emptyList()
-                    }
-                }
-            )
-        } else {
-            ValidationSuccessful
-        }
+        return results.merge()
     }
 }
 

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Test.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024, The Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.github.csaf.sbom.validation
+
+import io.github.csaf.sbom.schema.generated.Csaf
+
+/**
+ * Represents a test as described in
+ * [Section 6](https://docs.oasis-open.org/csaf/csaf/v2.0/os/csaf-v2.0-os.html#6-tests). They all
+ * target a CSAF document, represented by the [Csaf] type.
+ */
+interface Test {
+
+    fun test(doc: Csaf): ValidationResult
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/ValidationResult.kt

@@ -20,10 +20,10 @@ package io.github.csaf.sbom.validation
 sealed interface ValidationResult
 
 /** A successful validation. */
-object ValidationSuccessful : ValidationResult
+data object ValidationSuccessful : ValidationResult
 
 // TODO(oxisto): Does it make sense to have something like NotApplicable? Currently, this does not
-// propagate
+//  propagate
 val ValidationNotApplicable = ValidationSuccessful
 
 /**
@@ -35,3 +35,20 @@ data class ValidationFailed(
 ) : ValidationResult {
     fun toException() = ValidationException(errors)
 }
+
+/** Merges together the content of all [ValidationResult] objects in this list. */
+fun List<ValidationResult>.merge(): ValidationResult {
+    return if (any { it is ValidationFailed }) {
+        ValidationFailed(
+            flatMap {
+                if (it is ValidationFailed) {
+                    it.errors
+                } else {
+                    emptyList()
+                }
+            }
+        )
+    } else {
+        ValidationSuccessful
+    }
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/requirements/Requirements.kt

@@ -19,6 +19,8 @@ package io.github.csaf.sbom.validation.requirements
 import io.github.csaf.sbom.schema.generated.Csaf
 import io.github.csaf.sbom.schema.generated.Csaf.Label
 import io.github.csaf.sbom.validation.*
+import io.github.csaf.sbom.validation.tests.mandatoryTests
+import io.github.csaf.sbom.validation.tests.test
 import io.ktor.client.request.HttpRequest
 import io.ktor.client.statement.HttpResponse
 import io.ktor.client.statement.request
@@ -31,10 +33,10 @@ import io.ktor.http.*
 object Requirement1ValidCSAFDocument : Requirement {
     override fun check(ctx: ValidationContext): ValidationResult {
         // TODO(oxisto): We need to get the errors from the CSAF schema somehow :(
-        ctx.json ?: return ValidationFailed(listOf("We do not have a valid JSON"))
+        val json =
+            ctx.json as? Csaf ?: return ValidationFailed(listOf("We do not have a valid JSON"))
 
-        // TODO(oxisto): Check for further conformance that are not checked by CSAF schema
-        return ValidationSuccessful
+        return mandatoryTests.test(json)
     }
 }