Initial draft of a validation testing framework (Tests 6.1.1-6.1.7)

.github/workflows/gradle.yml

@@ -7,7 +7,6 @@ on:
     tags:
       - v*.**
   pull_request:
-    branches: [ "main" ]
 
 jobs:
   build:
@@ -18,6 +17,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: 'true'
     - name: Set up JDK 21
       uses: actions/setup-java@v4
       with:
@@ -49,7 +50,8 @@ jobs:
       run: ./gradlew spotlessCheck -x spotlessApply
 
     - name: Build ${{ env.version }}
-      run: ./gradlew build koverXmlReport
+      run: |
+        ./gradlew build koverXmlReport
       env:
         VERSION: ${{ env.version }}
 

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "csaf"]
+	path = csaf
+	url = https://github.com/oasis-tcs/csaf
+	branch = master

build-logic/src/main/kotlin/buildlogic.kotlin-common-conventions.gradle.kts

@@ -40,7 +40,7 @@ kover {
     reports {
         filters {
             excludes {
-                annotatedBy("io.github.csaf.sbom.retrieval.KoverIgnore")
+                annotatedBy("io.github.csaf.sbom.schema.KoverIgnore")
                 packages("io.github.csaf.sbom.schema.generated")
             }
         }

csaf-import/src/main/kotlin/io/github/csaf/sbom/retrieval/Main.kt

@@ -16,6 +16,7 @@
  */
 package io.github.csaf.sbom.retrieval
 
+import io.github.csaf.sbom.schema.KoverIgnore
 import kotlinx.coroutines.runBlocking
 
 @KoverIgnore("Entry point for demo purposes only")

csaf-import/src/main/kotlin/io/github/csaf/sbom/retrieval/KoverIgnore.kt → csaf-schema/src/main/kotlin/io/github/csaf/sbom/schema/KoverIgnore.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  *
  */
-package io.github.csaf.sbom.retrieval
+package io.github.csaf.sbom.schema
 
 @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION)
 annotation class KoverIgnore(@Suppress("unused") val reason: String)

csaf-validation/build.gradle.kts

@@ -1,5 +1,10 @@
 plugins {
     id("buildlogic.kotlin-library-conventions")
+    application
+}
+
+application {
+    mainClass = "io.github.csaf.sbom.validation.MainKt"
 }
 
 mavenPublishing {

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Main.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2024, The Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.github.csaf.sbom.validation
+
+import io.github.csaf.sbom.schema.KoverIgnore
+import io.github.csaf.sbom.schema.generated.Csaf
+import io.github.csaf.sbom.validation.tests.informativeTests
+import io.github.csaf.sbom.validation.tests.mandatoryTests
+import io.github.csaf.sbom.validation.tests.optionalTests
+import java.time.Duration
+import java.time.Instant
+import kotlin.io.path.Path
+import kotlin.io.path.readText
+import kotlinx.serialization.json.Json
+
+@KoverIgnore("Entry point for demo purposes only")
+fun main(args: Array<String>) {
+    val path = Path(args[0])
+    val doc = Json.decodeFromString<Csaf>(path.readText())
+
+    println("Analyzing file ${path}...\n")
+
+    val globalStart = Instant.now()
+
+    val allTests =
+        mapOf(
+            mandatoryTests to "mandatory",
+            optionalTests to "optional",
+            informativeTests to "informative",
+        )
+
+    for (entry in allTests) {
+        println("== ${entry.value.uppercase()} TESTS ==")
+
+        for (test in entry.key) {
+            val start = Instant.now()
+            val result = test.test(doc)
+            println(
+                "Test ${test::class.simpleName}: $result. It took ${
+                    Duration.between(start, Instant.now()).toMillis()
+                } ms"
+            )
+        }
+
+        println("")
+    }
+
+    println(
+        "Executing all tests took ${Duration.between(globalStart, Instant.now()).toMillis()} ms"
+    )
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Requirement.kt

@@ -51,19 +51,7 @@ fun allOf(vararg requirements: Requirement): Requirement {
 private class AllOf(private val list: List<Requirement>) : Requirement {
     override fun check(ctx: ValidationContext): ValidationResult {
         val results = list.map { it.check(ctx) }
-        return if (results.any { it is ValidationFailed }) {
-            ValidationFailed(
-                results.flatMap {
-                    if (it is ValidationFailed) {
-                        it.errors
-                    } else {
-                        emptyList()
-                    }
-                }
-            )
-        } else {
-            ValidationSuccessful
-        }
+        return results.merge()
     }
 }
 

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/Test.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024, The Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.github.csaf.sbom.validation
+
+import io.github.csaf.sbom.schema.generated.Csaf
+
+/**
+ * Represents a test as described in
+ * [Section 6](https://docs.oasis-open.org/csaf/csaf/v2.0/os/csaf-v2.0-os.html#6-tests). They all
+ * target a CSAF document, represented by the [Csaf] type.
+ */
+interface Test {
+
+    fun test(doc: Csaf): ValidationResult
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/ValidationResult.kt

@@ -20,10 +20,10 @@ package io.github.csaf.sbom.validation
 sealed interface ValidationResult
 
 /** A successful validation. */
-object ValidationSuccessful : ValidationResult
+data object ValidationSuccessful : ValidationResult
 
 // TODO(oxisto): Does it make sense to have something like NotApplicable? Currently, this does not
-// propagate
+//  propagate
 val ValidationNotApplicable = ValidationSuccessful
 
 /**
@@ -35,3 +35,20 @@ data class ValidationFailed(
 ) : ValidationResult {
     fun toException() = ValidationException(errors)
 }
+
+/** Merges together the content of all [ValidationResult] objects in this list. */
+fun List<ValidationResult>.merge(): ValidationResult {
+    return if (any { it is ValidationFailed }) {
+        ValidationFailed(
+            flatMap {
+                if (it is ValidationFailed) {
+                    it.errors
+                } else {
+                    emptyList()
+                }
+            }
+        )
+    } else {
+        ValidationSuccessful
+    }
+}

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/requirements/Requirements.kt

@@ -19,6 +19,8 @@ package io.github.csaf.sbom.validation.requirements
 import io.github.csaf.sbom.schema.generated.Csaf
 import io.github.csaf.sbom.schema.generated.Csaf.Label
 import io.github.csaf.sbom.validation.*
+import io.github.csaf.sbom.validation.tests.mandatoryTests
+import io.github.csaf.sbom.validation.tests.test
 import io.ktor.client.request.HttpRequest
 import io.ktor.client.statement.HttpResponse
 import io.ktor.client.statement.request
@@ -31,10 +33,10 @@ import io.ktor.http.*
 object Requirement1ValidCSAFDocument : Requirement {
     override fun check(ctx: ValidationContext): ValidationResult {
         // TODO(oxisto): We need to get the errors from the CSAF schema somehow :(
-        ctx.json ?: return ValidationFailed(listOf("We do not have a valid JSON"))
+        val json =
+            ctx.json as? Csaf ?: return ValidationFailed(listOf("We do not have a valid JSON"))
 
-        // TODO(oxisto): Check for further conformance that are not checked by CSAF schema
-        return ValidationSuccessful
+        return mandatoryTests.test(json)
     }
 }
 

csaf-validation/src/main/kotlin/io/github/csaf/sbom/validation/tests/Extensions.kt

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2024, The Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.github.csaf.sbom.validation.tests
+
+import io.github.csaf.sbom.schema.generated.Csaf
+import io.github.csaf.sbom.schema.generated.Csaf.Product
+import io.github.csaf.sbom.validation.tests.plusAssign
+
+/**
+ * Gathers product definitions at a [Csaf.Branche]. This is needed because we need to do it
+ * recursively.
+ */
+fun Csaf.Branche.gatherProductDefinitionsTo(products: MutableCollection<String>) {
+    // Add ID at this leaf
+    products += product?.product_id
+
+    // Go down the branch
+    this.branches?.forEach { it.gatherProductDefinitionsTo(products) }
+}
+
+/** Gathers all [Product.product_id] definitions in the current document. */
+fun Csaf.gatherProductDefinitions(): List<String> {
+    val ids = mutableListOf<String>()
+
+    // /product_tree/branches[](/branches[])*/product/product_id
+    ids +=
+        this.product_tree?.branches?.flatMap {
+            var inner = mutableListOf<String>()
+            it.gatherProductDefinitionsTo(inner)
+            inner
+        }
+
+    // /product_tree/full_product_names[]/product_id
+    ids += this.product_tree?.full_product_names?.map { it.product_id }
+
+    // /product_tree/relationships[]/full_product_name/product_id
+    ids += this.product_tree?.relationships?.map { it.full_product_name.product_id }
+
+    return ids
+}
+
+/** Gathers all product IDs in the current document. */
+fun Csaf.gatherProductReferences(): Set<String> {
+    val ids = mutableSetOf<String>()
+
+    // /product_tree/product_groups[]/product_ids[]
+    ids += product_tree?.product_groups?.flatMap { it.product_ids }
+
+    // /product_tree/relationships[]/product_reference
+    // /product_tree/relationships[]/relates_to_product_reference
+    ids +=
+        product_tree?.relationships?.flatMap {
+            listOf(it.product_reference, it.relates_to_product_reference)
+        }
+
+    // /vulnerabilities[]/product_status/first_affected[]
+    // /vulnerabilities[]/product_status/first_fixed[]
+    // /vulnerabilities[]/product_status/fixed[]
+    // /vulnerabilities[]/product_status/known_affected[]
+    // /vulnerabilities[]/product_status/known_not_affected[]
+    // /vulnerabilities[]/product_status/last_affected[]
+    // /vulnerabilities[]/product_status/recommended[]
+    // /vulnerabilities[]/product_status/under_investigation[]
+    // /vulnerabilities[]/remediations[]/product_ids[]
+    // /vulnerabilities[]/scores[]/products[]
+    // /vulnerabilities[]/threats[]/product_ids[]
+    ids +=
+        vulnerabilities?.flatMap {
+            var inner = mutableSetOf<String>()
+            inner += it.product_status?.first_affected
+            inner += it.product_status?.first_fixed
+            inner += it.product_status?.fixed
+            inner += it.product_status?.known_affected
+            inner += it.product_status?.known_not_affected
+            inner += it.product_status?.last_affected
+            inner += it.product_status?.recommended
+            inner += it.product_status?.under_investigation
+            inner += it.remediations?.flatMap { it.product_ids ?: emptySet() }
+            inner += it.scores?.flatMap { it.products }
+            inner += it.threats?.flatMap { it.product_ids ?: emptySet() }
+            inner
+        }
+
+    return ids
+}
+
+/** Gathers all [Csaf.ProductGroup.group_id] definitions in the current document. */
+fun Csaf.gatherProductGroups(): List<String> {
+    val groups = mutableListOf<String>()
+
+    // /product_tree/product_groups[]/group_id
+    groups += product_tree?.product_groups?.map { it.group_id }
+
+    return groups
+}
+
+/** Gather all group ID references in the current document. */
+fun Csaf.gatherProductGroupReferences(): Set<String> {
+    val ids = mutableSetOf<String>()
+
+    // /vulnerabilities[]/remediations[]/group_ids
+    // /vulnerabilities[]/threats[]/group_ids
+    ids +=
+        vulnerabilities?.flatMap {
+            var inner = mutableSetOf<String>()
+            inner += it.remediations?.flatMap { it.group_ids ?: setOf() }
+            inner += it.threats?.flatMap { it.group_ids ?: setOf() }
+            inner
+        }
+
+    return ids
+}
+
+internal operator fun <E> MutableCollection<E>.plusAssign(set: Collection<E>?) {
+    if (set != null) {
+        this.addAll(set)
+    }
+}
+
+internal operator fun <E> MutableCollection<E>.plusAssign(item: E?) {
+    if (item != null) {
+        this.add(item)
+    }
+}
+
+internal operator fun <E> Collection<E>?.plus(other: Collection<E>?): Collection<E> {
+    return if (other != null && this != null) {
+        this.union(other)
+    } else if (other != null) {
+        other
+    } else if (this != null) {
+        this
+    } else {
+        listOf()
+    }
+}