Impl test report

.gitignore

@@ -1 +1,5 @@
-TODO
+# TODO
+
+.vscode
+
+report/*/

report/Dockerfile

@@ -0,0 +1,6 @@
+ARG IMPLEMENTATION
+
+FROM ${IMPLEMENTATION}
+
+RUN apt-get update && \
+    apt-get install jq -y

report/README.md

@@ -0,0 +1,49 @@
+# Test Suite Runner
+
+This folder contains scripts and docker definitions that can run the suite against an implementation.  The implementations are configured in _implementations.json_.
+
+## Add your implementation
+
+Setting up your application requires several steps.
+
+### Create a CLI
+
+To add your implementation, you'll need to start by creating a CLI application that takes the following parameters:
+
+- `--schema <PATH#POINTER>` - The relative URI to the schema, e.g. `../tests/draft2019-09/additionalItems.json#/0/schema`.
+- `--instance <PATH#POINTER>` - The relative URI to the data instance, e.g. `../tests/draft2019-09/additionalItems.json#/0/tests/0/data`.
+- `--spec-version <VERSION>` - The draft/version for the schema, e.g. `draft2020-12`.  The value for this parameter will match a folder name under `/tests/` in this repo.
+<!-- Do we need this parameter? -->
+<!-- - `--validate-formats` - Indicates whether to validate formats. -->
+
+Internally, you may invoke your implementation as you see fit, setting any options you need to produce the correct outcome.
+
+Validation outcome is indicated by the CLI exit code as follows:
+
+- **-1** or **255** - The instance was determined to be invalid against the schema.
+- **0** - The instance was determined to be valid against the schema.
+- **1** - An application error occurred preventing the schema or instance from being processed.
+- **2** - The scenario is not supported.
+
+Each of these will be represented in the final report.
+
+### Package your CLI
+
+The CLI will need to be packaged into a Linux Docker image.
+
+At a minimum this docker image will need your CLI as well as any runtime needed.
+
+You'll need to push this to a public repository such as Docker Hub so that it can be pulled when the test suite runs.
+
+### Update this repository
+
+Lastly, update a file in this repo.
+
+_implementations.json_ (in this folder) needs an entry that includes the following:
+
+- `docker-image` - The name of your docker image.  The `latest` tag will be used.
+- `command` - The command to run your CLI.
+- `versions` - An array of versions/drafts supported by your implementations.  (To save time, unsupported versions will be skipped.)
+
+Once added, the runner will do the rest.
+

report/docker-compose.yml

@@ -0,0 +1,10 @@
+version: "3.9"
+services:
+  test-suite:
+    image: test-run
+    volumes:
+      - .:/report
+      - ./${IMPLEMENTATION}:/output
+      - ../tests:/tests
+      - ../remotes:/remotes
+    entrypoint: /report/run-suite.sh ${IMPLEMENTATION}

report/implementations.json

@@ -0,0 +1,7 @@
+[
+    {
+        "docker-image": "gregsdennis/json-everything",
+        "command": "dotnet json-everything.dll",
+        "versions": ["draft6", "draft7", "draft2019-09", "draft2020-12"]
+    }
+]

report/report-template.yml

@@ -0,0 +1,6 @@
+# This file was generated by running `/report/run-report.sh ${IMPLEMENTATION}`
+
+# This is the compliance report for the indicated implementation
+
+implementation: ${IMPLEMENTATION}
+results:

report/run-report.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+export IMPLEMENTATION=${1}
+
+echo "Running test suite with '${1}'"
+
+docker image rm test-run -f
+
+docker build -t test-run --build-arg IMPLEMENTATION=${1} .
+
+docker create --name test-run test-run
+
+docker-compose run test-suite
+
+# docker-compose logs --no-color > docker-compose.log
+# [[ -z "${failed:-}" ]] || exit 1

report/run-suite.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+echo "Running test suite with implementation ${1}"
+
+command=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .command" /report/implementations.json)
+versions=$(jq -r ".[] | select(.\"docker-image\"==\"${1}\") | .versions" /report/implementations.json)
+output="/output/result.yml"
+
+sed -e "s~\${IMPLEMENTATION}~${1}~" /report/report-template.yml > ${output}
+
+if [[ "${command}" == "" ]]
+then
+    echo "Implementation either not found or not configured with a command to execute"
+    exit 1
+fi
+
+for d in /tests/*/
+do
+    specVersion=$(basename $d)
+
+    if ! [[ ${versions[@]} =~ $specVersion ]]
+    then
+        # record that the draft isn't supported
+        continue
+    fi
+
+    for f in $(find $d) # $d contains the ending /
+    do
+        filename=$(basename $f)
+        if [[ $f == *"optional"* ]]
+        then
+            optional=true
+        else
+            optional=false
+        fi
+
+        scenarioIndex=0
+        jq -cr '.[]' < $f | while read j
+        do
+            scenario=$(echo $j | jq -r '.description')
+            schemaUri="$f#/$scenarioIndex/schema"
+            caseIndex=0
+            echo $j | jq -c '.tests[]' | while read c
+            do
+                case=$(echo $c | jq -r '.description')
+                instanceUri="$f#/$scenarioIndex/tests/$caseIndex/data"
+                case "$(echo $c | jq '.valid')" in
+                    true) expected="valid" ;;
+                    false) expected="invalid" ;;
+                esac
+
+                ${command} --schema $schemaUri --instance $instanceUri --spec-version $specVersion
+                evaluated=$?
+
+                case $evaluated in
+                    255) result="invalid" ;;
+                    0) result="valid" ;;
+                    1) result="error" ;;
+                    2) result="unsupported" ;;
+                esac
+
+                echo "  - draft: ${specVersion}" >> $output
+                echo "    file: ${filename%%.*}" >> $output
+                echo "    scenario: ${scenario}" >> $output
+                echo "    case: ${case}" >> $output
+                echo "    optional: ${optional}" >> $output
+                echo "    expected: ${expected}" >> $output
+                echo "    result: ${result}" >> $output
+
+                ((caseIndex++))
+            done
+            ((scenarioIndex++))
+        done
+    done
+done