From 67bc432f7d7108702aac4c346d5a5ab41c9c2b19 Mon Sep 17 00:00:00 2001 From: Damien Nozay Date: Mon, 17 Feb 2020 16:29:39 -0800 Subject: [PATCH] info / fixes about xUnit plugin (#215) - see #209; add a utility function for XSLT transformation. - minor doc fixes. - fix incorrect version, I called out xunit plugin 2.2.4 earlier and I must not have looked at the version properly. --- README.md | 83 +++++++--- tests/testsuite.py | 44 ++++-- .../junit-10.xsd | 0 .../junit-10.xsd | 147 ++++++++++++++++++ xmlrunner/extra/xunit_plugin.py | 28 ++++ 5 files changed, 273 insertions(+), 29 deletions(-) rename tests/vendor/jenkins/xunit-plugin/{ => 14c6e39c38408b9ed6280361484a13c6f5becca7}/junit-10.xsd (100%) create mode 100644 tests/vendor/jenkins/xunit-plugin/ae25da5089d4f94ac6c4669bf736e4d416cc4665/junit-10.xsd create mode 100644 xmlrunner/extra/xunit_plugin.py diff --git a/README.md b/README.md index 3af5835..7bdb9f3 100644 --- a/README.md +++ b/README.md @@ -14,35 +14,82 @@ A unittest test runner that can save test results to XML files in xUnit format. The files can be consumed by a wide range of tools, such as build systems, IDEs and continuous integration servers. -## Schema -There are many schemas with minor differences. -We use one that is compatible with Jenkins xUnit plugin, a copy is -available under `tests/vendor/jenkins/xunit-plugin/junit-10.xsd` (see attached license). +## Requirements + +* Python 3.5+ +* Please note Python 2.7 end-of-life was in Jan 2020, last version supporting 2.7 was 2.5.2 +* Please note Python 3.4 end-of-life was in Mar 2019, last version supporting 3.4 was 2.5.2 +* Please note Python 2.6 end-of-life was in Oct 2013, last version supporting 2.6 was 1.14.0 + + +## Limited support for `unittest.TestCase.subTest` + +https://docs.python.org/3/library/unittest.html#unittest.TestCase.subTest + +`unittest` has the concept of sub-tests for a `unittest.TestCase`; this doesn't map well to an existing xUnit concept, so you won't find it in the schema. What that means, is that you lose some granularity +in the reports for sub-tests. + +`unittest` also does not report successful sub-tests, so the accounting won't be exact. -- [Jenkins (junit-10.xsd), xunit plugin (2014-2018)](https://github.com/jenkinsci/xunit-plugin/blob/14c6e39c38408b9ed6280361484a13c6f5becca7/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd), please note the latest versions (2.2.4 and above are not backwards compatible) +## Jenkins plugins + +- Jenkins JUnit plugin : https://plugins.jenkins.io/junit/ +- Jenkins xUnit plugin : https://plugins.jenkins.io/xunit/ + +### Jenkins JUnit plugin + +This plugin does not perform XSD validation (at time of writing) and should parse the XML file without issues. + +### Jenkins xUnit plugin version 1.100 + +- [Jenkins (junit-10.xsd), xunit plugin (2014-2018)](https://github.com/jenkinsci/xunit-plugin/blob/14c6e39c38408b9ed6280361484a13c6f5becca7/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd), version `1.100`. + +This plugin does perfom XSD validation and uses the more lax XSD. This should parse the XML file without issues. + +### Jenkins xUnit plugin version 1.104+ + +- [Jenkins (junit-10.xsd), xunit plugin (2018-current)](https://github.com/jenkinsci/xunit-plugin/blob/ae25da5089d4f94ac6c4669bf736e4d416cc4665/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd), version `1.104`+. + +This plugin does perfom XSD validation and uses the more strict XSD. + +See https://github.com/xmlrunner/unittest-xml-reporting/issues/209 + +``` +import io +import unittest +import xmlrunner + +# run the tests storing results in memory +out = io.BytesIO() +unittest.main( + testRunner=xmlrunner.XMLTestRunner(output=out), + failfast=False, buffer=False, catchbreak=False, exit=False) +``` + +Transform the results removing extra attributes. +``` +from xmlrunner.extra.xunit_plugin import transform + +with open('TEST-report.xml', 'wb') as report: + report.write(transform(out.getvalue())) + +``` + +## JUnit Schema ? + +There are many tools claiming to write JUnit reports, so you will find many schemas with minor differences. + +We used the XSD that was available in the Jenkins xUnit plugin version `1.100`; a copy is available under `tests/vendor/jenkins/xunit-plugin/.../junit-10.xsd` (see attached license). You may also find these resources useful: - https://stackoverflow.com/questions/4922867/what-is-the-junit-xml-format-specification-that-hudson-supports - https://stackoverflow.com/questions/11241781/python-unittests-in-jenkins -- [Jenkins (junit-10.xsd), xunit plugin 2.2.4+](https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd) - [JUnit-Schema (JUnit.xsd)](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd) - [Windyroad (JUnit.xsd)](http://windyroad.com.au/dl/Open%20Source/JUnit.xsd) - [a gist (Jenkins xUnit test result schema)](https://gist.github.com/erikd/4192748) -## Things that are somewhat broken - -Python 3 has the concept of sub-tests for a `unittest.TestCase`; this doesn't map well to an existing -xUnit concept, so you won't find it in the schema. What that means, is that you lose some granularity -in the reports for sub-tests. - -## Requirements - -* Python 3.5+ -* Please note Python 2.7 end-of-life was in Jan 2020, last version supporting 2.7 was 2.5.2 -* Please note Python 3.4 end-of-life was in Mar 2019, last version supporting 3.4 was 2.5.2 -* Please note Python 2.6 end-of-life was in Oct 2013, last version supporting 2.6 was 1.14.0 ## Installation diff --git a/tests/testsuite.py b/tests/testsuite.py index 9e87353..431547b 100755 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -27,10 +27,10 @@ from unittest import mock -def _load_schema(): - path = os.path.join(os.path.dirname(__file__), - 'vendor/jenkins/xunit-plugin', - 'junit-10.xsd') +def _load_schema(version): + path = os.path.join( + os.path.dirname(__file__), + 'vendor/jenkins/xunit-plugin', version, 'junit-10.xsd') with open(path, 'r') as schema_file: schema_doc = etree.parse(schema_file) schema = etree.XMLSchema(schema_doc) @@ -38,12 +38,10 @@ def _load_schema(): raise RuntimeError('Could not load JUnit schema') # pragma: no cover -JUnitSchema = _load_schema() - - -def validate_junit_report(text): +def validate_junit_report(version, text): document = etree.parse(BytesIO(text)) - JUnitSchema.assertValid(document) + schema = _load_schema(version) + schema.assertValid(document) class TestCaseSubclassWithNoSuper(unittest.TestCase): @@ -650,7 +648,7 @@ def test_junitxml_xsd_validation_order(self): self.assertTrue(i_properties < i_testcase < i_system_out < i_system_err) # XSD validation - for good measure. - validate_junit_report(output) + validate_junit_report('14c6e39c38408b9ed6280361484a13c6f5becca7', output) def test_junitxml_xsd_validation_empty_properties(self): suite = unittest.TestSuite() @@ -665,7 +663,31 @@ def test_junitxml_xsd_validation_empty_properties(self): outdir.seek(0) output = outdir.read() self.assertNotIn(''.encode('utf8'), output) - validate_junit_report(output) + validate_junit_report('14c6e39c38408b9ed6280361484a13c6f5becca7', output) + + def test_xunit_plugin_transform(self): + suite = unittest.TestSuite() + suite.addTest(self.DummyTest('test_fail')) + suite.addTest(self.DummyTest('test_pass')) + suite.properties = None + outdir = BytesIO() + runner = xmlrunner.XMLTestRunner( + stream=self.stream, output=outdir, verbosity=self.verbosity, + **self.runner_kwargs) + runner.run(suite) + outdir.seek(0) + output = outdir.read() + + validate_junit_report('14c6e39c38408b9ed6280361484a13c6f5becca7', output) + with self.assertRaises(etree.DocumentInvalid): + validate_junit_report('ae25da5089d4f94ac6c4669bf736e4d416cc4665', output) + + from xmlrunner.extra.xunit_plugin import transform + transformed = transform(output) + validate_junit_report('14c6e39c38408b9ed6280361484a13c6f5becca7', transformed) + validate_junit_report('ae25da5089d4f94ac6c4669bf736e4d416cc4665', transformed) + self.assertIn('test_pass'.encode('utf8'), transformed) + self.assertIn('test_fail'.encode('utf8'), transformed) def test_xmlrunner_elapsed_times(self): self.runner_kwargs['elapsed_times'] = False diff --git a/tests/vendor/jenkins/xunit-plugin/junit-10.xsd b/tests/vendor/jenkins/xunit-plugin/14c6e39c38408b9ed6280361484a13c6f5becca7/junit-10.xsd similarity index 100% rename from tests/vendor/jenkins/xunit-plugin/junit-10.xsd rename to tests/vendor/jenkins/xunit-plugin/14c6e39c38408b9ed6280361484a13c6f5becca7/junit-10.xsd diff --git a/tests/vendor/jenkins/xunit-plugin/ae25da5089d4f94ac6c4669bf736e4d416cc4665/junit-10.xsd b/tests/vendor/jenkins/xunit-plugin/ae25da5089d4f94ac6c4669bf736e4d416cc4665/junit-10.xsd new file mode 100644 index 0000000..286fbf7 --- /dev/null +++ b/tests/vendor/jenkins/xunit-plugin/ae25da5089d4f94ac6c4669bf736e4d416cc4665/junit-10.xsd @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xmlrunner/extra/xunit_plugin.py b/xmlrunner/extra/xunit_plugin.py new file mode 100644 index 0000000..46b4cf4 --- /dev/null +++ b/xmlrunner/extra/xunit_plugin.py @@ -0,0 +1,28 @@ +import io +import lxml.etree as etree + + +TRANSFORM = etree.XSLT(etree.XML('''\ + + + + + + + + + + + + + + +''')) + + +def transform(xml_data): + out = io.BytesIO() + xml_doc = etree.XML(xml_data) + result = TRANSFORM(xml_doc) + result.write(out) + return out.getvalue()