Coverage for pyTooling/Testing/ReportWriter.py: 100%

1# ==================================================================================================================== # 

2# _____ _ _ _____ _ _ # 

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _|_ _|__ ___| |_(_)_ __ __ _ # 

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | |/ _ \/ __| __| | '_ \ / _` | # 

5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| | __/\__ \ |_| | | | | (_| | # 

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)_|\___||___/\__|_|_| |_|\__, | # 

7# |_| |___/ |___/ |___/ # 

8# ==================================================================================================================== # 

9# Authors: # 

10# Patrick Lehmann # 

11# # 

12# License: # 

13# ==================================================================================================================== # 

14# Copyright 2026-2026 Patrick Lehmann - Bötzingen, Germany # 

15# # 

16# Licensed under the Apache License, Version 2.0 (the "License"); # 

17# you may not use this file except in compliance with the License. # 

18# You may obtain a copy of the License at # 

19# # 

20# http://www.apache.org/licenses/LICENSE-2.0 # 

21# # 

22# Unless required by applicable law or agreed to in writing, software # 

23# distributed under the License is distributed on an "AS IS" BASIS, # 

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 

25# See the License for the specific language governing permissions and # 

26# limitations under the License. # 

27# # 

28# SPDX-License-Identifier: Apache-2.0 # 

29# ==================================================================================================================== # 

30# 

31""" 

32A pytest plugin writing a test report in :ref:`pyTooling's own format <TESTING/ReportFormat>`. 

33 

34JUnit XML cannot express two things a marked test suite has: test suites **nest**, where JUnit flattens them into a 

35dotted ``classname``; and every item carries **four names** - an identifier, a title, a summary and a description - 

36where JUnit has one name and a bag of flat ``<property>`` pairs. 

37 

38This writer is opt-in through ``--pytooling-xml=PATH`` and runs happily alongside ``--junit-xml``, so a pipeline 

39keeps the format its dashboard understands while the richer file is produced from the same reports. 

40 

41The format's version is the version of its schema - :file:`TestReport-v0.1.xsd` today - so a later version is added 

42beside it and a reader knows from ``xsi:noNamespaceSchemaLocation`` which one it is holding. 

43 

44.. todo:: TESTING::ReportWriter Name the generator in a ``<Generator>`` element 

45 

46 The tool that wrote a report is not part of the format's version, so it is no longer an attribute on the root 

47 element. Where it belongs is an element of its own at root level, carrying the tool's name and version and 

48 whatever further meta information a tool wants to add. 

49 

50.. todo:: TESTING::ReportWriter Add a ``<TestRun>`` level between the report and its test suites 

51 

52 A run happens on one machine, in one environment, at one time - so a report can hold several of them, and the 

53 things that describe a run belong to the run rather than to the report: 

54 

55 .. code-block:: text 

56 

57 <TestReport> 

58 <TestRun timestamp="..." duration="..."> 

59 <Environment> 

60 <Hostname>...</Hostname> 

61 <OperatingSystem>...</OperatingSystem> 

62 </Environment> 

63 <Summary>...</Summary> 

64 <Description>...</Description> 

65 <Testsuite ... /> 

66 </TestRun> 

67 </TestReport> 

68 

69 The environment is optional, which is what makes a hostname acceptable there: it is written when whoever 

70 generates the report decides it belongs in it, not by default. 

71 

72.. hint:: 

73 

74 See :ref:`high-level help <TESTING/ReportFormat>` for the schema and an example. 

75""" 

76from datetime import datetime, timezone 

77from pathlib import Path 

78from typing import Any, TypedDict, Optional as Nullable 

79from xml.etree.ElementTree import Element, ElementTree, SubElement, indent 

80 

81from pytest import Config, Parser, Session, StashKey 

82from pyTooling.Decorators import export 

83from pyTooling.MetaClasses import ExtendedType 

84from pyTooling.Testing.PyTest import hierarchyKey 

85 

86 

87__all__ = ["SCHEMA_VERSION_LATEST", "SCHEMA_FILES", "REPORT_WRITER_KEY"] 

88 

89 

90SCHEMA_VERSION_LATEST = "v0.1" #: Latest version of the report format, and the one this writer produces. 

91 

92SCHEMA_FILES: dict[str, str] = { 

93 "v0.1": "TestReport-v0.1.xsd", 

94} #: Schema file per format version, so a later version is added beside the one in use, not instead of it. 

95 

96REPORT_WRITER_KEY: StashKey["TestReportWriter"] = StashKey() #: Where the writer is stashed on the configuration. 

97 

98 

99class _Result(TypedDict, total=False): 

100 """What is collected per testcase, assembled from the reports of its phases.""" 

101 

102 nodeID: str #: Node ID of the testcase, whose parts name the test suite levels. 

103 duration: float #: Sum of the durations of the testcase's phases. 

104 message: str #: Text of the failure, or an empty string. 

105 status: str #: ``passed``, ``failed``, ``errored`` or ``skipped``. 

106 title: str #: Title of the testcase, if it is marked. 

107 summary: str #: Summary of the testcase, if its doc-string has one. 

108 description: str #: Description of the testcase, if its doc-string has one. 

109 

110 

111@export 

112class TestReportWriter(metaclass=ExtendedType, slots=True): 

113 """ 

114 Collects the reports of a session and writes them as one nested XML document. 

115 

116 The nesting comes from the dotted ``classname`` pytest reports: ``tests.unit.Versioning`` becomes three levels 

117 of ``<Testsuite>``, so a reader sees the hierarchy the test suite actually has. 

118 """ 

119 

120 _path: Path #: Where the report is written. 

121 _results: dict[str, _Result] #: Collected results, keyed by node ID. 

122 _hierarchy: dict[str, dict[str, str]] #: Names of every test suite level, keyed by its dotted path. 

123 

124 def __init__(self, path: Path, hierarchy: Nullable[dict[str, dict[str, str]]] = None) -> None: 

125 """ 

126 Initializes the writer with the path it writes to. 

127 

128 :param path: Path of the report file to write. 

129 :param hierarchy: Optional, the names of every test suite level, keyed by its dotted path, as 

130 :mod:`pyTooling.Testing.PyTest` collects them. Without it - when the marker plugin is not 

131 registered - the test suite elements carry no names. 

132 """ 

133 self._path = path 

134 self._results = {} 

135 self._hierarchy = {} if hierarchy is None else hierarchy 

136 

137 def pytest_runtest_logreport(self, report: Any) -> None: 

138 """ 

139 Collect one phase of one testcase. 

140 

141 :param report: The report of one phase of one testcase. 

142 """ 

143 entry = self._results.setdefault(report.nodeid, {"nodeID": report.nodeid, "duration": 0.0, "message": ""}) 

144 entry["duration"] += report.duration 

145 entry.update(dict(report.user_properties)) 

146 

147 if report.failed: 

148 entry["status"] = "errored" if report.when != "call" else "failed" 

149 entry["message"] = str(report.longrepr) if report.longrepr is not None else "" 

150 elif report.skipped: 

151 entry.setdefault("status", "skipped") 

152 elif report.when == "call": 

153 entry.setdefault("status", "passed") 

154 

155 def _testsuiteFor(self, root: Element, suites: dict[str, Element], nodeID: str) -> Element: 

156 """ 

157 Return the ``<Testsuite>`` element a testcase belongs into, creating the missing levels on the way. 

158 

159 :param root: The report element every path starts at. 

160 :param suites: Elements created so far, keyed by their dotted path. 

161 :param nodeID: Node ID of the testcase, whose ``::``-separated parts name the levels. 

162 :returns: The innermost test suite element. 

163 """ 

164 modulePath, _, remainder = nodeID.partition("::") 

165 levels = [*Path(modulePath).with_suffix("").parts, *remainder.split("::")[:-1]] 

166 

167 parent = root 

168 path = "" 

169 for level in levels: 

170 path = f"{path}.{level}" if path != "" else level 

171 if (suite := suites.get(path)) is None: 

172 suite = suites[path] = SubElement(parent, "Testsuite", {"name": level}) 

173 

174 # The names are the level's own, so they are written where the level is created - once, however many 

175 # testcases it holds. The schema requires this order, and a level contributes only the names it has. 

176 levelNames = self._hierarchy.get(path, {}) 

177 for name in ("Title", "Summary", "Description"): 

178 if (value := levelNames.get(name[0].lower() + name[1:], "")) != "": 

179 SubElement(suite, name).text = value 

180 

181 parent = suite 

182 

183 return parent 

184 

185 def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: 

186 """ 

187 Write the collected results as one XML document. 

188 

189 :param session: The finished session. 

190 :param exitstatus: The exit status the session ended with. 

191 """ 

192 statuses = [entry.get("status", "errored") for entry in self._results.values()] 

193 root = Element("TestReport", { 

194 "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", 

195 "xsi:noNamespaceSchemaLocation": SCHEMA_FILES[SCHEMA_VERSION_LATEST], 

196 "timestamp": datetime.now(timezone.utc).isoformat(), 

197 "duration": f"{sum(entry['duration'] for entry in self._results.values()):.6f}", 

198 "tests": str(len(statuses)), 

199 "failures": str(statuses.count("failed")), 

200 "errors": str(statuses.count("errored")), 

201 "skipped": str(statuses.count("skipped")), 

202 }) 

203 

204 suites: dict[str, Element] = {} 

205 for entry in self._results.values(): 

206 testsuite = self._testsuiteFor(root, suites, entry["nodeID"]) 

207 

208 testcase = SubElement(testsuite, "Testcase", { 

209 "name": entry["nodeID"].rsplit("::", 1)[-1], 

210 "status": entry.get("status", "errored"), 

211 "duration": f"{entry['duration']:.6f}", 

212 "nodeID": entry["nodeID"], 

213 }) 

214 # Ordered as the schema requires; an unmarked testcase carries none of them. 

215 for name in ("Title", "Summary", "Description"): 

216 if (value := entry.get(name[0].lower() + name[1:], "")) != "": 

217 SubElement(testcase, name).text = value 

218 

219 if entry["message"] != "": 

220 SubElement(testcase, "Message").text = entry["message"] 

221 

222 tree = ElementTree(root) 

223 indent(tree, space="\t") 

224 self._path.parent.mkdir(parents=True, exist_ok=True) 

225 tree.write(self._path, encoding="utf-8", xml_declaration=True) 

226 

227 

228@export 

229def pytest_addoption(parser: Parser) -> None: 

230 """ 

231 Add the ``--pytooling-xml`` option selecting the report's path. 

232 

233 :param parser: The command line parser to add the option to. 

234 """ 

235 group = parser.getgroup("pyTooling") 

236 group.addoption( 

237 "--pytooling-xml", action="store", default=None, metavar="PATH", 

238 help="Write a pyTooling test report to PATH." 

239 ) 

240 

241 

242@export 

243def pytest_configure(config: Config) -> None: 

244 """ 

245 Register the writer, if a path was given. 

246 

247 :param config: The session's configuration. 

248 """ 

249 if (path := config.getoption("--pytooling-xml")) is not None: 

250 # 'setdefault' rather than 'get': this hook runs before collection, so the marker plugin has not filled the 

251 # stash yet. Creating the dictionary here hands both plugins the same object, which it then updates in place. 

252 writer = TestReportWriter(Path(path), config.stash.setdefault(hierarchyKey, {})) 

253 config.stash[REPORT_WRITER_KEY] = writer 

254 config.pluginmanager.register(writer, "pyTooling.Testing.TestReportWriter")