Coverage for sphinx_reports/Unittest.py: 19%

1# ==================================================================================================================== #

2# _ _ _ #

3# ___ _ __ | |__ (_)_ __ __ __ _ __ ___ _ __ ___ _ __| |_ ___ #

4# / __| '_ \| '_ \| | '_ \\ \/ /____| '__/ _ \ '_ \ / _ \| '__| __/ __| #

5# \__ \ |_) | | | | | | | |> <_____| | | __/ |_) | (_) | | | |_\__ \ #

6# |___/ .__/|_| |_|_|_| |_/_/\_\ |_| \___| .__/ \___/|_| \__|___/ #

7# |_| |_| #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32**Report unit test results as Sphinx documentation page(s).**

33"""

34from datetime import timedelta

35from enum import Flag

36from pathlib import Path

37from typing import Dict, Tuple, Any, List, Mapping, Generator, TypedDict, ClassVar, Optional as Nullable

39from docutils import nodes

40from docutils.parsers.rst.directives import flag

41from pyTooling.Decorators import export

42from pyEDAA.Reports.Unittesting import TestcaseStatus, TestsuiteStatus

43from pyEDAA.Reports.Unittesting.JUnit import Testsuite, TestsuiteSummary, Testcase, Document

44from sphinx.application import Sphinx

45from sphinx.config import Config

47from sphinx_reports.Common import ReportExtensionError

48from sphinx_reports.Sphinx import strip, BaseDirective

51class report_DictType(TypedDict):

52 xml_report: Path

55@export

56class ShowTestcases(Flag):

57 passed = 1

58 failed = 2

59 skipped = 4

60 excluded = 8

61 errors = 16

62 aborted = 32

65 not_passed = all & ~passed

67 def __eq__(self, other):

68 if isinstance(other, TestcaseStatus):

69 if other is TestcaseStatus.Passed:

70 return ShowTestcases.passed in self

71 elif other is TestcaseStatus.Failed:

72 return ShowTestcases.failed in self

73 elif other is TestcaseStatus.Skipped:

74 return ShowTestcases.skipped in self

75 elif other is TestcaseStatus.Excluded:

76 return ShowTestcases.excluded in self

77 elif other is TestcaseStatus.Error or other is TestcaseStatus.SetupError:

78 return ShowTestcases.errors in self

79 elif other is TestcaseStatus.Aborted:

80 return ShowTestcases.aborted in self

82 return False

85@export

86class UnittestSummary(BaseDirective):

87 """

88 This directive will be replaced by a table representing unit test results.

89 """

90 has_content = False

91 required_arguments = 0

92 optional_arguments = 6

94 option_spec = {

95 "class": strip,

96 "reportid": strip,

97 "testsuite-summary-name": strip,

98 "show-testcases": strip,

99 "no-assertions": flag,

100 "hide-testsuite-summary": flag

101 }

102

103 directiveName: str = "unittest-summary"

104 configPrefix: str = "unittest"

105 configValues: Dict[str, Tuple[Any, str, Any]] = {

106 f"{configPrefix}_testsuites": ({}, "env", Dict)

107 } #: A dictionary of all configuration values used by unittest directives.

108

109 _testSummaries: ClassVar[Dict[str, report_DictType]] = {}

110

111 _cssClasses: List[str]

112 _reportID: str

113 _noAssertions: bool

114 _hideTestsuiteSummary: bool

115 _testsuiteSummaryName: Nullable[str]

116 _showTestcases: ShowTestcases

117 _xmlReport: Path

118 _testsuite: TestsuiteSummary

119

120 def _CheckOptions(self) -> None:

121 """

122 Parse all directive options or use default values.

123 """

124 cssClasses = self._ParseStringOption("class", "", r"(\w+)?( +\w+)*")

125 showTestcases = self._ParseStringOption("show-testcases", "all", r"all|not-passed")

126

127 self._cssClasses = [] if cssClasses == "" else cssClasses.split(" ")

128 self._reportID = self._ParseStringOption("reportid")

129 self._testsuiteSummaryName = self._ParseStringOption("testsuite-summary-name", "", r".+")

130 self._showTestcases = ShowTestcases[showTestcases.replace("-", "_")]

131 self._noAssertions = "no-assertions" in self.options

132 self._hideTestsuiteSummary = "hide-testsuite-summary" in self.options

133

134 try:

135 testSummary = self._testSummaries[self._reportID]

136 except KeyError as ex:

137 raise ReportExtensionError(f"No unit testing configuration item for '{self._reportID}'.") from ex

138 self._xmlReport = testSummary["xml_report"]

139

140 @classmethod

141 def CheckConfiguration(cls, sphinxApplication: Sphinx, sphinxConfiguration: Config) -> None:

142 """

143 Check configuration fields and load necessary values.

144

145 :param sphinxApplication: Sphinx application instance.

146 :param sphinxConfiguration: Sphinx configuration instance.

147 """

148 cls._CheckConfiguration(sphinxConfiguration)

149

150 @classmethod

151 def ReadReports(cls, sphinxApplication: Sphinx) -> None:

152 """

153 Read unittest report files.

154

155 :param sphinxApplication: Sphinx application instance.

156 """

157 print(f"[REPORT] Reading unittest reports ...")

158

159 @classmethod

160 def _CheckConfiguration(cls, sphinxConfiguration: Config) -> None:

161 from sphinx_reports import ReportDomain

162

163 variableName = f"{ReportDomain.name}_{cls.configPrefix}_testsuites"

164

165 try:

166 allTestsuites: Dict[str, report_DictType] = sphinxConfiguration[f"{ReportDomain.name}_{cls.configPrefix}_testsuites"]

167 except (KeyError, AttributeError) as ex:

168 raise ReportExtensionError(f"Configuration option '{variableName}' is not configured.") from ex

169

170 # try:

171 # testsuiteConfiguration = allTestsuites[self._reportID]

172 # except KeyError as ex:

173 # raise ReportExtensionError(f"conf.py: {ReportDomain.name}_{self.configPrefix}_testsuites: No configuration found for '{self._reportID}'.") from ex

174

175 for reportID, testSummary in allTestsuites.items():

176 summaryName = f"conf.py: {variableName}:[{reportID}]"

177

178 try:

179 xmlReport = Path(testSummary["xml_report"])

180 except KeyError as ex:

181 raise ReportExtensionError(f"{summaryName}.xml_report: Configuration is missing.") from ex

182

183 if not xmlReport.exists():

184 raise ReportExtensionError(f"{summaryName}.xml_report: Unittest report file '{xmlReport}' doesn't exist.") from FileNotFoundError(xmlReport)

185

186 cls._testSummaries[reportID] = {

187 "xml_report": xmlReport

188 }

189

190 def _sortedValues(self, d: Mapping[str, Testsuite]) -> Generator[Testsuite, None, None]:

191 for key in sorted(d.keys()):

192 yield d[key]

193

194 def _convertTestcaseStatusToSymbol(self, status: TestcaseStatus) -> str:

195 if status is TestcaseStatus.Passed:

196 return "✅"

197 elif status is TestcaseStatus.Failed:

198 return "❌"

199 elif status is TestcaseStatus.Skipped:

200 return "⚠"

201 elif status is TestcaseStatus.Aborted:

202 return "🚫"

203 elif status is TestcaseStatus.Excluded:

204 return "➖"

205 elif status is TestcaseStatus.Errored:

206 return "❗"

207 elif status is TestcaseStatus.SetupError:

208 return "⛔"

209 elif status is TestcaseStatus.Unknown:

210 return "❓"

211 else:

212 return "❌"

213

214 def _convertTestsuiteStatusToSymbol(self, status: TestsuiteStatus) -> str:

215 if status is TestsuiteStatus.Passed:

216 return "✅"

217 elif status is TestsuiteStatus.Failed:

218 return "❌"

219 elif status is TestsuiteStatus.Skipped:

220 return "⚠"

221 elif status is TestsuiteStatus.Aborted:

222 return "🚫"

223 elif status is TestsuiteStatus.Excluded:

224 return "➖"

225 elif status is TestsuiteStatus.Errored:

226 return "❗"

227 elif status is TestsuiteStatus.SetupError:

228 return "⛔"

229 elif status is TestsuiteStatus.Unknown:

230 return "❓"

231 else:

232 return "❌"

233

234 def _formatTimedelta(self, delta: timedelta) -> str:

235 if delta is None:

236 return ""

237

238 # Compute by hand, because timedelta._to_microseconds is not officially documented

239 microseconds = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds

240 milliseconds = (microseconds + 500) // 1000

241 seconds = milliseconds // 1000

242 minutes = seconds // 60

243 hours = minutes // 60

244 return f"{hours:02}:{minutes % 60:02}:{seconds % 60:02}.{milliseconds % 1000:03}"

245

246 def _GenerateTestSummaryTable(self) -> nodes.table:

247 # Create a table and table header with 8 columns

248 columns = [

249 ("Testsuite / Testcase", None, 500),

250 ("Testcases", None, 100),

251 ("Skipped", None, 100),

252 ("Errored", None, 100),

253 ("Failed", None, 100),

254 ("Passed", None, 100),

255 ("Assertions", None, 100),

256 ("Runtime (HH:MM:SS.sss)", None, 100),

257 ]

258

259 # If assertions shouldn't be displayed, remove column from columns list

260 if self._noAssertions:

261 columns.pop(6)

262

263 cssClasses = ["report-unittest-table", f"report-unittest-{self._reportID}"]

264 cssClasses.extend(self._cssClasses)

265

266 table, tableGroup = self._CreateTableHeader(

267 identifier=self._reportID,

268 columns=columns,

269 classes=cssClasses

270 )

271 tableBody = nodes.tbody()

272 tableGroup += tableBody

273

274 self.renderRoot(tableBody, self._testsuite, self._hideTestsuiteSummary, self._testsuiteSummaryName)

275

276 return table

277

278 def renderRoot(self, tableBody: nodes.tbody, testsuiteSummary: TestsuiteSummary, includeRoot: bool = True, testsuiteSummaryName: Nullable[str] = None) -> None:

279 level = 0

280

281 if includeRoot:

282 level += 1

283 state = self._convertTestsuiteStatusToSymbol(testsuiteSummary._status)

284

285 tableRow = nodes.row("", classes=["report-testsuitesummary", f"testsuitesummary-{testsuiteSummary._status.name.lower()}"])

286 tableBody += tableRow

287

288 tableRow += nodes.entry("", nodes.paragraph(text=f"{state}{testsuiteSummary.Name if testsuiteSummaryName == '' else testsuiteSummaryName}"))

289 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.TestcaseCount}"))

290 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Skipped}"))

291 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Errored}"))

292 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Failed}"))

293 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Passed}"))

294 if not self._noAssertions:

295 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

296 tableRow += nodes.entry("", nodes.paragraph(text=f"{self._formatTimedelta(testsuiteSummary.TotalDuration)}"))

297

298 for ts in self._sortedValues(testsuiteSummary._testsuites):

299 self.renderTestsuite(tableBody, ts, level)

300

301 self.renderSummary(tableBody, testsuiteSummary)

302

303 def renderTestsuite(self, tableBody: nodes.tbody, testsuite: Testsuite, level: int) -> None:

304 state = self._convertTestsuiteStatusToSymbol(testsuite._status)

305

306 tableRow = nodes.row("", classes=["report-testsuite", f"testsuite-{testsuite._status.name.lower()}"])

307 tableBody += tableRow

308

309 tableRow += nodes.entry("", nodes.paragraph(text=f"{' ' * level}{state}{testsuite.Name}"))

310 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuite.TestcaseCount}"))

311 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuite.Skipped}"))

312 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuite.Errored}"))

313 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuite.Failed}"))

314 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuite.Passed}"))

315 if not self._noAssertions:

316 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

317 tableRow += nodes.entry("", nodes.paragraph(text=f"{self._formatTimedelta(testsuite.TotalDuration)}"))

318

319 for ts in self._sortedValues(testsuite._testsuites):

320 self.renderTestsuite(tableBody, ts, level + 1)

321

322 for testcase in self._sortedValues(testsuite._testcases):

323 if testcase._status == self._showTestcases:

324 self.renderTestcase(tableBody, testcase, level + 1)

325

326 def renderTestcase(self, tableBody: nodes.tbody, testcase: Testcase, level: int) -> None:

327 state = self._convertTestcaseStatusToSymbol(testcase._status)

328

329 tableRow = nodes.row("", classes=["report-testcase", f"testcase-{testcase._status.name.lower()}"])

330 tableBody += tableRow

331

332 tableRow += nodes.entry("", nodes.paragraph(text=f"{' ' * level}{state}{testcase.Name}"))

333 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Expected}")),

334 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Covered}")),

335 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

336 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

337 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

338 if not self._noAssertions:

339 tableRow += nodes.entry("", nodes.paragraph(text=f"{testcase.AssertionCount}"))

340 tableRow += nodes.entry("", nodes.paragraph(text=f"{self._formatTimedelta(testcase.TotalDuration)}"))

341

342 def renderSummary(self, tableBody: nodes.tbody, testsuiteSummary: TestsuiteSummary) -> None:

343 state = self._convertTestsuiteStatusToSymbol(testsuiteSummary._status)

344

345 tableRow = nodes.row("", classes=["report-summary", f"testsuitesummary-{testsuiteSummary._status.name.lower()}"])

346 tableBody += tableRow

347

348 tableRow += nodes.entry("", nodes.paragraph(text=f"{state} {testsuiteSummary.Status.name.upper()}"))

349 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.TestcaseCount}"))

350 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Skipped}"))

351 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Errored}"))

352 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Failed}"))

353 tableRow += nodes.entry("", nodes.paragraph(text=f"{testsuiteSummary.Passed}"))

354 if not self._noAssertions:

355 tableRow += nodes.entry("", nodes.paragraph(text=f"")) # {testsuite.Uncovered}")),

356 tableRow += nodes.entry("", nodes.paragraph(text=f"{self._formatTimedelta(testsuiteSummary.TotalDuration)}"))

357

358 def run(self) -> List[nodes.Node]:

359 container = nodes.container()

360

361 try:

362 self._CheckOptions()

363 except ReportExtensionError as ex:

364 message = f"Caught {ex.__class__.__name__} when checking options for directive '{self.directiveName}'."

365 return self._internalError(container, __name__, message, ex)

366

367 # Assemble a list of Python source files

368 try:

369 doc = Document(self._xmlReport, analyzeAndConvert=True)

370 except Exception as ex:

371 message = f"Caught {ex.__class__.__name__} when reading and parsing '{self._xmlReport}'."

372 return self._internalError(container, __name__, message, ex)

373

374 doc.Aggregate()

375

376 try:

377 self._testsuite = doc.ToTestsuiteSummary()

378 except Exception as ex:

379 message = f"Caught {ex.__class__.__name__} when converting to a TestsuiteSummary for JUnit document '{self._xmlReport}'."

380 return self._internalError(container, __name__, message, ex)

381

382 self._testsuite.Aggregate()

383

384 try:

385 container += self._GenerateTestSummaryTable()

386 except Exception as ex:

387 message = f"Caught {ex.__class__.__name__} when generating the document structure for JUnit document '{self._xmlReport}'."

388 return self._internalError(container, __name__, message, ex)

389

390 return [container]