Coverage for sphinx_reports/Unittest.py: 20%

1# ==================================================================================================================== #

2# _ _ _ #

3# ___ _ __ | |__ (_)_ __ __ __ _ __ ___ _ __ ___ _ __| |_ ___ #

4# / __| '_ \| '_ \| | '_ \\ \/ /____| '__/ _ \ '_ \ / _ \| '__| __/ __| #

5# \__ \ |_) | | | | | | | |> <_____| | | __/ |_) | (_) | | | |_\__ \ #

6# |___/ .__/|_| |_|_|_| |_/_/\_\ |_| \___| .__/ \___/|_| \__|___/ #

7# |_| |_| #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32**Report unit test results as Sphinx documentation page(s).**

33"""

34from datetime import timedelta

35from enum import Flag

36from pathlib import Path

37from typing import Dict, Tuple, Any, List, Mapping, Generator, TypedDict, ClassVar, Optional as Nullable

39from docutils import nodes

40from docutils.parsers.rst.directives import flag

41from pyTooling.Decorators import export

42from pyEDAA.Reports.Unittesting import TestcaseStatus, TestsuiteStatus

43from pyEDAA.Reports.Unittesting.JUnit import Testsuite, TestsuiteSummary, Testcase, Document

44from sphinx.application import Sphinx

45from sphinx.config import Config

47from sphinx_reports.Common import ReportExtensionError

48from sphinx_reports.Node import Landscape

49from sphinx_reports.Sphinx import strip, stripAndNormalize, BaseDirective

52class report_DictType(TypedDict):

53 xml_report: Path

56@export

57class ShowTestcases(Flag):

58 passed = 1

59 failed = 2

60 skipped = 4

61 excluded = 8

62 errors = 16

63 aborted = 32

66 not_passed = all & ~passed

68 def __eq__(self, other: Any) -> bool:

69 if isinstance(other, TestcaseStatus):

70 if other is TestcaseStatus.Passed:

71 return ShowTestcases.passed in self

72 elif other is TestcaseStatus.Failed:

73 return ShowTestcases.failed in self

74 elif other is TestcaseStatus.Skipped:

75 return ShowTestcases.skipped in self

76 elif other is TestcaseStatus.Excluded:

77 return ShowTestcases.excluded in self

78 elif other is TestcaseStatus.Errored or other is TestcaseStatus.SetupError:

79 return ShowTestcases.errors in self

80 elif other is TestcaseStatus.Aborted:

81 return ShowTestcases.aborted in self

83 return False

86@export

87class UnittestSummary(BaseDirective):

88 """

89 This directive will be replaced by a table representing unit test results.

90 """

91 has_content = False

92 required_arguments = 0

93 optional_arguments = 6

95 option_spec = {

96 "class": strip,

97 "reportid": stripAndNormalize,

98 "testsuite-summary-name": strip,

99 "show-testcases": stripAndNormalize,

100 "no-assertions": flag,

101 "hide-testsuite-summary": flag

102 }

103

104 directiveName: str = "unittest-summary"

105 configPrefix: str = "unittest"

106 configValues: Dict[str, Tuple[Any, str, Any]] = {

107 f"{configPrefix}_testsuites": ({}, "env", Dict)

108 } #: A dictionary of all configuration values used by unittest directives.

109

110 _testSummaries: ClassVar[Dict[str, report_DictType]] = {}

111

112 _cssClasses: List[str]

113 _reportID: str

114 _noAssertions: bool

115 _hideTestsuiteSummary: bool

116 _testsuiteSummaryName: Nullable[str]

117 _showTestcases: ShowTestcases

118 _xmlReport: Path

119 _testsuite: TestsuiteSummary

120

121 def _CheckOptions(self) -> None:

122 """

123 Parse all directive options or use default values.

124 """

125 cssClasses = self._ParseStringOption("class", "", r"(\w+)?( +\w+)*")

126 showTestcases = self._ParseStringOption("show-testcases", "all", r"all|not-passed")

127

128 self._cssClasses = [] if cssClasses == "" else cssClasses.split(" ")

129 self._reportID = self._ParseStringOption("reportid")

130 self._testsuiteSummaryName = self._ParseStringOption("testsuite-summary-name", "", r".+")

131 self._showTestcases = ShowTestcases[showTestcases.replace("-", "_")]

132 self._noAssertions = "no-assertions" in self.options

133 self._hideTestsuiteSummary = "hide-testsuite-summary" in self.options

134

135 try:

136 testSummary = self._testSummaries[self._reportID]

137 except KeyError as ex:

138 raise ReportExtensionError(f"No unit testing configuration item for '{self._reportID}'.") from ex

139 self._xmlReport = testSummary["xml_report"]

140

141 @classmethod

142 def CheckConfiguration(cls, sphinxApplication: Sphinx, sphinxConfiguration: Config) -> None:

143 """

144 Check configuration fields and load necessary values.

145

146 :param sphinxApplication: Sphinx application instance.

147 :param sphinxConfiguration: Sphinx configuration instance.

148 """

149 cls._CheckConfiguration(sphinxConfiguration)

150

151 @classmethod

152 def ReadReports(cls, sphinxApplication: Sphinx) -> None:

153 """

154 Read unittest report files.

155

156 :param sphinxApplication: Sphinx application instance.

157 """

158 print(f"[REPORT] Reading unittest reports ...")

159

160 @classmethod

161 def _CheckConfiguration(cls, sphinxConfiguration: Config) -> None:

162 from sphinx_reports import ReportDomain

163

164 variableName = f"{ReportDomain.name}_{cls.configPrefix}_testsuites"

165

166 try:

167 allTestsuites: Dict[str, report_DictType] = sphinxConfiguration[f"{ReportDomain.name}_{cls.configPrefix}_testsuites"]

168 except (KeyError, AttributeError) as ex:

169 raise ReportExtensionError(f"Configuration option '{variableName}' is not configured.") from ex

170

171 # try:

172 # testsuiteConfiguration = allTestsuites[self._reportID]

173 # except KeyError as ex:

174 # raise ReportExtensionError(f"conf.py: {ReportDomain.name}_{self.configPrefix}_testsuites: No configuration found for '{self._reportID}'.") from ex

175

176 for reportID, testSummary in allTestsuites.items():

177 summaryName = f"conf.py: {variableName}:[{reportID}]"

178

179 try:

180 xmlReport = Path(testSummary["xml_report"])

181 except KeyError as ex:

182 raise ReportExtensionError(f"{summaryName}.xml_report: Configuration is missing.") from ex

183

184 if not xmlReport.exists():

185 raise ReportExtensionError(f"{summaryName}.xml_report: Unittest report file '{xmlReport}' doesn't exist.") from FileNotFoundError(xmlReport)

186

187 cls._testSummaries[reportID] = {

188 "xml_report": xmlReport

189 }

190

191 def _sortedValues(self, d: Mapping[str, Testsuite]) -> Generator[Testsuite, None, None]:

192 for key in sorted(d.keys()):

193 yield d[key]

194

195 def _convertTestcaseStatusToSymbol(self, status: TestcaseStatus) -> str:

196 if status is TestcaseStatus.Passed:

197 return "✅"

198 elif status is TestcaseStatus.Failed:

199 return "❌"

200 elif status is TestcaseStatus.Skipped:

201 return "⚠️"

202 elif status is TestcaseStatus.Aborted:

203 return "🚫"

204 elif status is TestcaseStatus.Excluded:

205 return "➖"

206 elif status is TestcaseStatus.Errored:

207 return "❗"

208 elif status is TestcaseStatus.SetupError:

209 return "⛔"

210 elif status is TestcaseStatus.Unknown:

211 return "❓"

212 else:

213 return "❌"

214

215 def _convertTestsuiteStatusToSymbol(self, status: TestsuiteStatus) -> str:

216 if status is TestsuiteStatus.Passed:

217 return "✅"

218 elif status is TestsuiteStatus.Failed:

219 return "❌"

220 elif status is TestsuiteStatus.Skipped:

221 return "⚠️"

222 elif status is TestsuiteStatus.Aborted:

223 return "🚫"

224 elif status is TestsuiteStatus.Excluded:

225 return "➖"

226 elif status is TestsuiteStatus.Errored:

227 return "❗"

228 elif status is TestsuiteStatus.SetupError:

229 return "⛔"

230 elif status is TestsuiteStatus.Unknown:

231 return "❓"

232 else:

233 return "❌"

234

235 def _formatTimedelta(self, delta: timedelta) -> str:

236 if delta is None:

237 return ""

238

239 # Compute by hand, because timedelta._to_microseconds is not officially documented

240 microseconds = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds

241 milliseconds = (microseconds + 500) // 1000

242 seconds = milliseconds // 1000

243 minutes = seconds // 60

244 hours = minutes // 60

245 return f"{hours:02}:{minutes % 60:02}:{seconds % 60:02}.{milliseconds % 1000:03}"

246

247 def _GenerateTestSummaryTable(self) -> nodes.table:

248 # Create a table and table header with 8 columns

249 columns = [

250 ("Testsuite / Testcase", 6),

251 ("Testcases", 1),

252 ("Skipped", 1),

253 ("Errored", 1),

254 ("Failed", 1),

255 ("Passed", 1),

256 ("Assertions", 1),

257 ("Runtime (HH:MM:SS.sss)", 2),

258 ]

259

260 # If assertions shouldn't be displayed, remove column from columns list

261 if self._noAssertions:

262 columns.pop(6)

263

264 cssClasses = ["report-unittest-table", f"report-unittest-{self._reportID}"]

265 cssClasses.extend(self._cssClasses)

266

267 tableGroup = self._CreateSingleTableHeader(

268 identifier=self._reportID,

269 columns=columns,

270 classes=cssClasses

271 )

272 tableBody = nodes.tbody()

273 tableGroup += tableBody

274

275 self.renderRoot(tableBody, self._testsuite, not self._hideTestsuiteSummary, self._testsuiteSummaryName)

276

277 return tableGroup.parent

278

279 def renderRoot(self, tableBody: nodes.tbody, testsuiteSummary: TestsuiteSummary, includeRoot: bool = True, testsuiteSummaryName: Nullable[str] = None) -> None:

280 level = 0

281

282 if includeRoot:

283 level += 1

284 state = self._convertTestsuiteStatusToSymbol(testsuiteSummary._status)

285

286 tableRow = nodes.row("", classes=["report-testsuitesummary", f"testsuitesummary-{testsuiteSummary._status.name.lower()}"])

287 tableBody += tableRow

288

289 tableRow += nodes.entry("", nodes.Text(f"{state}{testsuiteSummary.Name if testsuiteSummaryName == '' else testsuiteSummaryName}"))

290 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.TestcaseCount}"))

291 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Skipped}"))

292 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Errored}"))

293 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Failed}"))

294 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Passed}"))

295 if not self._noAssertions:

296 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

297 tableRow += nodes.entry("", nodes.Text(f"{self._formatTimedelta(testsuiteSummary.TotalDuration)}"))

298

299 for ts in self._sortedValues(testsuiteSummary._testsuites):

300 self.renderTestsuite(tableBody, ts, level)

301

302 self.renderSummary(tableBody, testsuiteSummary)

303

304 def renderTestsuite(self, tableBody: nodes.tbody, testsuite: Testsuite, level: int) -> None:

305 state = self._convertTestsuiteStatusToSymbol(testsuite._status)

306

307 tableRow = nodes.row("", classes=["report-testsuite", f"testsuite-{testsuite._status.name.lower()}"])

308 tableBody += tableRow

309

310 tableRow += nodes.entry("", nodes.Text(f"{' ' * level}{state}{testsuite.Name}"))

311 tableRow += nodes.entry("", nodes.Text(f"{testsuite.TestcaseCount}"))

312 tableRow += nodes.entry("", nodes.Text(f"{testsuite.Skipped}"))

313 tableRow += nodes.entry("", nodes.Text(f"{testsuite.Errored}"))

314 tableRow += nodes.entry("", nodes.Text(f"{testsuite.Failed}"))

315 tableRow += nodes.entry("", nodes.Text(f"{testsuite.Passed}"))

316 if not self._noAssertions:

317 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

318 tableRow += nodes.entry("", nodes.Text(f"{self._formatTimedelta(testsuite.TotalDuration)}"))

319

320 for ts in self._sortedValues(testsuite._testsuites):

321 self.renderTestsuite(tableBody, ts, level + 1)

322

323 for testcase in self._sortedValues(testsuite._testcases):

324 if testcase._status == self._showTestcases:

325 self.renderTestcase(tableBody, testcase, level + 1)

326

327 def renderTestcase(self, tableBody: nodes.tbody, testcase: Testcase, level: int) -> None:

328 state = self._convertTestcaseStatusToSymbol(testcase._status)

329

330 tableRow = nodes.row("", classes=["report-testcase", f"testcase-{testcase._status.name.lower()}"])

331 tableBody += tableRow

332

333 tableRow += nodes.entry("", nodes.Text(f"{' ' * level}{state}{testcase.Name}"))

334 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Expected}")),

335 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Covered}")),

336 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

337 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

338 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

339 if not self._noAssertions:

340 tableRow += nodes.entry("", nodes.Text(f"{testcase.AssertionCount}"))

341 tableRow += nodes.entry("", nodes.Text(f"{self._formatTimedelta(testcase.TotalDuration)}"))

342

343 def renderSummary(self, tableBody: nodes.tbody, testsuiteSummary: TestsuiteSummary) -> None:

344 state = self._convertTestsuiteStatusToSymbol(testsuiteSummary._status)

345

346 tableRow = nodes.row("", classes=["report-summary", f"testsuitesummary-{testsuiteSummary._status.name.lower()}"])

347 tableBody += tableRow

348

349 tableRow += nodes.entry("", nodes.Text(f"{state} {testsuiteSummary.Status.name.upper()}"))

350 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.TestcaseCount}"))

351 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Skipped}"))

352 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Errored}"))

353 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Failed}"))

354 tableRow += nodes.entry("", nodes.Text(f"{testsuiteSummary.Passed}"))

355 if not self._noAssertions:

356 tableRow += nodes.entry("", nodes.Text(f"")) # {testsuite.Uncovered}")),

357 tableRow += nodes.entry("", nodes.Text(f"{self._formatTimedelta(testsuiteSummary.TotalDuration)}"))

358

359 def run(self) -> List[nodes.Node]:

360 container = Landscape()

361

362 try:

363 self._CheckOptions()

364 except ReportExtensionError as ex:

365 message = f"Caught {ex.__class__.__name__} when checking options for directive '{self.directiveName}'."

366 return self._internalError(container, __name__, message, ex)

367

368 # Assemble a list of Python source files

369 try:

370 doc = Document(self._xmlReport, analyzeAndConvert=True)

371 except Exception as ex:

372 message = f"Caught {ex.__class__.__name__} when reading and parsing '{self._xmlReport}'."

373 return self._internalError(container, __name__, message, ex)

374

375 doc.Aggregate()

376

377 try:

378 self._testsuite = doc.ToTestsuiteSummary()

379 except Exception as ex:

380 message = f"Caught {ex.__class__.__name__} when converting to a TestsuiteSummary for JUnit document '{self._xmlReport}'."

381 return self._internalError(container, __name__, message, ex)

382

383 self._testsuite.Aggregate()

384

385 try:

386 container += self._GenerateTestSummaryTable()

387 except Exception as ex:

388 message = f"Caught {ex.__class__.__name__} when generating the document structure for JUnit document '{self._xmlReport}'."

389 return self._internalError(container, __name__, message, ex)

390

391 return [container]

Coverage for sphinx_reports / Unittest.py: 20%

243 statements