Coverage for pyTooling/Testing/__init_

1# ==================================================================================================================== #

2# _____ _ _ _____ _ _ #

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _|_ _|__ ___| |_(_)_ __ __ _ #

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | |/ _ \/ __| __| | '_ \ / _` | #

5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| | __/\__ \ |_| | | | | (_| | #

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)_|\___||___/\__|_|_| |_|\__, | #

7# |_| |___/ |___/ |___/ #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32Enhanced classes for writing unit tests with Python's :mod:`unittest` framework, which pytest runs as well.

34The pieces here are the ones every test suite otherwise rewrites. Currently that is application testing: starting

35the program under test the way a user does, because importing it cannot cover the console-script wiring, the

36argument parsing or the exit codes.

38.. hint::

40 See :ref:`high-level help <TESTING>` for explanations and usage examples.

41"""

42from inspect import cleandoc

43from pathlib import Path

44from re import compile as re_compile

45from shutil import which

46from subprocess import CompletedProcess, run as subprocess_run

47from unittest import TestCase

48from sys import executable as PythonExecutable, version_info

49from typing import Any, Callable, ClassVar, Union, Optional as Nullable

51from pyTooling.Common import getFullyQualifiedName

52from pyTooling.Decorators import export

53from pyTooling.Documentation import splitDocString

54from pyTooling.Exceptions import ToolingException

55from pyTooling.MetaClasses import C, M

58_ANSI_COLOR_CODES = re_compile(r"\x1B\[[0-9;]*m") #: Pattern matching an ANSI escape sequence selecting a color.

61@export

62class TestingError(ToolingException):

63 """Base-exception of all exceptions raised by :mod:`pyTooling.Testing`."""

66@export

67class ApplicationTestingError(TestingError):

68 """

69 The exception is raised when a testcase exercising an application through its command line is not set up.

71 It reports what the test class did not declare - a console script, or the runnable module a testcase asked to

72 run - or that the console script it named is not installed, because every testcase in that class would otherwise

73 fail with a less obvious error.

74 """

77@export

78def stripANSIColorCodes(text: str) -> str:

79 """

80 Remove ANSI color codes from a text, so it can be compared to an expected output.

82 A program writing to a terminal colors its output; the same program in a pipe usually does not, but that depends

83 on the program. Comparing against an expectation is more robust with the codes removed than with a rule about

84 when they appear.

86 :param text: The text to remove the color codes from.

87 :returns: The text without ANSI color codes.

88 """

89 return _ANSI_COLOR_CODES.sub("", text)

92@export

93def testsuite(title: Union[str, C, None] = None) -> Union[C, Callable[[C], C]]:

94 """

95 Mark a class as a test suite, so it is collected however it is named.

97 Without a marker, a test runner decides what a test suite is from the class' *name*: pytest's default

98 ``python_classes`` matches ``Test*``, and :mod:`unittest` collects every :class:`~unittest.TestCase`. The name

99 therefore carries two jobs at once - it identifies the class *and* it enables collection - and the reader of a

100 test report sees the identifier rather than a description.

101

102 This decorator separates them. The class is collected because it is marked, and it is reported under the title

103 given here, which can be a sentence.

104

105 It is usable with and without parentheses, and with or without a title.

106

107 .. admonition:: ``example.py``

108

109 .. code-block:: python

110

111 from pyTooling.Testing import Testcase, testsuite, testcase

112

113 @testsuite("Version comparison")

114 class VersionComparison(Testcase):

115 @testcase("A newer version compares greater.")

116 def NewerIsGreater(self) -> None:

117 self.assertGreater(Version("2.0"), Version("1.9"))

118

119 :param title: Optional, title the test suite is reported under, or the class itself when the decorator is

120 used without parentheses. Default: the class' name.

121 :returns: Decorator marking the class with ``__testsuite_title__``, ``__testsuite_summary__`` and

122 ``__testsuite_description__``, or the marked class itself when used without parentheses.

123 :raises TypeError: If parameter 'title' is neither a string nor a class.

124

125 .. seealso::

126

127 :deco:`~pyTooling.Testing.testcase`

128 |rarr| Mark a *method* as a testcase.

129 :ref:`TESTING/Markers`

130 |rarr| How a test runner is taught to collect what is marked.

131 """

132 def decorator(cls: C) -> C:

133 """

134 Attach the test suite's title to the decorated class.

135

136 :param cls: Class that is marked as a test suite.

137 :returns: Same class, but with an additional ``<class>.__testsuite_title__`` field.

138 :raises TypeError: If applied to anything but a class.

139 """

140 if not isinstance(cls, type):

141 ex = TypeError(f"Decorator 'testsuite' is applied to '{getFullyQualifiedName(cls)}' instead of a class.")

142 ex.add_note("A method is marked as a testcase with the 'testcase' decorator.")

143 raise ex

144

145 cls.__testsuite_title__ = cls.__name__ if title is None or isinstance(title, type) else title

146 cls.__testsuite_summary__, _ = splitDocString(cls.__doc__)

147 cls.__testsuite_description__ = "" if cls.__doc__ is None else cleandoc(cls.__doc__)

148 return cls

149

150 if isinstance(title, type): # used without parentheses: the class itself was passed

151 return decorator(title)

152

153 if title is not None and not isinstance(title, str):

154 ex = TypeError("Parameter 'title' is neither a string nor a class.")

155 ex.add_note(f"Got type '{getFullyQualifiedName(title)}'.")

156 raise ex

157

158 return decorator

159

160

161testsuite.__test__ = False #: The marker is not a testcase itself, whatever a test runner makes of its name.

162

163

164@export

165def testcase(title: Union[str, M, None] = None) -> Union[M, Callable[[M], M]]:

166 """

167 Mark a method as a testcase, so it is collected however it is named.

168

169 Without a marker, the method's name enables its collection - pytest's default ``python_functions`` matches

170 ``test_*`` and :mod:`unittest`'s loader matches the ``test`` prefix - so ``test_`` ends up in the test report,

171 and what the testcase actually checks has to be squeezed into an identifier.

172

173 This decorator separates the two. The method is collected because it is marked, and it is reported under the

174 title given here. Like :deco:`testsuite`, it is usable with and without parentheses.

175

176 .. admonition:: ``example.py``

177

178 .. code-block:: python

179

180 @testcase("an empty list has no first element")

181 def EmptyListHasNoFirstElement(self) -> None:

182 with self.assertRaises(EmptyListError):

183 _ = LinkedList().FirstElement

184

185 :param title: Optional, title the testcase is reported under, or the method itself when the decorator is

186 used without parentheses. Default: the method's name.

187 :returns: Decorator marking the method with ``__testcase_title__``, ``__testcase_summary__`` and

188 ``__testcase_description__``, or the marked method itself when used without parentheses.

189 :raises TypeError: If parameter 'title' is neither a string nor a method.

190

191 .. seealso::

192

193 :deco:`~pyTooling.Testing.testsuite`

194 |rarr| Mark a *class* as a test suite.

195 :ref:`TESTING/Markers`

196 |rarr| How a test runner is taught to collect what is marked.

197 """

198 def decorator(method: M) -> M:

199 """

200 Attach the testcase's title to the decorated method.

201

202 :param method: Method that is marked as a testcase.

203 :returns: Same method, but with an additional ``<method>.__testcase_title__`` field.

204 :raises TypeError: If applied to a class instead of a method.

205 """

206 if isinstance(method, type):

207 ex = TypeError(f"Decorator 'testcase' is applied to class '{method.__name__}' instead of a method.")

208 ex.add_note("A class is marked as a test suite with the 'testsuite' decorator.")

209 raise ex

210

211 method.__testcase_title__ = method.__name__ if title is None or callable(title) else title

212 method.__testcase_summary__, _ = splitDocString(method.__doc__)

213 method.__testcase_description__ = "" if method.__doc__ is None else cleandoc(method.__doc__)

214 return method

215

216 if title is None or isinstance(title, str):

217 return decorator

218

219 if not callable(title): # used without parentheses: the method itself was passed

220 ex = TypeError("Parameter 'title' is neither a string nor a method.")

221 ex.add_note(f"Got type '{getFullyQualifiedName(title)}'.")

222 raise ex

223

224 return decorator(title)

225

226

227testcase.__test__ = False #: The marker is not a testcase itself, whatever a test runner makes of its name.

228

229

230@export

231class Testcase(TestCase):

232 """

233 The base class for pyTooling's testcases, deriving from :class:`unittest.TestCase`.

234

235 It adds the assertions Python's :mod:`unittest` gained later than the oldest Python version pyTooling supports,

236 so a test suite can use them whichever interpreter runs it:

237

238 .. code-block:: python

239

240 class Slots(Testcase):

241 def test_SlotsAreDerived(self) -> None:

242 self.assertHasAttr(MyClass, "__slots__")

243

244 On Python 3.14 and newer, :class:`unittest.TestCase` implements them and this class defines nothing, so the

245 standard library's implementations and messages are used.

246 """

247

248 if version_info < (3, 14): # pragma: no cover

249 def assertHasAttr(self, obj: Any, name: str, msg: Nullable[str] = None) -> None:

250 """

251 Assert an object has an attribute of the given name.

252

253 Available in :class:`unittest.TestCase` from Python 3.14 on.

254

255 :param obj: The object to check.

256 :param name: Name of the attribute the object is expected to have.

257 :param msg: Optional, message replacing the generated one.

258 """

259 if not hasattr(obj, name):

260 self.fail(msg or f"{type(obj).__name__!r} object has no attribute {name!r}")

261

262 def assertNotHasAttr(self, obj: Any, name: str, msg: Nullable[str] = None) -> None:

263 """

264 Assert an object has no attribute of the given name.

265

266 Available in :class:`unittest.TestCase` from Python 3.14 on.

267

268 :param obj: The object to check.

269 :param name: Name of the attribute the object is expected not to have.

270 :param msg: Optional, message replacing the generated one.

271 """

272 if hasattr(obj, name):

273 self.fail(msg or f"{type(obj).__name__!r} object has unexpected attribute {name!r}")

274

275

276@export

277class ApplicationTestcase(Testcase):

278 """

279 The base class for testcases exercising an application through its command line.

280

281 It resolves the installed console script once per test class, offers two ways to run the program - through the

282 installed entry point and, if the program has a ``__main__`` module, through ``python -m <module>`` - and an

283 assertion that reports the exit code together with what the program printed.

284

285 Derive from it and name what is being tested:

286

287 .. code-block:: python

288

289 class Commands(ApplicationTestcase):

290 _consoleScript = "myprogram"

291 _runnableModule = "myPackage.CLI"

292

293 def test_Version(self) -> None:

294 result = self.RunEntrypoint("--version")

295

296 self.assertExitCode(result, 0)

297 self.assertIn("myprogram", result.stdout)

298 """

299

300 _consoleScript: ClassVar[Nullable[str]] = None #: Name of the installed console script, resolved on ``PATH``.

301 _runnableModule: ClassVar[Nullable[str]] = None #: Optional, dotted name of the module to run with ``python -m``.

302 _executable: ClassVar[Nullable[str]] = None #: The resolved console script, set by :meth:`setUpClass`.

303

304 @classmethod

305 def setUpClass(cls) -> None:

306 """

307 Check the test class is set up, and resolve the console script on ``PATH``, once per class.

308

309 :raises ApplicationTestingError: If the test class named no console script, or if the console script it named is

310 not installed. Every testcase in the class would otherwise fail, each with a

311 less obvious error.

312 """

313 super().setUpClass()

314

315 if cls._consoleScript is None:

316 raise ApplicationTestingError(f"Testcase '{cls.__name__}' has no console script. Set '_consoleScript'.")

317

318 if (resolved := which(cls._consoleScript)) is None:

319 ex = ApplicationTestingError(f"Console script '{cls._consoleScript}' was not found in PATH.")

320 raise ex from FileNotFoundError(str(cls._consoleScript))

321

322 cls._executable = resolved

323

324 def RunEntrypoint(

325 self,

326 *arguments: str,

327 timeout: float = 10.0,

328 stdInput: Nullable[str] = None,

329 environment: Nullable[dict[str, str]] = None,

330 workingDirectory: Nullable[Path] = None

331 ) -> CompletedProcess:

332 """

333 Run the installed console script.

334

335 This is the path a user takes, so it covers the entry-point wiring as well as the program itself.

336

337 :param arguments: Command line arguments to pass to the program.

338 :param timeout: Optional, seconds to wait before the program is killed and :exc:`subprocess.TimeoutExpired`

339 is raised. A test should fail rather than hang.

340 :param stdInput: Optional, text to send to the program's standard input.

341 :param environment: Optional, the environment to run in, or ``None`` to inherit this process's environment.

342 :param workingDirectory: Optional, directory to run in, or ``None`` for the current one.

343 :returns: The completed process, with ``stdout`` and ``stderr`` captured as text.

344 """

345 return subprocess_run(

346 [self._executable, *arguments],

347 capture_output=True,

348 text=True,

349 timeout=timeout,

350 input=stdInput,

351 env=environment,

352 cwd=None if workingDirectory is None else str(workingDirectory)

353 )

354

355 def RunModule(

356 self,

357 *arguments: str,

358 timeout: float = 10.0,

359 stdInput: Nullable[str] = None,

360 environment: Nullable[dict[str, str]] = None,

361 workingDirectory: Nullable[Path] = None

362 ) -> CompletedProcess:

363 """

364 Run the program as ``python -m <module>``, bypassing the console script.

365

366 Use it to tell a broken entry point apart from a broken program: if this passes while

367 :meth:`RunEntrypoint` fails, the packaging is at fault, not the code.

368

369 :param arguments: Command line arguments to pass to the program.

370 :param timeout: Optional, seconds to wait before the program is killed.

371 :param stdInput: Optional, text to send to the program's standard input.

372 :param environment: Optional, the environment to run in, or ``None`` to inherit this process's

373 environment.

374 :param workingDirectory: Optional, directory to run in, or ``None`` for the current one.

375 :returns: The completed process, with ``stdout`` and ``stderr`` captured as text.

376 :raises ApplicationTestingError: If the test class named no runnable module.

377 """

378 if self._runnableModule is None:

379 ex = ApplicationTestingError(f"Testcase '{self.__class__.__name__}' has no runnable module.")

380 ex.add_note("Set '_runnableModule' to run the program with 'python -m'.")

381 raise ex

382

383 return subprocess_run(

384 [PythonExecutable, "-m", self._runnableModule, *arguments],

385 capture_output=True,

386 text=True,

387 timeout=timeout,

388 input=stdInput,

389 env=environment,

390 cwd=None if workingDirectory is None else str(workingDirectory)

391 )

392

393 def assertExitCode(self, result: CompletedProcess, expected: int = 0) -> None:

394 """

395 Check the exit code of a completed process, reporting what the program printed when it doesn't match.

396

397 The output is what explains the failure, and it is gone once the test has finished, so it goes into the

398 assertion message rather than into the console.

399

400 :param result: The completed process to check.

401 :param expected: Optional, the expected exit code, zero by default.

402 """

403 self.assertEqual(

404 expected,

405 result.returncode,

406 msg=(

407 f"Expected exit code {expected}, got {result.returncode} from: {result.args!r}\n"

408 f"--- stdout ---\n{result.stdout}\n"

409 f"--- stderr ---\n{result.stderr}"

410 )

411 )

Coverage for pyTooling/Testing/init.py: 100%

85 statements