Coverage for pyTooling/Testing/__init_

1# ==================================================================================================================== #

2# _____ _ _ _____ _ _ #

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _|_ _|__ ___| |_(_)_ __ __ _ #

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | |/ _ \/ __| __| | '_ \ / _` | #

5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| | __/\__ \ |_| | | | | (_| | #

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)_|\___||___/\__|_|_| |_|\__, | #

7# |_| |___/ |___/ |___/ #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32Enhanced classes for writing unit tests with Python's :mod:`unittest` framework, which pytest runs as well.

34The pieces here are the ones every test suite otherwise rewrites. Currently that is application testing: starting

35the program under test the way a user does, because importing it cannot cover the console-script wiring, the

36argument parsing or the exit codes.

38.. hint::

40 See :ref:`high-level help <TESTING>` for explanations and usage examples.

41"""

42from inspect import cleandoc

43from pathlib import Path

44from re import compile as re_compile

45from shutil import which

46from os import environ

47from subprocess import CompletedProcess, run as subprocess_run

48from unittest import TestCase

49from sys import executable as PythonExecutable, version_info

50from typing import Any, Callable, ClassVar, Mapping, Union, Optional as Nullable

52from pyTooling.Common import getFullyQualifiedName, getTypeName

53from pyTooling.Decorators import export

54from pyTooling.Documentation import splitDocString

55from pyTooling.Exceptions import ToolingException

56from pyTooling.MetaClasses import C, M

59_ANSI_COLOR_CODES = re_compile(r"\x1B\[[0-9;]*m") #: Pattern matching an ANSI escape sequence selecting a color.

62@export

63class TestingError(ToolingException):

64 """Base-exception of all exceptions raised by :mod:`pyTooling.Testing`."""

66 __test__ = False #: Not a test class, although its name starts with ``Test``: pytest doesn't collect it.

69@export

70class ApplicationTestingError(TestingError):

71 """

72 The exception is raised when a testcase exercising an application through its command line is not set up.

74 It reports what the test class did not declare - a console script, or the runnable module a testcase asked to

75 run - or that the console script it named is not installed, because every testcase in that class would otherwise

76 fail with a less obvious error.

77 """

80@export

81def stripANSIColorCodes(text: str) -> str:

82 """

83 Remove ANSI color codes from a text, so it can be compared to an expected output.

85 A program writing to a terminal colors its output; the same program in a pipe usually does not, but that depends

86 on the program. Comparing against an expectation is more robust with the codes removed than with a rule about

87 when they appear.

89 :param text: The text to remove the color codes from.

90 :returns: The text without ANSI color codes.

91 """

92 return _ANSI_COLOR_CODES.sub("", text)

95@export

96def testsuite(title: Nullable[Union[str, C]] = None) -> Union[C, Callable[[C], C]]:

97 """

98 Mark a class as a test suite, so it is collected however it is named.

100 Without a marker, a test runner decides what a test suite is from the class' *name*: pytest's default

101 ``python_classes`` matches ``Test*``, and :mod:`unittest` collects every :class:`~unittest.TestCase`. The name

102 therefore carries two jobs at once - it identifies the class *and* it enables collection - and the reader of a

103 test report sees the identifier rather than a description.

104

105 This decorator separates them. The class is collected because it is marked, and it is reported under the title

106 given here, which can be a sentence.

107

108 It is usable with and without parentheses, and with or without a title.

109

110 .. admonition:: ``example.py``

111

112 .. code-block:: python

113

114 from pyTooling.Testing import Testcase, testsuite, testcase

115

116 @testsuite("Version comparison")

117 class VersionComparison(Testcase):

118 @testcase("A newer version compares greater.")

119 def NewerIsGreater(self) -> None:

120 self.assertGreater(Version("2.0"), Version("1.9"))

121

122 :param title: Optional, title the test suite is reported under, or the class itself when the decorator is

123 used without parentheses. Default: the class' name.

124 :returns: Decorator marking the class with ``__testsuite_title__``, ``__testsuite_summary__`` and

125 ``__testsuite_description__``, or the marked class itself when used without parentheses.

126 :raises TypeError: If parameter 'title' is neither a string nor a class.

127

128 .. seealso::

129

130 :deco:`~pyTooling.Testing.testcase`

131 |rarr| Mark a *method* as a testcase.

132 :ref:`TESTING/Markers`

133 |rarr| How a test runner is taught to collect what is marked.

134 """

135 def decorator(cls: C) -> C:

136 """

137 Attach the test suite's title to the decorated class.

138

139 :param cls: Class that is marked as a test suite.

140 :returns: Same class, but with an additional ``<class>.__testsuite_title__`` field.

141 :raises TypeError: If applied to anything but a class.

142 """

143 if not isinstance(cls, type):

144 ex = TypeError(f"Decorator 'testsuite' is applied to '{getFullyQualifiedName(cls)}' instead of a class.")

145 ex.add_note("A method is marked as a testcase with the 'testcase' decorator.")

146 raise ex

147

148 cls.__testsuite_title__ = cls.__name__ if title is None or isinstance(title, type) else title

149 cls.__testsuite_summary__, _ = splitDocString(cls.__doc__)

150 cls.__testsuite_description__ = "" if cls.__doc__ is None else cleandoc(cls.__doc__)

151 return cls

152

153 if isinstance(title, type): # used without parentheses: the class itself was passed

154 return decorator(title)

155

156 if title is not None and not isinstance(title, str):

157 ex = TypeError("Parameter 'title' is neither a string nor a class.")

158 ex.add_note(f"Got type '{getFullyQualifiedName(title)}'.")

159 raise ex

160

161 return decorator

162

163

164testsuite.__test__ = False #: The marker is not a testcase itself, whatever a test runner makes of its name.

165

166

167@export

168def testcase(title: Nullable[Union[str, M]] = None) -> Union[M, Callable[[M], M]]:

169 """

170 Mark a method as a testcase, so it is collected however it is named.

171

172 Without a marker, the method's name enables its collection - pytest's default ``python_functions`` matches

173 ``test_*`` and :mod:`unittest`'s loader matches the ``test`` prefix - so ``test_`` ends up in the test report,

174 and what the testcase actually checks has to be squeezed into an identifier.

175

176 This decorator separates the two. The method is collected because it is marked, and it is reported under the

177 title given here. Like :deco:`testsuite`, it is usable with and without parentheses.

178

179 .. admonition:: ``example.py``

180

181 .. code-block:: python

182

183 @testcase("an empty list has no first element")

184 def EmptyListHasNoFirstElement(self) -> None:

185 with self.assertRaises(EmptyListError):

186 _ = LinkedList().FirstElement

187

188 :param title: Optional, title the testcase is reported under, or the method itself when the decorator is

189 used without parentheses. Default: the method's name.

190 :returns: Decorator marking the method with ``__testcase_title__``, ``__testcase_summary__`` and

191 ``__testcase_description__``, or the marked method itself when used without parentheses.

192 :raises TypeError: If parameter 'title' is neither a string nor a method.

193

194 .. seealso::

195

196 :deco:`~pyTooling.Testing.testsuite`

197 |rarr| Mark a *class* as a test suite.

198 :ref:`TESTING/Markers`

199 |rarr| How a test runner is taught to collect what is marked.

200 """

201 def decorator(method: M) -> M:

202 """

203 Attach the testcase's title to the decorated method.

204

205 :param method: Method that is marked as a testcase.

206 :returns: Same method, but with an additional ``<method>.__testcase_title__`` field.

207 :raises TypeError: If applied to a class instead of a method.

208 """

209 if isinstance(method, type):

210 ex = TypeError(f"Decorator 'testcase' is applied to class '{method.__name__}' instead of a method.")

211 ex.add_note("A class is marked as a test suite with the 'testsuite' decorator.")

212 raise ex

213

214 method.__testcase_title__ = method.__name__ if title is None or callable(title) else title

215 method.__testcase_summary__, _ = splitDocString(method.__doc__)

216 method.__testcase_description__ = "" if method.__doc__ is None else cleandoc(method.__doc__)

217 return method

218

219 if title is None or isinstance(title, str):

220 return decorator

221

222 if not callable(title): # used without parentheses: the method itself was passed

223 ex = TypeError("Parameter 'title' is neither a string nor a method.")

224 ex.add_note(f"Got type '{getFullyQualifiedName(title)}'.")

225 raise ex

226

227 return decorator(title)

228

229

230testcase.__test__ = False #: The marker is not a testcase itself, whatever a test runner makes of its name.

231

232

233@export

234class Testcase(TestCase):

235 """

236 The base class for pyTooling's testcases, deriving from :class:`unittest.TestCase`.

237

238 It adds the assertions Python's :mod:`unittest` gained later than the oldest Python version pyTooling supports,

239 so a test suite can use them whichever interpreter runs it:

240

241 .. code-block:: python

242

243 class Slots(Testcase):

244 def test_SlotsAreDerived(self) -> None:

245 self.assertHasAttr(MyClass, "__slots__")

246

247 On Python 3.14 and newer, :class:`unittest.TestCase` implements them and this class defines nothing, so the

248 standard library's implementations and messages are used.

249 """

250

251 if version_info < (3, 14): # pragma: no cover

252 def assertHasAttr(self, obj: Any, name: str, msg: Nullable[str] = None) -> None:

253 """

254 Assert an object has an attribute of the given name.

255

256 Available in :class:`unittest.TestCase` from Python 3.14 on.

257

258 :param obj: The object to check.

259 :param name: Name of the attribute the object is expected to have.

260 :param msg: Optional, message replacing the generated one.

261 """

262 if not hasattr(obj, name):

263 self.fail(msg or f"{type(obj).__name__!r} object has no attribute {name!r}")

264

265 def assertNotHasAttr(self, obj: Any, name: str, msg: Nullable[str] = None) -> None:

266 """

267 Assert an object has no attribute of the given name.

268

269 Available in :class:`unittest.TestCase` from Python 3.14 on.

270

271 :param obj: The object to check.

272 :param name: Name of the attribute the object is expected not to have.

273 :param msg: Optional, message replacing the generated one.

274 """

275 if hasattr(obj, name):

276 self.fail(msg or f"{type(obj).__name__!r} object has unexpected attribute {name!r}")

277

278

279@export

280class ApplicationTestcase(Testcase):

281 """

282 The base class for testcases exercising an application through its command line.

283

284 It resolves the installed console script once per test class, offers two ways to run the program - through the

285 installed entry point and, if the program has a ``__main__`` module, through ``python -m <module>`` - and an

286 assertion that reports the exit code together with what the program printed.

287

288 Derive from it and name what is being tested:

289

290 .. code-block:: python

291

292 class Commands(ApplicationTestcase):

293 _CONSOLE_SCRIPT = "myprogram"

294 _RUNNABLE_MODULE = "myPackage.CLI"

295

296 def test_Version(self) -> None:

297 result = self.RunEntrypoint("--version")

298

299 self.assertExitCode(result, 0)

300 self.assertIn("myprogram", result.stdout)

301 """

302

303 _CONSOLE_SCRIPT: ClassVar[Nullable[str]] = None #: Name of the installed console script, resolved on ``PATH``.

304 _RUNNABLE_MODULE: ClassVar[Nullable[str]] = None #: Optional, dotted name of the module to run with ``python -m``.

305 _executable: ClassVar[Nullable[str]] = None #: The resolved console script, set by :meth:`setUpClass`.

306

307 @classmethod

308 def setUpClass(cls) -> None:

309 """

310 Check the test class is set up, and resolve the console script on ``PATH``, once per class.

311

312 :raises ApplicationTestingError: If the test class named no console script, or if the console script it named is

313 not installed. Every testcase in the class would otherwise fail, each with a

314 less obvious error.

315 """

316 super().setUpClass()

317

318 if cls._CONSOLE_SCRIPT is None:

319 raise ApplicationTestingError(f"Testcase '{cls.__name__}' has no console script. Set '_CONSOLE_SCRIPT'.")

320

321 if (resolved := which(cls._CONSOLE_SCRIPT)) is None:

322 ex = ApplicationTestingError(f"Console script '{cls._CONSOLE_SCRIPT}' was not found in PATH.")

323 raise ex from FileNotFoundError(str(cls._CONSOLE_SCRIPT))

324

325 cls._executable = resolved

326

327 @staticmethod

328 def _MergeEnvironment(environment: Nullable[Mapping[str, Nullable[str]]]) -> Nullable[dict[str, str]]:

329 """

330 Return a copy of this process's environment with the given variables set or removed.

331

332 :param environment: Variables to set, or - with value ``None`` - to remove; ``None`` inherits the environment.

333 :returns: The merged environment, or ``None`` to inherit this process's environment unchanged.

334 """

335 if environment is None:

336 return None

337

338 merged = dict(environ)

339 for name, value in environment.items():

340 if value is None:

341 merged.pop(name, None)

342 else:

343 merged[name] = value

344

345 return merged

346

347 def RunEntrypoint(

348 self,

349 *arguments: str,

350 timeout: float = 60.0,

351 stdInput: Nullable[str] = None,

352 environment: Nullable[Mapping[str, Nullable[str]]] = None,

353 workingDirectory: Nullable[Path] = None

354 ) -> CompletedProcess:

355 """

356 Run the installed console script.

357

358 This is the path a user takes, so it covers the entry-point wiring as well as the program itself.

359

360 :param arguments: Command line arguments to pass to the program.

361 :param timeout: Optional, seconds to wait before the program is killed and :exc:`subprocess.TimeoutExpired`

362 is raised. A test should fail rather than hang. Default: 60 seconds.

363 :param stdInput: Optional, text to send to the program's standard input.

364 :param environment: Optional, variables to set - or, with value ``None``, to remove - in this process's

365 environment, which the program inherits.

366 :param workingDirectory: Optional, directory to run in, or ``None`` for the current one.

367 :returns: The completed process, with ``stdout`` and ``stderr`` captured as text.

368 """

369 return subprocess_run(

370 [self._executable, *arguments],

371 capture_output=True,

372 text=True,

373 timeout=timeout,

374 input=stdInput,

375 env=self._MergeEnvironment(environment),

376 cwd=None if workingDirectory is None else str(workingDirectory)

377 )

378

379 def RunModule(

380 self,

381 *arguments: str,

382 timeout: float = 60.0,

383 stdInput: Nullable[str] = None,

384 environment: Nullable[Mapping[str, Nullable[str]]] = None,

385 workingDirectory: Nullable[Path] = None

386 ) -> CompletedProcess:

387 """

388 Run the program as ``python -m <module>``, bypassing the console script.

389

390 Use it to tell a broken entry point apart from a broken program: if this passes while

391 :meth:`RunEntrypoint` fails, the packaging is at fault, not the code.

392

393 :param arguments: Command line arguments to pass to the program.

394 :param timeout: Optional, seconds to wait before the program is killed and

395 :exc:`subprocess.TimeoutExpired` is raised. Default: 60 seconds.

396 :param stdInput: Optional, text to send to the program's standard input.

397 :param environment: Optional, variables to set - or, with value ``None``, to remove - in this

398 process's environment, which the program inherits.

399 :param workingDirectory: Optional, directory to run in, or ``None`` for the current one.

400 :returns: The completed process, with ``stdout`` and ``stderr`` captured as text.

401 :raises ApplicationTestingError: If the test class named no runnable module.

402 """

403 if self._RUNNABLE_MODULE is None:

404 ex = ApplicationTestingError(f"Testcase '{getTypeName(self)}' has no runnable module.")

405 ex.add_note("Set '_RUNNABLE_MODULE' to run the program with 'python -m'.")

406 raise ex

407

408 return subprocess_run(

409 [PythonExecutable, "-m", self._RUNNABLE_MODULE, *arguments],

410 capture_output=True,

411 text=True,

412 timeout=timeout,

413 input=stdInput,

414 env=self._MergeEnvironment(environment),

415 cwd=None if workingDirectory is None else str(workingDirectory)

416 )

417

418 def assertExitCode(self, result: CompletedProcess, expected: int = 0) -> None:

419 """

420 Check the exit code of a completed process, reporting what the program printed when it doesn't match.

421

422 The output is what explains the failure, and it is gone once the test has finished, so it goes into the

423 assertion message rather than into the console.

424

425 :param result: The completed process to check.

426 :param expected: Optional, the expected exit code, zero by default.

427 """

428 self.assertEqual(

429 expected,

430 result.returncode,

431 msg=(

432 f"Expected exit code {expected}, got {result.returncode} from: {result.args!r}\n"

433 f"--- stdout ---\n{result.stdout}\n"

434 f"--- stderr ---\n{result.stderr}"

435 )

436 )

Coverage for pyTooling/Testing/init.py: 100%

97 statements