Coverage for pyTooling/Documentation/Sphinx/DependencyTable.py: 45%
478 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-03 23:02 +0000
1# ==================================================================================================================== #
2# _____ _ _ ____ _ _ _ #
3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ | _ \ ___ ___ _ _ _ __ ___ ___ _ __ | |_ __ _| |_(_) ___ _ __ #
4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | | | |/ _ \ / __| | | | '_ ` _ \ / _ \ '_ \| __/ _` | __| |/ _ \| '_ \ #
5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| |_| | (_) | (__| |_| | | | | | | __/ | | | || (_| | |_| | (_) | | | |#
6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/ \___/ \___|\__,_|_| |_| |_|\___|_| |_|\__\__,_|\__|_|\___/|_| |_|#
7# |_| |___/ |___/ #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2026-2026 Patrick Lehmann - Bötzingen, Germany #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""
32A Sphinx directive rendering a project's dependencies as a table, from the requirements rather than by hand.
34A dependency table states, per package, which version is required, what it is licensed under, and what it drags in.
35Written by hand it is wrong within a release or two: pyTooling's own documentation table was missing four of the
36packages :file:`doc/requirements.txt` requires, listed one that isn't required at all, and called Sphinx BSD-3-Clause
37when it is BSD-2-Clause.
39**The entrypoints are declared in** :file:`conf.py` **and named by the documents**, the way
40:mod:`sphinx_reports` declares its reports:
42.. code-block:: Python
44 # doc/conf.py
45 pyTooling_Dependency_PackageOverrides = "Dependency.PackageOverrides.yaml"
46 pyTooling_Dependency_Requirements = {
47 "package": {"file": "../requirements.txt"},
48 "documentation": {"file": "requirements.txt"},
49 "yaml": {"package": "pyTooling[yaml]"}
50 }
52.. code-block:: rest
54 .. dependency-table:: documentation
55 :caption: Documentation dependencies
57A requirements file is read - with its ``-r`` includes followed - while :file:`conf.py` is being processed, so a
58path that doesn't exist ends the build with one clear message instead of an error box in the middle of a page.
60**The data is fetched live** from the package index, once per build and shared between every table of that build:
61:file:`requirements.txt`, :file:`tests/requirements.txt` and :file:`doc/requirements.txt` overlap heavily, and a
62package they share is downloaded once. That still costs real time, so every table reports what it spent, measured
63with a :class:`~pyTooling.Stopwatch.Stopwatch`, and the build ends with the total.
65:raises MissingDependencyError: If the 'sphinx' extra isn't installed.
66"""
67from __future__ import annotations
69from enum import Enum, auto
70from pathlib import Path
71from typing import TYPE_CHECKING, Any, ClassVar, Iterable, Literal, Mapping, Optional as Nullable
72from typing import TypeVar, cast
74from pyTooling.Common import getFullyQualifiedName
75from pyTooling.Decorators import export, readonly
76from pyTooling.Dependency import UnknownLicenseWarning
77from pyTooling.Exceptions import ConfigurationError, MissingDependencyError
78from pyTooling.MetaClasses import ExtendedType
79from pyTooling.Stopwatch import Stopwatch
80from pyTooling.Warning import WarningCollector
82try:
83 from docutils import nodes
84 from docutils.parsers.rst import directives
85 from sphinx.application import Sphinx
86 from sphinx.util import logging
87except ImportError as ex: # pragma: no cover
88 raise MissingDependencyError(dependency="sphinx", extra="sphinx") from ex
90if TYPE_CHECKING: # pragma: no cover
91 # Only this directive needs a package index, so the model is imported when the configuration declares an
92 # entrypoint rather than when the extension is loaded - otherwise every documentation build using any of these
93 # roles would need the 'pypi' extra.
94 from sphinx.config import Config
95 from packaging.requirements import Requirement
96 from packaging.specifiers import SpecifierSet
97 from pyTooling.Dependency.Python import LicenseOverrides, Project, PythonPackageDependencyGraph
98 from pyTooling.Dependency.Python import PythonPackageIndex, Release, RequirementsFile
100from pyTooling.Documentation.Sphinx.Directives import BaseDirective, SphinxExtensionError, strip
101from pyTooling.Documentation.Sphinx.Directives import stripAndNormalize
104#: URL of the package index the tables are built from, unless :file:`conf.py` names another.
105DEFAULT_INDEX_URL = "https://pypi.org"
107#: URL of that index's JSON API.
108DEFAULT_API_URL = "https://pypi.org/pypi/"
110#: Levels of sub-dependencies rendered when nothing says otherwise; ``0`` expands until the tree ends.
111DEFAULT_DEPTH = 0
113#: Whether a version constraint is reduced to its lower bound when the document doesn't say.
114DEFAULT_SIMPLIFIED_VERSIONS = True
117@export
118class VersionFormat(Enum):
119 """How many parts of a version number a dependency table prints."""
121 Major = auto() #: The major part alone, ``≥9``.
122 MajorMinor = auto() #: Major and minor, ``≥9.1`` - the default.
123 MajorMinorPatch = auto() #: Major, minor and patch, ``≥9.1.2``.
124 All = auto() #: Every part the constraint states, ``≥9.1.2.dev3``.
126 def __str__(self) -> str:
127 """
128 Return this format's name, as a document writes it.
130 :returns: The enum member's name.
131 """
132 return self.name
135@export
136class DependencyFormat(Enum):
137 """What a line of a dependency tree states about a package."""
139 Package = auto() #: The name alone.
140 PackageVersion = auto() #: Name and version constraint.
141 PackageLicense = auto() #: Name and license.
142 PackageVersionLicense = auto() #: Name, version constraint and license - the default.
144 @readonly
145 def ShowsVersion(self) -> bool:
146 """
147 Whether this format states a version constraint.
149 :returns: ``True`` if the version is printed.
150 """
151 return self in (DependencyFormat.PackageVersion, DependencyFormat.PackageVersionLicense)
153 @readonly
154 def ShowsLicense(self) -> bool:
155 """
156 Whether this format states a license.
158 :returns: ``True`` if the license is printed.
159 """
160 return self in (DependencyFormat.PackageLicense, DependencyFormat.PackageVersionLicense)
162 def __str__(self) -> str:
163 """
164 Return this format's name, as a document writes it.
166 :returns: The enum member's name.
167 """
168 return self.name
171#: How many parts of a version number a table prints when the document doesn't say.
172DEFAULT_VERSION_FORMAT = VersionFormat.MajorMinor
174#: What a line of a dependency tree states when the document doesn't say.
175DEFAULT_DEPENDENCY_FORMAT = DependencyFormat.PackageVersionLicense
177#: Comparison operators as a reader writes them. Order matters - the two-character forms have to be tried first.
178OPERATOR_SYMBOLS = (("<=", "≤"), (">=", "≥"), ("!=", "≠"), ("==", "="))
180#: Operators a simplified constraint drops: an upper bound and an exclusion say what is *not* required.
181_DROPPED_OPERATORS = ("<", "<=", "!=")
183#: The table's columns, as ``(title, relative width)``.
184TABLE_COLUMNS = (("Package", 3), ("Version", 1), ("License", 2), ("Dependencies", 4))
186#: The fields one entrypoint may state in :file:`conf.py`, exactly one of them.
187#:
188#: The singular forms take a string and the plural forms an iterable of strings; they are otherwise the same
189#: statement, and a project writes whichever reads better where it stands.
190ENTRYPOINT_FIELDS = ("file", "files", "package", "packages")
192#: Prefix every configuration value of this extension carries in :file:`conf.py`.
193CONFIG_PREFIX = "pyTooling_Dependency"
195#: What Sphinx accepts as the rebuild condition of a configuration value - the one this extension uses.
196_ConfigRebuild = Literal["env"]
198__all__ = [
199 "DEFAULT_INDEX_URL", "DEFAULT_API_URL", "DEFAULT_DEPTH", "DEFAULT_SIMPLIFIED_VERSIONS", "OPERATOR_SYMBOLS",
200 "TABLE_COLUMNS", "ENTRYPOINT_FIELDS", "CONFIG_PREFIX",
201 "DEFAULT_VERSION_FORMAT", "DEFAULT_DEPENDENCY_FORMAT"
202]
204_logger = logging.getLogger(__name__)
206#: The two option enumerations, so one parser serves both.
207_FormatType = TypeVar("_FormatType", VersionFormat, DependencyFormat)
210def _OneLevelDown(depth: Nullable[int]) -> Nullable[int]:
211 """
212 Return the depth one level deeper, leaving an unlimited depth unlimited.
214 :param depth: Levels still to expand, or ``None`` for unlimited.
215 :returns: One level fewer, or ``None``.
216 """
217 return None if depth is None else depth - 1
220#: The collector of each running build, by the id of its Sphinx application.
221#:
222#: It can't live on the build environment, which Sphinx pickles between runs - an open HTTP session and the locks
223#: guarding lazy loading are not picklable, and a cached view of a package index would be stale anyway.
224_COLLECTORS: dict[int, "DependencyCollector"] = {}
227@export
228class Entrypoint(metaclass=ExtendedType, slots=True):
229 """
230 One entry of ``pyTooling_Dependency_Requirements``: an identifier and the requirements it stands for.
232 A file entrypoint is read while :file:`conf.py` is being processed and carries its requirements from then on.
233 A package entrypoint can only be resolved by asking the package index, so it carries the package's name and
234 extra and is resolved the first time a table names it.
235 """
237 _identifier: str #: Name the documents refer to this entrypoint by.
238 _files: tuple[Path, ...] #: Every requirements file read, references included.
239 _packages: tuple[tuple[str, Nullable[str]], ...] #: The packages to read, as name and extra.
240 _requirements: Nullable[dict[str, Requirement]] #: The resolved requirements, by canonical package name.
242 def __init__(
243 self,
244 identifier: str,
245 files: tuple[Path, ...] = (),
246 packages: tuple[tuple[str, Nullable[str]], ...] = (),
247 requirements: Nullable[dict[str, Requirement]] = None
248 ) -> None:
249 """
250 Describe one entrypoint.
252 :param identifier: Name the documents refer to this entrypoint by.
253 :param files: Optional, every requirements file read, for a file entrypoint. Default: ``()``.
254 :param packages: Optional, the packages and their extras, for a package entrypoint. Default: ``()``.
255 :param requirements: Optional, the requirements, if they are known already. Default: ``None``.
256 """
257 self._identifier = identifier
258 self._files = files
259 self._packages = packages
260 self._requirements = requirements
262 @readonly
263 def Identifier(self) -> str:
264 """
265 Name the documents refer to this entrypoint by.
267 :returns: The identifier.
268 """
269 return self._identifier
271 @readonly
272 def Files(self) -> tuple[Path, ...]:
273 """
274 The requirements file and every file it includes.
276 :returns: The files this entrypoint was read from; empty for a package entrypoint.
277 """
278 return self._files
280 @readonly
281 def Packages(self) -> tuple[tuple[str, Nullable[str]], ...]:
282 """
283 The packages this entrypoint reads the requirements of, as ``(name, extra)`` pairs.
285 :returns: The packages, or an empty tuple for a file entrypoint.
286 """
287 return self._packages
289 @readonly
290 def Requirements(self) -> Nullable[dict[str, Requirement]]:
291 """
292 The requirements this entrypoint stands for.
294 :returns: Every required package by its canonical name, or ``None`` if they weren't resolved yet.
295 """
296 return self._requirements
298 def CacheRequirements(self, requirements: dict[str, Requirement]) -> None:
299 """
300 Remember the requirements the package index answered with.
302 A package entrypoint can only be resolved by asking the index; remembering the answer is what keeps a second
303 table naming the same entrypoint from asking again.
305 :param requirements: Every required package, by its canonical name.
306 """
307 self._requirements = requirements
309 def __repr__(self) -> str:
310 """
311 Return a representation naming what this entrypoint reads.
313 :returns: The identifier and its source.
314 """
315 if len(self._files) > 0:
316 source = ", ".join(str(file) for file in self._files)
317 else:
318 source = ", ".join(name if extra is None else f"{name}[{extra}]" for name, extra in self._packages)
320 return f"<Entrypoint {self._identifier}: {source}>"
323@export
324class DependencyCollector(metaclass=ExtendedType, slots=True):
325 """
326 The entrypoints a build declared, the package index it queries, and what querying it cost.
328 One collector is shared by every table of a build: a package required by two entrypoints is downloaded once, and
329 the time is accumulated so the build can report a total. It exists because the alternative - a table that queries
330 the index for itself - multiplies a documentation build's runtime by however many tables it has, and
331 :file:`requirements.txt`, :file:`tests/requirements.txt` and :file:`doc/requirements.txt` share most of what they
332 require.
334 The index is opened the first time a table asks for a package, not when the collector is created: a project may
335 declare its entrypoints and then build a document that shows none of them, and that build should not open an
336 HTTP session.
337 """
339 _entrypoints: dict[str, Entrypoint] #: The entrypoints declared in :file:`conf.py`.
340 _indexURL: str #: URL of the package index's website.
341 _apiURL: str #: URL of the package index's JSON API.
342 _overrides: LicenseOverrides #: Licenses stated by hand, where the index can't.
343 _graph: Nullable[PythonPackageDependencyGraph] #: Graph the downloaded packages are collected in.
344 _index: Nullable[PythonPackageIndex] #: The package index this build queries, once opened.
345 _projects: dict[str, Nullable[Project]] #: Projects downloaded so far; ``None`` if unknown.
346 _detailed: set[str] #: Releases whose details were downloaded.
347 _undescribed: set[str] #: Releases the index lists but can't describe.
348 _stopwatch: Stopwatch #: Runs only while a request to the index is in flight.
349 _unresolved: dict[str, tuple[str, ...]] #: Packages whose license the index couldn't answer for,
350 #: mapped to what it published instead.
352 def __init__(
353 self,
354 entrypoints: dict[str, Entrypoint],
355 indexURL: str,
356 apiURL: str,
357 overrides: LicenseOverrides
358 ) -> None:
359 """
360 Collect what a build declared, without opening the package index yet.
362 :param entrypoints: The entrypoints declared in :file:`conf.py`, by identifier.
363 :param indexURL: URL of the package index's website.
364 :param apiURL: URL of the package index's JSON API.
365 :param overrides: Licenses stated by hand.
366 """
367 self._entrypoints = entrypoints
368 self._indexURL = indexURL
369 self._apiURL = apiURL
370 self._overrides = overrides
371 self._graph = None
372 self._index = None
373 self._projects = {}
374 self._detailed = set()
375 self._undescribed = set()
376 self._unresolved = {}
378 # 'preferPause', so each 'with' around a request is one active span: 'Activity' is the time spent waiting for
379 # the index rather than the age of the collector, and 'ActiveCount' is the number of requests
380 self._stopwatch = Stopwatch(preferPause=True)
382 @readonly
383 def Entrypoints(self) -> dict[str, Entrypoint]:
384 """
385 The entrypoints declared in :file:`conf.py`.
387 :returns: Every entrypoint by its identifier.
388 """
389 return self._entrypoints
391 @readonly
392 def Index(self) -> PythonPackageIndex:
393 """
394 The package index this build queries, opened the first time it is asked for.
396 :returns: The package index.
397 """
398 from pyTooling.Dependency.Python import PythonPackageDependencyGraph, PythonPackageIndex
400 if self._index is None:
401 self._graph = PythonPackageDependencyGraph("documentation")
402 self._index = PythonPackageIndex("index", self._indexURL, self._apiURL, self._graph, self._overrides)
404 return self._index
406 @readonly
407 def RequestCount(self) -> int:
408 """
409 Number of requests sent to the package index.
411 The stopwatch runs for exactly one span per request, so this is its
412 :attr:`~pyTooling.Stopwatch.Stopwatch.ActiveCount` - counting them a second time in a field of our own
413 would be a second answer to one question.
415 :returns: Number of requests sent.
416 """
417 return self._stopwatch.ActiveCount
419 @readonly
420 def Seconds(self) -> float:
421 """
422 Time spent waiting for the package index, in seconds.
424 This is the stopwatch's :attr:`~pyTooling.Stopwatch.Stopwatch.Activity` - the sum of the intervals it ran -
425 not its duration, because it is paused between requests and everything the build does in between is not
426 time this collector spent.
428 :returns: Seconds spent on the index.
429 """
430 return self._stopwatch.Activity
432 @readonly
433 def UnresolvedLicenses(self) -> dict[str, tuple[str, ...]]:
434 """
435 Packages whose license the index couldn't answer for, and what it published instead.
437 The published fields are what the override file has to answer for, so they are kept rather than only the
438 package's name: ``License :: OSI Approved :: BSD License`` names three licenses and is never guessed at, and
439 a ``license`` field holding a license's title instead of its SPDX identifier doesn't parse.
441 :returns: Names of the packages needing a license override, mapped to what the index published for them.
442 """
443 return self._unresolved
445 def Project(self, packageName: str) -> Nullable[Project]:
446 """
447 Return a project, downloading it the first time it is asked for.
449 A package the index doesn't know is remembered as unknown, so a table naming it doesn't ask again for every
450 row that mentions it.
452 :param packageName: Name of the package to look up.
453 :returns: The project, or ``None`` if the index doesn't know it.
454 :raises MissingDependencyError: If the 'pypi' extra isn't installed.
455 """
456 try:
457 from requests import RequestException
458 except ImportError as ex: # pragma: no cover
459 raise MissingDependencyError(dependency="requests", extra="pypi") from ex
461 from pyTooling.Dependency import DependencyError
462 from pyTooling.Dependency.Python import LazyLoaderState
464 if packageName in self._projects:
465 return self._projects[packageName]
467 project: Nullable[Project]
468 with self._stopwatch:
469 try:
470 project = self.Index.DownloadProject(packageName, LazyLoaderState.PartiallyLoaded)
471 except (DependencyError, RequestException, ValueError, KeyError):
472 project = None
474 self._projects[packageName] = project
476 return project
478 def Details(self, release: Release) -> Nullable[Release]:
479 """
480 Make sure a release knows its own requirements and its license.
482 A release the index lists but can't describe - a yanked one, or a version its release endpoint spells
483 differently - is remembered as unusable and answered with ``None``. Handing back the release itself would
484 be worse than useless: its lazily loaded properties would each retry the download and raise.
486 :param release: The release to fill in.
487 :returns: The release with its details, or ``None`` if the index can't describe it.
488 :raises MissingDependencyError: If the 'pypi' extra isn't installed.
489 """
490 try:
491 from requests import RequestException
492 except ImportError as ex: # pragma: no cover
493 raise MissingDependencyError(dependency="requests", extra="pypi") from ex
495 from pyTooling.Dependency import DependencyError
497 key = f"{release.Package.Name}=={release.Version}"
498 if key in self._detailed:
499 return release if key not in self._undescribed else None
501 warnings: list[BaseException] = []
502 with self._stopwatch, WarningCollector(warnings):
503 try:
504 release.DownloadDetails()
505 except (DependencyError, RequestException, ValueError, KeyError):
506 self._undescribed.add(key)
508 self._detailed.add(key)
510 for warning in warnings:
511 if isinstance(warning, UnknownLicenseWarning):
512 # the warning's notes are what the index published, which is the reason an override is needed
513 self._unresolved[release.Package.Name] = warning.Notes
515 return None if key in self._undescribed else release
518@export
519def readEntrypoints(configuration: Any, confDirectory: Path) -> dict[str, Entrypoint]:
520 """
521 Turn ``pyTooling_Dependency_Requirements`` into entrypoints, reading every requirements file it names.
523 A requirements file is read here rather than when a table is built, so a path that doesn't exist ends the build
524 with one message naming the identifier instead of an error box in the middle of a page - and so two tables
525 naming the same file read it once.
527 :param configuration: Value of ``pyTooling_Dependency_Requirements``.
528 :param confDirectory: Directory :file:`conf.py` lives in; relative paths are resolved against it.
529 :returns: Every declared entrypoint, by its identifier.
530 :raises MissingDependencyError: If the 'pypi' extra isn't installed.
531 :raises SphinxExtensionError: If the configuration is malformed, or a requirements file can't be read.
532 """
533 if not isinstance(configuration, dict):
534 raise SphinxExtensionError(
535 f"conf.py: {CONFIG_PREFIX}_Requirements: Expected a dictionary, "
536 f"got '{getFullyQualifiedName(configuration)}'."
537 )
539 entrypoints: dict[str, Entrypoint] = {}
540 for identifier, declaration in configuration.items():
541 location = f"conf.py: {CONFIG_PREFIX}_Requirements:[{identifier}]"
543 if not isinstance(declaration, dict):
544 raise SphinxExtensionError(
545 f"{location}: Expected a dictionary, got '{getFullyQualifiedName(declaration)}'."
546 )
548 if (unknown := set(declaration) - set(ENTRYPOINT_FIELDS)) != set(): 548 ↛ 549line 548 didn't jump to line 549 because the condition on line 548 was never true
549 raise SphinxExtensionError(
550 f"{location}: Unknown field(s): {', '.join(sorted(unknown))}. "
551 f"Known are: {', '.join(ENTRYPOINT_FIELDS)}."
552 )
554 if len(stated := [field for field in ENTRYPOINT_FIELDS if field in declaration]) != 1:
555 known = ", ".join(ENTRYPOINT_FIELDS)
556 raise SphinxExtensionError(
557 f"{location}: Exactly one of {known} has to be configured, "
558 f"{'none is' if len(stated) == 0 else f'{len(stated)} are'}."
559 )
561 field = stated[0]
562 fieldLocation = f"{location}.{field}"
563 value = declaration[field]
564 values: tuple[str, ...]
566 if field in ("file", "package"):
567 if not isinstance(value, str):
568 raise SphinxExtensionError(
569 f"{fieldLocation}: Expected a string, got '{getFullyQualifiedName(value)}'."
570 )
572 values = (value,)
573 else:
574 # a string is an iterable of strings itself, so the plural form has to reject one explicitly - otherwise
575 # {"files": "requirements.txt"} would silently become sixteen one-character paths
576 if isinstance(value, str) or not isinstance(value, Iterable):
577 raise SphinxExtensionError(
578 f"{fieldLocation}: Expected an iterable of strings, got '{getFullyQualifiedName(value)}'. "
579 f"Use '{field[:-1]}' for a single value."
580 )
582 # materialized before the items are checked, because an iterable may be a generator this would consume
583 values = tuple(value)
584 for item in values:
585 if not isinstance(item, str): 585 ↛ 586line 585 didn't jump to line 586 because the condition on line 585 was never true
586 raise SphinxExtensionError(
587 f"{fieldLocation}: Expected strings, got '{getFullyQualifiedName(item)}'."
588 )
590 if field in ("file", "files"):
591 entrypoints[identifier] = _FileEntrypoint(identifier, fieldLocation, values, confDirectory)
592 else:
593 entrypoints[identifier] = _PackageEntrypoint(identifier, values)
595 return entrypoints
598def _FileEntrypoint(
599 identifier: str,
600 location: str,
601 files: tuple[str, ...],
602 confDirectory: Path
603) -> Entrypoint:
604 """
605 Read one entrypoint's requirements files.
607 Several files are read as several trees and flattened in the order they are declared, so a later file's
608 statement wins - the rule a single file's ``-r`` references already follow.
610 :param identifier: Identifier of the entrypoint.
611 :param location: Where in :file:`conf.py` this came from.
612 :param files: The declared paths.
613 :param confDirectory: Directory relative paths resolve against.
614 :returns: The entrypoint, with its requirements read.
615 :raises MissingDependencyError: If the 'pypi' extra isn't installed.
616 :raises SphinxExtensionError: If a file can't be read.
617 """
618 try:
619 from packaging.utils import canonicalize_name
620 except ImportError as ex: # pragma: no cover
621 raise MissingDependencyError(dependency="packaging", extra="pypi") from ex
623 from pyTooling.Dependency import DependencyError
624 from pyTooling.Dependency.Python import RequirementsFile
626 readFiles: list[Path] = []
627 requirements: dict[str, Requirement] = {}
629 for file in files:
630 path = Path(file)
631 if not path.is_absolute(): 631 ↛ 634line 631 didn't jump to line 634 because the condition on line 631 was always true
632 path = confDirectory / path
634 try:
635 requirementsFile = RequirementsFile(path)
636 except (DependencyError, OSError, UnicodeDecodeError) as cause:
637 raise SphinxExtensionError(f"{location}: Requirements file '{path}' can't be read: {cause}") from cause
639 # the tree knows every file it was read from; walking it here would be a second answer to one question
640 readFiles.extend(requirementsFile.AnalyzedRequirementFiles)
641 requirements.update({canonicalize_name(req.name): req for req in requirementsFile.AllRequirements})
643 return Entrypoint(identifier, files=tuple(readFiles), requirements=requirements)
646def _PackageEntrypoint(identifier: str, packages: tuple[str, ...]) -> Entrypoint:
647 """
648 Describe one entrypoint's packages, which only the package index can resolve.
650 :param identifier: Identifier of the entrypoint.
651 :param packages: The declared packages, each optionally with one extra.
652 :returns: The entrypoint, with its packages recorded and its requirements still unresolved.
653 """
654 requested: list[tuple[str, Nullable[str]]] = []
655 for package in packages:
656 name, _, bracket = package.partition("[")
657 requested.append((name.strip(), bracket.rstrip("]").strip() or None))
659 return Entrypoint(identifier, packages=tuple(requested))
662@export
663def prepareEntrypoints(sphinx: Sphinx, config: Config) -> None:
664 """
665 Call-back for Sphinx' ``config-inited`` event, reading the entrypoints and the license overrides.
667 A build declaring no entrypoint does nothing here - not even import :mod:`pyTooling.Dependency.Python`, so a
668 project using only this extension's roles doesn't need the ``pypi`` extra.
670 :param sphinx: The Sphinx application.
671 :param config: The configuration, after :file:`conf.py` was read.
672 :raises SphinxExtensionError: If the configuration is malformed, or a
673 requirements or license override file can't be read.
674 """
675 if len(declarations := getattr(config, f"{CONFIG_PREFIX}_Requirements", {})) == 0: 675 ↛ 678line 675 didn't jump to line 678 because the condition on line 675 was always true
676 return
678 confDirectory = Path(sphinx.confdir)
680 try:
681 from pyTooling.Dependency import DependencyError
682 from pyTooling.Dependency.Python import LicenseOverrides
683 except MissingDependencyError as cause: # pragma: no cover
684 raise SphinxExtensionError(
685 f"conf.py: {CONFIG_PREFIX}_Requirements: Querying a package index needs the 'pypi' extra: "
686 f"pip install pyTooling[pypi]"
687 ) from cause
689 overrides = LicenseOverrides()
690 if (overrideFile := getattr(config, f"{CONFIG_PREFIX}_PackageOverrides", None)) is not None:
691 path = Path(overrideFile)
692 if not path.is_absolute():
693 path = confDirectory / path
695 try:
696 overrides = LicenseOverrides.FromFile(path)
697 except (DependencyError, ConfigurationError, OSError) as cause:
698 raise SphinxExtensionError(
699 f"conf.py: {CONFIG_PREFIX}_PackageOverrides: Override file '{path}' can't be read: {cause}"
700 ) from cause
702 _COLLECTORS[id(sphinx)] = DependencyCollector(
703 readEntrypoints(declarations, confDirectory),
704 getattr(config, f"{CONFIG_PREFIX}_IndexURL", DEFAULT_INDEX_URL),
705 getattr(config, f"{CONFIG_PREFIX}_APIURL", DEFAULT_API_URL),
706 overrides
707 )
710@export
711class DependencyTable(BaseDirective):
712 """
713 The ``dependency-table`` directive: an entrypoint's dependencies, rendered from the requirements.
715 One argument, the identifier of an entrypoint declared in ``pyTooling_Dependency_Requirements``. ``:depth:``
716 says how many levels of sub-dependencies to expand, ``:simplified-versions:`` whether a constraint is reduced to
717 its lower bound, and ``:caption:`` puts a caption under the table; which package index is queried and which
718 licenses are stated by hand are build-wide and configured in :file:`conf.py`.
719 """
721 directiveName: str = "dependency-table" #: Name the directive is invoked by.
723 #: The configuration values this directive adds to :file:`conf.py`, as ``name: (default, rebuild, types)``. Each is
724 #: registered with :data:`CONFIG_PREFIX` as prefix, e.g. ``pyTooling_Dependency_Requirements``.
725 #:
726 #: ``Requirements`` maps an identifier to what it names - a file, files, a package or packages. The other three are
727 #: build-wide, because one package index is queried per build and one override file answers for it. All four are
728 #: ``"env"``-rebuilt: changing any of them changes every table.
729 configValues: ClassVar[dict[str, tuple[Any, _ConfigRebuild, Any]]] = {
730 "Requirements": ({}, "env", dict),
731 "PackageOverrides": (None, "env", (str, Path)),
732 "IndexURL": (DEFAULT_INDEX_URL, "env", str),
733 "APIURL": (DEFAULT_API_URL, "env", str),
734 }
736 _simplify: bool #: Whether this table's version constraints are reduced to their lower bound.
737 _versionFormat: VersionFormat #: How many parts of a version number this table prints.
738 _dependencyFormat: DependencyFormat #: What a line of this table's dependency trees states.
740 has_content = False #: A boolean; ``True`` if content is allowed.
741 required_arguments = 1 #: Number of required directive arguments: the entrypoint's identifier.
742 optional_arguments = 0 #: Number of optional arguments after the required ones.
743 final_argument_whitespace = False #: A boolean; ``True`` if the last argument may contain spaces.
744 # docutils declares 'option_spec' on 'Directive' and 'BaseDirective' assigns it, so mypy calls every
745 # spelling of this override a conflict with one of them
746 #: Mapping of option names to validator functions.
747 option_spec: dict[str, Any] = { # type: ignore[misc]
748 "caption": strip,
749 "depth": directives.nonnegative_int,
750 "simplified-versions": stripAndNormalize,
751 "version-format": stripAndNormalize,
752 "dependency-format": stripAndNormalize,
753 }
755 def run(self) -> list[nodes.Node]:
756 """
757 Resolve the named entrypoint against the package index and return its requirements as a table.
759 :returns: A ``table`` node, or an error node when the entrypoint couldn't be resolved.
760 """
761 identifier = self.arguments[0].strip()
762 self._simplify = self._ParseBooleanOption("simplified-versions", DEFAULT_SIMPLIFIED_VERSIONS)
763 self._versionFormat = self._ParseFormatOption("version-format", VersionFormat, DEFAULT_VERSION_FORMAT)
764 self._dependencyFormat = self._ParseFormatOption(
765 "dependency-format", DependencyFormat, DEFAULT_DEPENDENCY_FORMAT
766 )
768 with Stopwatch() as stopwatch:
769 try:
770 collector = self._Collector()
771 requestsBefore = collector.RequestCount
772 requirements = self._Resolve(identifier, collector)
773 table = self._CreateTable(identifier, requirements, collector)
774 except SphinxExtensionError as cause:
775 return [self.state.document.reporter.error(
776 f"{self.directiveName}: {cause}", line=self.lineno
777 )]
779 # what this table cost, not what the build has spent so far - a package another table already downloaded is
780 # free here, and that is the point of sharing the collector
781 _logger.info(
782 f"[{self.directiveName}] {identifier}: {len(requirements)} package(s), "
783 f"{collector.RequestCount - requestsBefore} request(s), {stopwatch.Duration:.2f} s"
784 )
786 return [table]
788 def _ParseFormatOption(self, optionName: str, enumType: type[_FormatType], default: _FormatType) -> _FormatType:
789 """
790 Read an option naming a member of an enumeration, or fall back to its default.
792 :attr:`~pyTooling.Documentation.Sphinx.Directives.BaseDirective._ParseEnumOption` requires the option and
793 lower-cases what it reads; these two have a default and are written the way the members are spelled, so a
794 document says ``:version-format: MajorMinor`` rather than ``major_minor``.
796 :param optionName: Name of the option to read.
797 :param enumType: The enumeration its value names a member of.
798 :param default: The member to use when the option isn't given.
799 :returns: The named member.
800 :raises SphinxExtensionError: If the value names no member.
801 """
802 if (option := self.options.get(optionName, None)) is None:
803 return default
805 for member in enumType:
806 if option.lower() == member.name.lower():
807 return member
809 known = ", ".join(member.name for member in enumType)
810 raise SphinxExtensionError(
811 f"{self.directiveName}::{optionName}: '{option}' is not one of: {known}."
812 )
814 def _Collector(self) -> DependencyCollector:
815 """
816 Return the build's collector, which :func:`prepareEntrypoints` created when :file:`conf.py` was read.
818 The collector belongs to the running application rather than to the directive, because a document with three
819 tables would otherwise open three indexes and download the same packages three times. It deliberately does
820 *not* live on the build environment: Sphinx pickles that between runs, and neither an open HTTP session nor a
821 cached view of a package index survives being pickled - or should.
823 :returns: The collector shared by every table of this build.
824 :raises SphinxExtensionError: If no entrypoint was configured.
825 """
826 if (collector := _COLLECTORS.get(id(self.env.app), None)) is None:
827 raise SphinxExtensionError(
828 f"No entrypoint is configured. Declare one in conf.py: {CONFIG_PREFIX}_Requirements."
829 )
831 return collector
833 def _Resolve(self, identifier: str, collector: DependencyCollector) -> dict[str, Requirement]:
834 """
835 Return what the named entrypoint requires.
837 A file entrypoint was read when :file:`conf.py` was processed and answers immediately; a package entrypoint
838 is resolved against the package index the first time a table names it, and remembers the answer.
840 :param identifier: Identifier the document names.
841 :param collector: The build's collector.
842 :returns: Every required package, by its canonical name.
843 :raises MissingDependencyError: If the 'pypi' extra isn't installed.
844 :raises SphinxExtensionError: If the identifier is unknown, or the
845 package index can't answer for the entrypoint's package.
846 """
847 try:
848 from packaging.utils import canonicalize_name
849 except ImportError as ex: # pragma: no cover
850 raise MissingDependencyError(dependency="packaging", extra="pypi") from ex
852 if (entrypoint := collector.Entrypoints.get(identifier, None)) is None:
853 known = ", ".join(sorted(collector.Entrypoints)) or "none"
854 raise SphinxExtensionError(
855 f"Entrypoint '{identifier}' is not configured in conf.py: {CONFIG_PREFIX}_Requirements. "
856 f"Known are: {known}."
857 )
859 for file in entrypoint.Files:
860 self.env.note_dependency(str(file))
862 if (requirements := entrypoint.Requirements) is not None:
863 return requirements
865 # several packages flatten in the order they are declared, the rule a file's '-r' references already follow
866 requirements = {}
867 for packageName, extra in entrypoint.Packages:
868 for requirement in self._PublishedRequirements(packageName, extra, collector):
869 requirements[canonicalize_name(requirement.name)] = requirement
871 entrypoint.CacheRequirements(requirements)
873 return requirements
875 def _PublishedRequirements(
876 self,
877 packageName: str,
878 extra: Nullable[str],
879 collector: DependencyCollector
880 ) -> list[Requirement]:
881 """
882 Ask the package index what a package's latest release requires.
884 :param packageName: Name of the package to ask about.
885 :param extra: Extra whose requirements are wanted, or ``None`` for the package's own.
886 :param collector: The build's collector.
887 :returns: What that release requires.
888 :raises SphinxExtensionError: If the index doesn't know the
889 package, can't describe its latest release, or the package has no such extra.
890 """
891 if (project := collector.Project(packageName)) is None:
892 raise SphinxExtensionError(f"Package '{packageName}' is unknown to the package index.")
894 if (release := collector.Details(project.LatestRelease)) is None:
895 raise SphinxExtensionError(f"The package index can't describe the latest release of '{packageName}'.")
897 published: Nullable[list[Requirement]] = release.Requirements.get(extra, None)
898 if published is None:
899 known = ", ".join(sorted(str(key) for key in release.Requirements if key is not None))
900 raise SphinxExtensionError(f"Package '{packageName}' has no extra '{extra}'. Known are: {known}.")
902 return published
904 def _CreateTable(
905 self,
906 identifier: str,
907 requirements: dict[str, Requirement],
908 collector: DependencyCollector
909 ) -> nodes.table:
910 """
911 Render the requirements as a four-column table.
913 :param identifier: Identifier of the entrypoint, used as the table's identifier.
914 :param requirements: Every required package, by its canonical name.
915 :param collector: The build's collector.
916 :returns: The finished table.
917 """
918 tableGroup = self._CreateSingleRowTableHeader(
919 columns=list(TABLE_COLUMNS),
920 identifier=identifier,
921 classes=["dependency-table"]
922 )
923 tableGroup += (tableBody := nodes.tbody())
925 # ':depth: 0' - and the default - means expand until the tree ends; 'None' is that, internally, because
926 # 'depth - 1' would otherwise walk 0 into negative numbers and mean two different things at once
927 depth = self.options.get("depth", DEFAULT_DEPTH)
928 levels: Nullable[int] = None if depth == 0 else depth
930 if len(requirements) == 0:
931 tableBody += self._CreateEmptyRow(len(TABLE_COLUMNS))
932 else:
933 for name in sorted(requirements, key=str.lower):
934 tableBody += self._CreateRow(requirements[name], collector, levels)
936 table = cast(nodes.table, tableGroup.parent)
937 if (caption := self.options.get("caption", None)) is not None:
938 # the caption is ReST, not text: it is written with markup - ``packaging`` in pyTooling's own captions -
939 # and a 'title' built from a string would print the backticks
940 captionNodes, messages = self.state.inline_text(caption, self.lineno)
941 table.insert(0, nodes.title(caption, "", *captionNodes, *messages))
943 return table
945 @staticmethod
946 def _CreateEmptyRow(columnCount: int) -> nodes.row:
947 """
948 Render the one row a table with no requirements gets: a single cell spanning every column.
950 A table showing nothing but its header reads as a defect. pyTooling's own :file:`requirements.txt` is empty
951 - the package has no mandatory dependencies - and that is a statement worth printing.
953 :param columnCount: Number of columns the cell has to span.
954 :returns: The table row.
955 """
956 tableRow = nodes.row("", classes=["dependency-table-row"])
958 entry = nodes.entry("", morecols=columnCount - 1)
959 entry += nodes.paragraph("", "", nodes.emphasis(text="No dependencies"))
960 tableRow += entry
962 return tableRow
964 def _CreateRow(
965 self,
966 requirement: Requirement,
967 collector: DependencyCollector,
968 depth: Nullable[int]
969 ) -> nodes.row:
970 """
971 Render one required package as a table row.
973 A package the index doesn't know, or one with no release matching the requirement, still gets a row - the
974 specifier the entrypoint states is worth showing even when nothing else could be resolved.
976 :param requirement: The requirement to render.
977 :param collector: The build's collector.
978 :param depth: Levels of sub-dependencies still to expand.
979 :returns: The table row.
980 """
981 tableRow = nodes.row("", classes=["dependency-table-row"])
983 project = collector.Project(requirement.name)
984 release = self._SelectRelease(project, requirement, collector)
986 tableRow += self._PackageEntry(requirement, project)
987 specifier = self._FormatSpecifier(requirement.specifier, self._simplify, self._versionFormat)
989 tableRow += nodes.entry("", nodes.paragraph(text=specifier))
990 tableRow += self._LicenseEntry(release)
991 tableRow += self._DependenciesEntry(release, collector, depth, {requirement.name.lower()})
993 return tableRow
995 def _SelectRelease(
996 self,
997 project: Nullable[Project],
998 requirement: Requirement,
999 collector: DependencyCollector
1000 ) -> Nullable[Release]:
1001 """
1002 Return the newest release satisfying a requirement.
1004 Pre-releases are skipped unless the specifier asks for them, because that is what an installer would resolve
1005 to and the table describes what would be installed.
1007 :param project: The project to pick a release of, or ``None`` if the index doesn't know it.
1008 :param requirement: The requirement to satisfy.
1009 :param collector: The build's collector.
1010 :returns: The newest matching release, or ``None`` if nothing matches.
1011 """
1012 if project is None:
1013 return None
1015 matching = [
1016 release for version, release in project.Releases.items()
1017 if requirement.specifier.contains(str(version))
1018 ]
1019 if len(matching) == 0:
1020 return None
1022 return collector.Details(max(matching, key=lambda release: release.Version))
1024 @staticmethod
1025 def _FormatVersion(version: str, versionFormat: VersionFormat) -> str:
1026 """
1027 Shorten a version number to the parts a table prints.
1029 ``≥0.4.6`` says more than a reader of a dependency table needs; the parts that matter are the ones a
1030 constraint is usually written against. A version with fewer parts than asked for is left as it is - ``≥9``
1031 does not become ``≥9.0`` - because padding would state a precision the requirement didn't.
1033 :param version: The version, as the constraint writes it.
1034 :param versionFormat: How many parts to keep.
1035 :returns: The shortened version.
1036 """
1037 if versionFormat is VersionFormat.All:
1038 return version
1040 parts = {VersionFormat.Major: 1, VersionFormat.MajorMinor: 2, VersionFormat.MajorMinorPatch: 3}[versionFormat]
1042 return ".".join(version.split(".")[:parts])
1044 @staticmethod
1045 def _FormatSpecifier(specifier: SpecifierSet, simplify: bool, versionFormat: VersionFormat) -> str:
1046 """
1047 Render a version constraint the way a reader writes one.
1049 The comparison operators become their mathematical symbols and the versions are shortened to
1050 ``versionFormat``. A simplified constraint keeps only what a package *has to be at least*: an upper bound
1051 and an exclusion say what a release must not be, which is the packaging problem rather than the reader's,
1052 and ``~=`` is written as the lower bound it implies. Simplifying everything away leaves the constraint as it
1053 was written - ``<4.0`` alone is still the whole statement.
1055 :param specifier: The constraint to render.
1056 :param simplify: Whether to reduce the constraint to its lower bound.
1057 :param versionFormat: How many parts of each version to keep.
1058 :returns: The constraint, or ``any`` when nothing is constrained.
1059 """
1060 def render(operator: str, version: str) -> str:
1061 shortened = DependencyTable._FormatVersion(version, versionFormat)
1062 for written, symbol in OPERATOR_SYMBOLS:
1063 if operator == written:
1064 return f"{symbol}{shortened}"
1066 return f"{operator}{shortened}"
1068 if len(specifier) == 0:
1069 return "any"
1071 specifiers = sorted(specifier, key=lambda item: (item.version, item.operator))
1072 if simplify:
1073 kept = [
1074 render(">=" if item.operator == "~=" else item.operator, item.version)
1075 for item in specifiers if item.operator not in _DROPPED_OPERATORS
1076 ]
1077 if len(kept) > 0:
1078 # shortening can make two constraints identical - '>=1.2.3, >1.2.9' is '≥1.2, >1.2' at MajorMinor -
1079 # and printing one statement twice reads as a defect
1080 return ", ".join(dict.fromkeys(kept))
1082 rendered = [render(item.operator, item.version) for item in specifiers]
1084 return ", ".join(dict.fromkeys(rendered))
1086 @staticmethod
1087 def _PackageURL(project: Nullable[Project]) -> Nullable[str]:
1088 """
1089 Return the page a package's name should link to, most useful first.
1091 A project states none of these reliably, so there are three chances at one: its **documentation** answers
1092 *what is this*, its **repository** answers *where does it come from*, and its page on the **package index**
1093 is what the index itself can always answer. Only a package the index doesn't know at all goes unlinked.
1095 :param project: The project, or ``None`` if the index doesn't know it.
1096 :returns: The URL to link the name to, or ``None`` if there is nothing to link to.
1097 """
1098 if project is None:
1099 return None
1101 for url in (project.DocumentationURL, project.RepositoryURL, project.URL):
1102 if url is not None:
1103 return str(url)
1105 return None
1107 @classmethod
1108 def _PackageEntry(cls, requirement: Requirement, project: Nullable[Project]) -> nodes.entry:
1109 """
1110 Render the package's name, linked to where a reader can find out about it.
1112 :param requirement: The requirement naming the package.
1113 :param project: The project, or ``None`` if the index doesn't know it.
1114 :returns: The table entry.
1115 """
1116 entry = nodes.entry()
1117 name = project.Name if project is not None else requirement.name
1119 if (url := cls._PackageURL(project)) is not None:
1120 entry += nodes.paragraph("", "", nodes.reference("", name, refuri=url))
1121 else:
1122 entry += nodes.paragraph(text=name)
1124 return entry
1126 @staticmethod
1127 def _LicenseEntry(release: Nullable[Release]) -> nodes.entry:
1128 """
1129 Render a release's license, linked to its text where one is known.
1131 The license' **name** is shown rather than its SPDX identifier - ``Apache License 2.0``, not ``Apache-2.0`` -
1132 because the table is read by a person and the identifier is what an expression writes.
1134 A license that didn't resolve is an :class:`~pyTooling.Licensing.UnknownLicense`, never a blank cell, and it
1135 is shown **as the index published it** - in italics, so it reads as a quotation rather than as an identifier.
1136 The reader should see that the index said *something*, and what it was.
1138 :param release: The release to render the license of, or ``None``.
1139 :returns: The table entry.
1140 """
1141 entry = nodes.entry()
1143 if (text := DependencyTable._LicenseName(release)) is None:
1144 entry += nodes.paragraph("", "", nodes.emphasis(text=DependencyTable._PublishedLicense(release)))
1146 return entry
1148 if (url := DependencyTable._LicenseURL(release)) is not None:
1149 entry += nodes.paragraph("", "", nodes.reference("", text, refuri=url))
1150 else:
1151 entry += nodes.paragraph(text=text)
1153 return entry
1155 @staticmethod
1156 def _LicenseName(release: Nullable[Release]) -> Nullable[str]:
1157 """
1158 Return the name(s) of the licenses a release is published under.
1160 :param release: The release to name the license of, or ``None``.
1161 :returns: The license' name, or ``None`` if nothing resolved.
1162 """
1163 from pyTooling.Licensing import UnknownLicense
1165 if release is None:
1166 return None
1168 # 'Licenses' never comes back empty: what didn't resolve is an 'UnknownLicense', which is SPDX's own way of
1169 # saying so and keeps the published text
1170 licenses = release.Licenses
1171 if all(isinstance(license, UnknownLicense) for license in licenses):
1172 return None
1174 return ", ".join(license.Name for license in licenses)
1176 @staticmethod
1177 def _PublishedLicense(release: Nullable[Release]) -> str:
1178 """
1179 Return what the package index published, for a license that didn't resolve.
1181 :param release: The release, or ``None`` if the index couldn't describe it.
1182 :returns: What was published, or ``unknown`` when that was nothing either.
1183 """
1184 if release is None:
1185 return "unknown"
1187 published = release.LicenseExpression.OriginalText.strip()
1189 return published if published != "" else ", ".join(license.Name for license in release.Licenses) or "unknown"
1191 @staticmethod
1192 def _LicenseURL(release: Nullable[Release]) -> Nullable[str]:
1193 """
1194 Return the page a license should link to, most specific first.
1196 **The project's own** :file:`LICENSE` **file wins**: it is the license as this project publishes it, which
1197 is the document a reader auditing a dependency actually wants. Most projects don't state one, though - it
1198 comes from ``project_urls`` or from the override file - so a license on the SPDX List falls back to its own
1199 published pages, in the order of who is speaking: the licensor's own page, then OSI's entry, then SPDX's.
1200 A ``LicenseRef-`` has none of those and stays unlinked, because nothing published it.
1202 :param release: The release to link the license of, or ``None``.
1203 :returns: The URL to link to, or ``None`` if nothing published this license.
1204 """
1205 from pyTooling.Licensing import SPDXLicense
1207 if release is None:
1208 return None
1210 if release.LicenseURL is not None:
1211 return str(release.LicenseURL)
1213 for license in release.Licenses:
1214 if isinstance(license, SPDXLicense):
1215 for url in (license.License.URL, license.License.OSIURL, license.License.SPDXURL):
1216 if url is not None:
1217 return url
1219 return None
1221 def _DependenciesEntry(
1222 self,
1223 release: Nullable[Release],
1224 collector: DependencyCollector,
1225 depth: Nullable[int],
1226 visited: set[str]
1227 ) -> nodes.entry:
1228 """
1229 Render a release's own requirements as a nested bullet list.
1231 Only the unconditional requirements are listed - what an extra pulls in is that extra's table, not this one.
1232 A package already on the path is not expanded again, so a dependency cycle terminates.
1234 :param release: The release to render the dependencies of, or ``None``.
1235 :param collector: The build's collector.
1236 :param depth: Levels still to expand; at zero nothing is expanded.
1237 :param visited: Packages already on this path, lower-cased.
1238 :returns: The table entry.
1239 """
1240 entry = nodes.entry()
1242 if release is None or (depth is not None and depth <= 0):
1243 entry += nodes.paragraph("", "", nodes.emphasis(text="not evaluated"))
1244 return entry
1246 requirements = [
1247 requirement for requirement in release.Requirements.get(None, [])
1248 if requirement.name.lower() not in visited
1249 ]
1250 if len(requirements) == 0:
1251 entry += nodes.paragraph("", "", nodes.emphasis(text="none"))
1252 return entry
1254 entry += self._CreateBulletList(requirements, collector, _OneLevelDown(depth), visited)
1256 return entry
1258 def _CreateBulletList(
1259 self,
1260 requirements: list[Requirement],
1261 collector: DependencyCollector,
1262 depth: Nullable[int],
1263 visited: set[str]
1264 ) -> nodes.bullet_list:
1265 """
1266 Render requirements as a bullet list, each item expanded by one more level.
1268 :param requirements: The requirements to list.
1269 :param collector: The build's collector.
1270 :param depth: Levels still to expand below this list.
1271 :param visited: Packages already on this path, lower-cased.
1272 :returns: The bullet list.
1273 """
1274 bulletList = nodes.bullet_list()
1276 for requirement in sorted(requirements, key=lambda item: item.name.lower()):
1277 item = nodes.list_item()
1279 # a leaf is resolved too when the line states a license - that is the whole point of stating it - but a
1280 # ':dependency-format:' that prints no license has no reason to send the requests
1281 project = collector.Project(requirement.name)
1282 release = (
1283 self._SelectRelease(project, requirement, collector)
1284 if depth is None or depth > 0 or self._dependencyFormat.ShowsLicense
1285 else None
1286 )
1288 item += self._RequirementParagraph(requirement, project, release)
1290 if (depth is None or depth > 0) and release is not None:
1291 nested = [
1292 nestedRequirement for nestedRequirement in release.Requirements.get(None, [])
1293 if nestedRequirement.name.lower() not in visited
1294 ]
1295 if len(nested) > 0:
1296 item += self._CreateBulletList(
1297 nested, collector, _OneLevelDown(depth), visited | {requirement.name.lower()}
1298 )
1300 bulletList += item
1302 return bulletList
1304 def _RequirementParagraph(
1305 self,
1306 requirement: Requirement,
1307 project: Nullable[Project],
1308 release: Nullable[Release]
1309 ) -> nodes.paragraph:
1310 """
1311 Render one line of a dependency tree: the package, what is required of it, and what it is licensed under.
1313 The license is the reason a dependency tree is in this table at all - a package pulls in what its own
1314 dependencies are licensed under, and reading that off the tree is the point. It is linked and parenthesised
1315 so the line still reads as one requirement.
1317 :param requirement: The requirement to render.
1318 :param project: The project, or ``None`` if the index doesn't know it.
1319 :param release: The release satisfying the requirement, or ``None`` if none was found.
1320 :returns: The paragraph.
1321 """
1322 paragraph = nodes.paragraph()
1324 name = project.Name if project is not None else requirement.name
1325 if (url := self._PackageURL(project)) is not None:
1326 paragraph += nodes.reference("", name, refuri=url)
1327 else:
1328 paragraph += nodes.Text(name)
1330 if self._dependencyFormat.ShowsVersion:
1331 specifier = self._FormatSpecifier(requirement.specifier, self._simplify, self._versionFormat)
1332 if specifier != "any":
1333 paragraph += nodes.Text(f" {specifier}")
1335 if self._dependencyFormat.ShowsLicense:
1336 paragraph += nodes.Text(" (")
1337 if (license := self._LicenseName(release)) is None:
1338 # the same statement the License column makes: what the index published, in italics, so it reads as
1339 # a quotation rather than as an identifier
1340 paragraph += nodes.emphasis(text=self._PublishedLicense(release))
1341 elif (licenseURL := self._LicenseURL(release)) is not None:
1342 paragraph += nodes.reference("", license, refuri=licenseURL)
1343 else:
1344 paragraph += nodes.Text(license)
1345 paragraph += nodes.Text(")")
1347 return paragraph
1350@export
1351def formatUnresolvedLicenses(unresolved: Mapping[str, tuple[str, ...]]) -> str:
1352 """
1353 Describe the packages needing a license override, grouped by what the package index published for them.
1355 Grouped rather than listed one per line, because one ambiguous statement usually accounts for most of the list:
1356 ``License :: OSI Approved :: BSD License`` names three licenses, so every package whose only license information
1357 is that classifier lands here for the same reason and is worth reading as one group.
1359 :param unresolved: Names of the packages needing an override, mapped to what the index published for them.
1360 :returns: The message, as one line naming the count and two lines per reason.
1361 """
1362 byReason: dict[tuple[str, ...], list[str]] = {}
1363 for packageName, published in sorted(unresolved.items()):
1364 byReason.setdefault(published, []).append(packageName)
1366 lines = [f"[dependency-table] {len(unresolved)} package(s) need a license override:"]
1368 # the biggest group first, so the statement to fix first is the one at the top
1369 for published, packageNames in sorted(byReason.items(), key=lambda item: (-len(item[1]), item[0])):
1370 reason = "; ".join(published) if len(published) > 0 else "the index published no license information"
1371 lines.append(f" {reason}")
1372 lines.append(f" {', '.join(packageNames)}")
1374 return "\n".join(lines)
1377def reportBuildTime(app: Sphinx, exception: Nullable[Exception]) -> None:
1378 """
1379 Report what querying the package index cost this build.
1381 The tables are fetched live, so this is the number to look at before deciding what a cache would be worth. The
1382 packages whose license had to be guessed at - or couldn't be - are named too, because that is the list the
1383 override file has to answer for.
1385 :param app: The Sphinx application that finished building.
1386 :param exception: The exception that ended the build, or ``None`` if it succeeded.
1387 """
1388 if (collector := _COLLECTORS.pop(id(app), None)) is None or collector.RequestCount == 0: 1388 ↛ 1391line 1388 didn't jump to line 1391 because the condition on line 1388 was always true
1389 return
1391 _logger.info(
1392 f"[dependency-table] {collector.RequestCount} request(s) to the package index, "
1393 f"{collector.Seconds:.2f} s in total."
1394 )
1396 if len(unresolved := collector.UnresolvedLicenses) > 0:
1397 _logger.warning(formatUnresolvedLicenses(unresolved))