Coverage for pyTooling/Dependency/Python.py: 85%
620 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-03 23:02 +0000
1# ==================================================================================================================== #
2# _____ _ _ ____ _ #
3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ | _ \ ___ _ __ ___ _ __ __| | ___ _ __ ___ _ _ #
4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | | | |/ _ \ '_ \ / _ \ '_ \ / _` |/ _ \ '_ \ / __| | | | #
5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| |_| | __/ |_) | __/ | | | (_| | __/ | | | (__| |_| | #
6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/ \___| .__/ \___|_| |_|\__,_|\___|_| |_|\___|\__, | #
7# |_| |___/ |___/ |_| |___/ #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2025-2026 Patrick Lehmann - Bötzingen, Germany #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""
32Implementation of package dependencies.
34Importing this module needs the ``pypi`` extra, because it reads a package index over HTTP and parses PEP 440
35requirements:
37* :mod:`aiohttp`,
38* :mod:`packaging` and
39* :mod:`requests`
41are imported at module level and each is guarded, so a missing one names itself rather than failing as a bare
42:exc:`ImportError`.
44:raises MissingDependencyError: If the 'pypi' extra isn't installed.
46.. hint::
48 See :ref:`high-level help <DEPENDENCIES>` for explanations and usage examples.
49"""
50from __future__ import annotations
52from asyncio import run as asyncio_run, gather as asyncio_gather
53from collections import deque
54from datetime import date, datetime
55from enum import IntEnum
56from functools import wraps, update_wrapper
57from pathlib import Path
58from re import compile as re_compile, Pattern
59from threading import RLock
60from typing import Any, ClassVar, Deque, Optional as Nullable, Union, Iterable, Iterator, Mapping, Self
62from pyTooling.Configuration import Dictionary
63from pyTooling.Exceptions import MissingDependencyError
65try:
66 from aiohttp import ClientSession
67except ImportError as ex: # pragma: no cover
68 raise MissingDependencyError(dependency="aiohttp", extra="pypi") from ex
70try:
71 from packaging.requirements import InvalidRequirement, Requirement
72 from packaging.utils import canonicalize_name
73except ImportError as ex: # pragma: no cover
74 raise MissingDependencyError(dependency="packaging", extra="pypi") from ex
76try:
77 from requests import Session, HTTPError
78 from requests.adapters import HTTPAdapter
79 from urllib3.util.retry import Retry
80except ImportError as ex: # pragma: no cover
81 raise MissingDependencyError(dependency="requests", extra="pypi") from ex
83from pyTooling.Decorators import export, readonly
84from pyTooling.MetaClasses import ExtendedType, abstractmethod
85from pyTooling.Common import getFullyQualifiedName, firstValue
86from pyTooling.Dependency import Package, PackageStorage, PackageVersion, PackageDependencyGraph
87from pyTooling.Dependency import BrokenRequirementWarning, ReleaseDetailsWarning, UnknownLicenseWarning
88from pyTooling.Dependency import ProjectNotFoundError, DependencyError, NoSessionAvailableError
89from pyTooling.Dependency import ReleaseNotFoundError, CircularRequirementsFileError, RequirementsFileNotFoundError
90from pyTooling.Licensing import LicenseExpression, LicenseExpressionError, LICENSES_BY_CLASSIFIER
91from pyTooling.Licensing import LicenseAbsence, ProprietaryLicense, UnknownLicense
92from pyTooling.Warning import WarningCollector
93from pyTooling.GenericPath.URL import URL
94from pyTooling.Versioning import Parts, PythonVersion, PythonVersionExpression, SemanticVersion
97#: Longest prefix of a free-text ``license`` field quoted in a warning note, so a full license text doesn't flood it.
98_LICENSE_NOTE_LENGTH = 64
100#: PyPI's classifier for a license that isn't open source. SPDX can't name one, so it becomes a
101#: :class:`~pyTooling.Licensing.ProprietaryLicense` rather than an expression to parse.
102_PROPRIETARY_CLASSIFIER = "License :: Other/Proprietary License"
104#: Aliases matched against the free-text keys of ``project_urls``, lower-cased, most specific first.
105_REPOSITORY_URL_ALIASES = ("source code", "source", "code", "repository", "github", "gitlab")
106_DOCUMENTATION_URL_ALIASES = ("documentation", "docs", "read the docs")
107_ISSUE_TRACKER_URL_ALIASES = ("bug tracker", "issue tracker", "issues", "bug reports", "tracker")
108_PROJECT_URL_ALIASES = ("homepage", "home page", "home")
109_CHANGELOG_URL_ALIASES = ("changelog", "changes", "release notes", "whatsnew", "what's new")
111#: Pattern of an ``extra == "<name>"`` comparison in a requirement's marker.
112_EXTRA_MARKER = re_compile(r'''extra\s*==\s*["']([^"']+)["']''')
114#: How often a request to a package index is retried before it is reported as an error.
115RETRY_ATTEMPTS = 4
117#: Seconds the delay between two retries grows by, doubling per attempt: 0.5 s, 1 s, 2 s, 4 s.
118RETRY_BACKOFF = 0.5
120#: Status codes worth retrying: a rate-limit and the transient server-side failures.
121RETRY_STATUS_CODES = (429, 500, 502, 503, 504)
124@export
125class RequirementsFile(metaclass=ExtendedType, slots=True):
126 """
127 A ``requirements.txt`` file, together with the files it references.
129 A ``-r other.txt`` line references another file, and the tree of those references is kept rather than flattened:
130 ``tests/requirements.txt`` is nothing but four ``-r`` lines, so a flattened list would say nothing about which of
131 them a package came from - which is exactly what a table per entrypoint has to show.
133 It is a **tree**, so every file knows where it sits in one: :attr:`Parent` is the file that referenced it and
134 :attr:`Root` the entrypoint the whole tree was read from. The root alone keeps
135 :attr:`AnalyzedRequirementFiles`, every file of the tree by its resolved path, which is both the answer to
136 *where did this package come from* and what makes a cycle detectable.
138 **A cycle raises.** A file referencing itself, directly or through a chain, is a statement nobody wrote on
139 purpose; reading it once and continuing would hide it.
140 """
142 _root: RequirementsFile #: Entrypoint this tree was read from.
143 _parent: Nullable[RequirementsFile] #: Referencing file; ``None`` for a root.
144 _path: Path #: Path of this requirements file.
145 _entries: list[Union[Requirement, RequirementsFile]] #: What this file states, in file order.
146 _analyzedRequirementFiles: Nullable[dict[Path, RequirementsFile]] #: Every file of the tree, by resolved path.
148 def __init__(self, path: Path, parent: Nullable[RequirementsFile] = None) -> None:
149 """
150 Read a requirements file and the files it references.
152 :param path: Path of the requirements file to read.
153 :param parent: Optional, the file referencing this one. Default: ``None``, a root.
154 :raises TypeError: If parameter 'path' is not of type :class:`~pathlib.Path`.
155 :raises TypeError: If parameter 'parent' is not of type :class:`RequirementsFile`.
156 :raises RequirementsFileNotFoundError: If the requirements file doesn't exist.
157 :raises CircularRequirementsFileError: If a ``-r`` line references a file already being read.
158 """
159 if not isinstance(path, Path): 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true
160 ex = TypeError("Parameter 'path' is not of type 'Path'.")
161 ex.add_note(f"Got type '{getFullyQualifiedName(path)}'.")
162 raise ex
163 elif not path.exists():
164 raise RequirementsFileNotFoundError(f"Requirements file '{path}' does not exist.") from FileNotFoundError(path)
166 if parent is not None and not isinstance(parent, RequirementsFile):
167 ex = TypeError("Parameter 'parent' is not of type 'RequirementsFile'.")
168 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")
169 raise ex
171 self._path = path
172 self._parent = parent
173 self._root = self if parent is None else parent._root
174 self._entries = []
176 # resolved as the mapping's key only, not as '_path': the key is compared against a '-r' reference, which is
177 # resolved below, so a file reachable under two spellings has to arrive here as one
178 if parent is None:
179 self._analyzedRequirementFiles = {
180 path.resolve(): self
181 }
182 else:
183 self._analyzedRequirementFiles = None
184 self._root._analyzedRequirementFiles[path.resolve()] = self
186 lines = path.read_text(encoding="utf-8").splitlines()
187 for lineNumber, line in enumerate(lines, start=1):
188 if (line := line.split("#")[0].strip()) == "":
189 continue
191 if line.startswith("-r"):
192 referenced = (path.parent / line[2:].strip()).resolve()
194 if referenced in self._root._analyzedRequirementFiles:
195 chain = " → ".join(str(file.Path) for file in self.Hierarchy)
196 ex = CircularRequirementsFileError(
197 f"Requirements file '{referenced}' referenced in '{path}' line {lineNumber} is already being read."
198 )
199 ex.add_note(f"Chain: {chain} → {referenced}")
200 raise ex
202 self._entries.append(RequirementsFile(referenced, self))
203 continue
205 # '--index-url', '-e' and a bare URL are instructions to the installer, not requirements
206 if line.startswith("-") or line.startswith("http"):
207 continue
209 try:
210 self._entries.append(Requirement(line))
211 except InvalidRequirement as ex:
212 WarningCollector.Raise(
213 BrokenRequirementWarning(f"Requirement '{line}' in '{path}' line {lineNumber} can't be parsed."),
214 ex
215 )
217 @readonly
218 def Path(self) -> Path:
219 """
220 Read-only property to access this requirement file's path (:attr:`_path`).
222 :returns: Path of the requirements file, spelled the way it was handed in.
223 """
224 return self._path
226 @readonly
227 def Root(self) -> RequirementsFile:
228 """
229 Read-only property to access the entrypoint this tree was read from (:attr:`_root`).
231 :returns: The tree's root, which is this file itself when it is one.
232 """
233 return self._root
235 @readonly
236 def Parent(self) -> Nullable[RequirementsFile]:
237 """
238 Read-only property to access the file whose ``-r`` line referenced this one (:attr:`_parent`).
240 :returns: The referencing file, or ``None`` for a root.
241 """
242 return self._parent
244 @readonly
245 def Hierarchy(self) -> tuple[RequirementsFile, ...]:
246 """
247 Read-only property to return the path from the root down to this file as a tuple.
249 :returns: A tuple of requirements files.
250 """
251 hierarchy: Deque[RequirementsFile] = deque([self])
252 parentRequirementsFile: Nullable[RequirementsFile] = self
253 while (parentRequirementsFile := parentRequirementsFile._parent) is not None:
254 hierarchy.appendleft(parentRequirementsFile)
256 return tuple(hierarchy)
258 @readonly
259 def AnalyzedRequirementFiles(self) -> Nullable[dict[Path, RequirementsFile]]:
260 """
261 Read-only property to access every file of this tree, by its resolved path.
263 **Only the root's is filled**; ask :attr:`Root` for it. It is what detects a cycle while reading, and what
264 answers which files a tree was read from afterward - the list a documentation build registers so a change
265 to any of them rebuilds the page.
267 :returns: Every file of the tree by resolved path, or ``None`` for a referenced file.
268 """
269 return self._analyzedRequirementFiles
271 @readonly
272 def Entries(self) -> list[Union[Requirement, RequirementsFile]]:
273 """
274 Read-only property to access what this file states, in the order it states it (:attr:`_entries`).
276 Requirements and referenced files are kept in one list, because a file states them interleaved.
277 :attr:`Requirements` and :attr:`ReferencedFiles` are the two filtered views of it.
279 :returns: This file's requirements and referenced files.
280 """
281 return self._entries
283 @readonly
284 def Requirements(self) -> Iterator[Requirement]:
285 """
286 Read-only property to iterate the requirements stated in this file.
288 This is what *this* file states; what the files it references state is reachable through
289 :attr:`ReferencedFiles`.
291 :returns: An iterator of this file's requirements, in the order they are written.
292 """
293 return (entry for entry in self._entries if isinstance(entry, Requirement))
295 @readonly
296 def ReferencedFiles(self) -> Iterator[RequirementsFile]:
297 """
298 Read-only property to iterate the files referenced with ``-r``.
300 :returns: An iterator of the referenced files, in the order they are referenced.
301 """
302 return (entry for entry in self._entries if isinstance(entry, RequirementsFile))
304 def IterateTree(self) -> Iterator[RequirementsFile]:
305 """
306 Iterate this file and every file it references, depth first.
308 :returns: A generator of requirements files, this one first.
309 """
310 yield self
311 for referenced in self.ReferencedFiles:
312 yield from referenced.IterateTree()
314 @readonly
315 def AllRequirements(self) -> Iterator[Requirement]:
316 """
317 Read-only property to iterate the requirements of this file and of every file it references.
319 **The file's order is kept**, and **the nearer statement wins**: a requirement stated in this file overrides
320 the same package required by a referenced file, wherever the two stand, because that is the constraint the
321 entrypoint was written for. Overriding keeps the position, so every package is yielded exactly once.
323 .. code-block:: text
325 # base.txt # requirements.txt AllRequirements
326 pytest ~= 8.0 pytest ~= 9.1 pytest ~= 9.1
327 sphinx ~= 9.1 -r base.txt sphinx ~= 9.1
328 colorama ~= 0.4.6 colorama ~= 0.4.6
330 :returns: A generator of requirements, deduplicated by canonical package name, in the order stated.
331 """
332 requirements: dict[str, Requirement] = {}
333 stated: dict[str, Requirement] = {}
335 for entry in self._entries:
336 if isinstance(entry, RequirementsFile):
337 for requirement in entry.AllRequirements:
338 requirements[canonicalize_name(requirement.name)] = requirement
339 else:
340 name = canonicalize_name(entry.name)
341 requirements[name] = stated[name] = entry
343 # a key keeps the position of its first insertion, so this overrides the value without moving the package
344 requirements.update(stated)
346 yield from requirements.values()
348 def __len__(self) -> int:
349 """
350 Return the number of requirements this file states, not counting the files it references.
352 :returns: Number of requirements stated in this file.
353 """
354 return sum(1 for _ in self.Requirements)
356 def __iter__(self) -> Iterator[Requirement]:
357 """
358 Iterate the requirements this file states, not the ones it references.
360 :returns: An iterator of this file's requirements.
361 """
362 return self.Requirements
364 def __str__(self) -> str:
365 """
366 Return this file's path and how much it states.
368 :returns: A string representation of this requirements file.
369 """
370 referenced = sum(1 for _ in self.ReferencedFiles)
372 return f"{self._path}: {len(self)} requirement(s), {referenced} referenced file(s)"
375@export
376class LicenseOverrides(metaclass=ExtendedType, slots=True):
377 """
378 Licenses stated by hand, for the packages a package index can't answer for.
380 A package index is not a reliable source of license information: roughly half of a typical dependency set
381 publishes a PEP 639 ``license_expression``, some state a license *name* where an identifier is expected, and the
382 classifier ``License :: OSI Approved :: BSD License`` means either ``BSD-2-Clause`` or ``BSD-3-Clause`` with no
383 way to tell which. Those packages are answered here instead of guessed at.
385 The file is YAML, and a license may be stated for the package as a whole or per version - a package that
386 relicensed has one license before the switch and another after it:
388 .. code-block:: yaml
390 version: "0.1"
391 analysedAt: 2026-09-02
393 packages:
394 colorama:
395 license: BSD-3-Clause
396 licenseURL: https://GitHub.com/tartley/colorama/blob/master/LICENSE.txt
397 repository: https://GitHub.com/tartley/colorama
398 "igraph >=0.10":
399 license: GPL-2.0-or-later
400 "igraph <0.10":
401 license: GPL-2.0-only
402 "igraph 0.9.10":
403 license: GPL-2.0-only
405 ``version`` states the structure this file is written for and is checked against
406 :attr:`SCHEMA_VERSION`; ``analysedAt`` is the day a human last checked the statements - see :attr:`AnalysedAt`.
408 **A key is a package name, optionally followed by a version expression** - the shape a requirement line has.
409 The expression is read by :class:`~pyTooling.Versioning.PythonVersionExpression`, so it is the same language a
410 requirement file writes, and two of its rules are what make one key form enough:
412 * **a bare version is an equality**, so ``igraph 0.9.10`` and ``igraph ==0.9.10`` state the same thing, and
413 * **an expression with no constraints matches every version**, so ``igraph`` on its own is the statement for
414 every version rather than a special case.
416 Keys are matched in the order the file writes them and the first one a version satisfies wins, so a narrower
417 statement goes above the bare name it refines. Asking without a version answers from the bare-name key only.
419 A key states a whole entry, so ``licenseURL`` and ``repository`` can differ per version too - a project that
420 moved forge between releases has two ``repository`` statements and no special case for it.
421 """
423 #: Structure this parser reads. A file states the one it was written for as its ``version`` field, and a file
424 #: stating a different one is rejected rather than read on the chance that it still fits.
425 SCHEMA_VERSION: ClassVar[SemanticVersion] = SemanticVersion(0, 1)
427 #: Splits a key into the package name and whatever follows it. A name stops at the first character an operator
428 #: can start with, so ``igraph>=0.10`` splits the same way ``igraph >=0.10`` does.
429 _PACKAGE_KEY: ClassVar[Pattern[str]] = re_compile(r"^\s*(?P<name>[^\s<>=!~]+)\s*(?P<expression>.*?)\s*$")
431 _analysedAt: Nullable[datetime] #: When the statements were last checked by a human.
432 #: License expression per package, by the version expression its key states, in the file's order.
433 _licenses: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]
434 #: URL of the license's text per package, by the version expression its key states, in the file's order.
435 _licenseURLs: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]
436 #: URL of the source repository per package, by the version expression its key states, in the file's order.
437 _repositories: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]
439 def __init__(self, analysedAt: Nullable[datetime] = None) -> None:
440 """
441 Initialize an empty set of overrides.
443 :param analysedAt: Optional, the day these statements were last checked by a human.
444 """
445 self._analysedAt = analysedAt
446 self._licenses = {}
447 self._licenseURLs = {}
448 self._repositories = {}
450 @readonly
451 def AnalysedAt(self) -> Nullable[datetime]:
452 """
453 Read-only property to access when these statements were last checked (:attr:`_analysedAt`).
455 A package index answers for itself every time it is asked, so what it says is as old as the request. These
456 statements are written by hand and are as old as whoever last looked, which nothing else records - so a
457 report can mark an entry *overridden*, and *stale* once this date is far enough back.
459 :meth:`FromFile` requires it; :meth:`FromDictionary` doesn't, because overrides assembled in code are as old
460 as the code.
462 :returns: The day of the last analysis, or ``None`` if the overrides were built without one.
463 """
464 return self._analysedAt
466 @classmethod
467 def FromFile(cls, path: Path) -> Self:
468 """
469 Read overrides from a YAML file.
471 The file is read through :class:`pyTooling.Configuration.YAML.Configuration`, and the ``packages`` node is
472 handed to :meth:`FromDictionary` **as it is** - a configuration node answers ``items()`` and ``get()``, so
473 there is nothing to convert and no second constructor for the node tree.
475 ``version`` is **required** and is the first thing checked: it states which structure the file was written
476 for, and is compared against :attr:`SCHEMA_VERSION`. **Quote it** - unquoted, YAML reads ``0.1`` as a float,
477 and a float loses a trailing zero, so ``1.10`` would arrive as ``1.1``.
479 ``analysedAt`` is **required** too, and is the day a human last checked these statements. It is an ISO-8601
480 date, written either way round - ``analysedAt: 2026-09-02`` reads as YAML's own date type, and
481 ``analysedAt: "2026-09-02"`` as a string. A configuration hands both over in the same ISO-8601 spelling.
483 :param path: Path of the YAML file to read.
484 :returns: The overrides the file states.
485 :raises MissingDependencyError: If ``ruamel.yaml`` isn't installed.
486 :raises FileNotFoundError: If the file doesn't exist.
487 :raises ConfigurationError: If the file isn't a YAML document describing a mapping. |br|
488 An empty file is one, and states no overrides - but it states no
489 ``version`` either, so it is rejected by the next check.
490 :raises DependencyError: If ``version`` is missing, isn't a version, or isn't
491 :attr:`SCHEMA_VERSION`.
492 :raises DependencyError: If ``analysedAt`` is missing or isn't an ISO-8601 date.
493 :raises DependencyError: If ``packages`` isn't a mapping. |br|
494 A file stating no packages is fine and gives no overrides.
495 :raises DependencyError: If a version expression in the file can't be parsed.
496 """
497 # Imported here rather than at module level, so a missing 'ruamel.yaml' is reported when the overrides are
498 # read instead of when 'pyTooling.Dependency.Python' is imported.
499 from pyTooling.Configuration.YAML import Configuration
501 # 'Configuration' reports a missing file too, as a 'ConfigurationError' naming the *format*. This says which
502 # file of ours is missing, and is what the signature promises, so it stays.
503 if not path.exists():
504 raise FileNotFoundError(f"License override file '{path}' not found.")
506 configuration = Configuration(path)
508 if (schemaVersion := configuration.get("version", None)) is None:
509 ex = DependencyError(f"License override file '{path}' states no 'version'.")
510 ex.add_note("It says which structure the file is written for, so a later one can be told apart.")
511 ex.add_note(f'Add it as the first field: version: "{cls.SCHEMA_VERSION}"')
512 raise ex
514 try:
515 fileVersion = SemanticVersion.Parse(str(schemaVersion))
516 except ValueError as cause:
517 ex = DependencyError(f"License override file '{path}' states a 'version' that isn't a version number.")
518 ex.add_note(f"Got '{schemaVersion}'.")
519 raise ex from cause
521 if fileVersion != cls.SCHEMA_VERSION:
522 ex = DependencyError(f"License override file '{path}' is written for structure '{fileVersion}'.")
523 ex.add_note(f"This reads '{cls.SCHEMA_VERSION}'.")
524 raise ex
526 if (analysedMoment := configuration.get("analysedAt", None)) is None:
527 ex = DependencyError(f"License override file '{path}' states no 'analysedAt' timestamp.")
528 ex.add_note("These statements are written by hand, so nothing else records how old they are.")
529 ex.add_note("Add an ISO-8601 timestamp, for example: analysedAt: 2026-09-04T21:45:00+00:00")
530 raise ex
532 try:
533 analysedAt = datetime.fromisoformat(str(analysedMoment))
534 except ValueError as cause:
535 ex = DependencyError(
536 f"License override file '{path}' states an 'analysedAt' that isn't an ISO-8601 timestamp."
537 )
538 ex.add_note(f"Got '{analysedMoment}'.")
539 ex.add_note("Write it as an ISO-8601 timestamp: analysedAt: 2026-09-04T21:45:00+00:00")
540 raise ex from cause
542 packages = configuration.get("packages", None)
543 if packages is None:
544 return cls(analysedAt)
545 elif not isinstance(packages, Dictionary):
546 ex = DependencyError(f"License override file '{path}' states a 'packages' node that isn't a mapping.")
547 ex.add_note(f"Got '{packages}'.")
548 raise ex
550 return cls.FromDictionary(packages, analysedAt)
552 @classmethod
553 def FromDictionary(
554 cls,
555 packages: Union[Mapping[str, Any], Dictionary],
556 analysedAt: Nullable[datetime] = None
557 ) -> Self:
558 """
559 Build overrides from an already parsed mapping.
561 Keeping this apart from :meth:`FromFile` is what lets the overrides be assembled in code, and tested, without
562 a file and without YAML.
564 A :class:`~pyTooling.Configuration.Dictionary` is accepted beside a plain :class:`dict`, which is what lets
565 :meth:`FromFile` hand over a configuration node without flattening it first. Only ``items()`` and ``get()``
566 are read, so a configuration node of either backend fits - it is **not** a :class:`~typing.Mapping`, because
567 its bare iteration yields values rather than keys.
569 :param packages: Mapping of a package's name to what is stated for it.
570 :param analysedAt: Optional, the day these statements were last checked. :meth:`FromFile` requires one;
571 overrides assembled in code are as old as the code and need none.
572 :returns: The overrides the mapping states.
573 :raises DependencyError: If a version specifier can't be parsed.
574 """
575 overrides = cls(analysedAt)
577 # A configuration node types its values as the whole 'ValueT' union, which every '.get' below would then
578 # have to be narrowed against. What is read here is a document either way, so it is read as one.
579 statement: Any
580 for packageKey, statement in packages.items():
581 name, versionExpression = cls._SplitKey(str(packageKey))
583 for field, table in (
584 ("license", overrides._licenses),
585 ("licenseURL", overrides._licenseURLs),
586 ("repository", overrides._repositories),
587 ):
588 if (value := statement.get(field, None)) is not None:
589 table.setdefault(name, []).append((versionExpression, str(value)))
591 return overrides
593 @classmethod
594 def _SplitKey(cls, packageKey: str) -> tuple[str, PythonVersionExpression[SemanticVersion]]:
595 """
596 Split a key into the package it names and the version expression it restricts that package to.
598 A key is the shape a requirement line has - a name, then optionally a version expression, with or without a
599 space between them. A key naming only a package gives an expression with no constraints, which matches every
600 version.
602 :param packageKey: The key as the file writes it.
603 :returns: The canonical package name, and the version expression.
604 :raises DependencyError: If what follows the name isn't a version expression.
605 """
606 match = cls._PACKAGE_KEY.match(packageKey)
607 if match is None: # pragma: no cover
608 ex = DependencyError(f"Key '{packageKey}' doesn't name a package.")
609 raise ex
611 try:
612 versionExpression: PythonVersionExpression[SemanticVersion] = PythonVersionExpression.Parse(match["expression"])
613 except (LicenseExpressionError, ValueError) as cause:
614 ex = DependencyError(f"Key '{packageKey}' states a version expression that can't be parsed.")
615 ex.add_note(f"Got '{match['expression']}'.")
616 ex.add_note("A bare version is an equality, so 'igraph 0.10' and 'igraph ==0.10' state the same thing.")
617 raise ex from cause
619 return canonicalize_name(match["name"]), versionExpression
621 def LicenseOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:
622 """
623 Return the license expression stated for a package, or for one of its versions.
625 Keys are tried in the order the file writes them and the first one this version satisfies wins, so a
626 narrower statement answers before the bare name it refines.
628 :param packageName: Name of the package.
629 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.
630 :returns: The license expression stated, or ``None`` if the package isn't overridden.
631 """
632 return self._Lookup(self._licenses, packageName, version)
634 def LicenseURLOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:
635 """
636 Return the URL of a package's license text, where one was stated.
638 :param packageName: Name of the package.
639 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.
640 :returns: The URL stated, or ``None``.
641 """
642 return self._Lookup(self._licenseURLs, packageName, version)
644 @staticmethod
645 def _Lookup(
646 table: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]],
647 packageName: str,
648 version: Nullable[SemanticVersion]
649 ) -> Nullable[str]:
650 """
651 Return the first statement in one table whose key applies to a version.
653 :param table: One of the per-package tables.
654 :param packageName: Name of the package.
655 :param version: Optional, the version to answer for. Without one, only an unrestricted key answers.
656 :returns: What that key states, or ``None`` if none applies.
657 """
658 if (entries := table.get(canonicalize_name(packageName), None)) is None:
659 return None
661 for versionExpression, value in entries:
662 # Without a version, only a key that restricts nothing can be said to apply.
663 if version in versionExpression if version is not None else len(versionExpression) == 0:
664 return value
666 return None
668 def RepositoryOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:
669 """
670 Return the URL of a package's source repository, where one was stated.
672 :param packageName: Name of the package.
673 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.
674 :returns: The URL stated, or ``None``.
675 """
676 return self._Lookup(self._repositories, packageName, version)
678 def __len__(self) -> int:
679 """
680 Return the number of packages that are overridden.
682 :returns: Number of overridden packages.
683 """
684 return len(set(self._licenses) | set(self._licenseURLs) | set(self._repositories))
686 def __str__(self) -> str:
687 """
688 Return a string representation of these overrides.
690 :returns: The number of packages that are overridden.
691 """
692 return f"LicenseOverrides({len(self)} packages)"
695@export
696class LazyLoaderState(IntEnum):
697 """
698 Loading states of a lazy-loadable object, in the order they are reached.
700 The states are ordered, so a loader can be asked for *at least* a given state and does nothing when the object is
701 already loaded that far.
702 """
703 Uninitialized = 0 #: No data or minimal data like ID or name.
704 Initialized = 1 #: Initialized by some __init__ parameters.
705 PartiallyLoaded = 2 #: Some additional data was loaded.
706 FullyLoaded = 3 #: All data is loaded.
707 PostProcessed = 4 #: Loaded data triggered further processing.
710@export
711class lazy:
712 """
713 Unified decorator that supports:
714 1. @lazy(state) def method()
715 2. @lazy(state) @property def prop()
716 """
718 def __init__(self, _requiredState: LazyLoaderState = LazyLoaderState.PartiallyLoaded):
719 """
720 Initialize the decorator with the loading state its member needs.
722 :param _requiredState: Optional, state the object has to be loaded to before the decorated member is used.
723 """
724 self._requiredState = _requiredState
725 self._wrapped = None
727 def __call__(self, wrapped):
728 """
729 Apply the decorator to a method or property.
731 :param wrapped: The method or property to load lazily.
732 :returns: The decorator itself, which acts as the descriptor of the decorated member.
733 """
734 self._wrapped = wrapped
735 # If it's a function, we update metadata.
736 # If it's a property, it doesn't support update_wrapper directly.
737 if hasattr(wrapped, "__name__"):
738 update_wrapper(self, wrapped)
740 return self
742 def __get__(self, obj, objtype=None):
743 """
744 Load the object far enough, then hand out the decorated property's value or a bound method.
746 :param obj: The object the decorated member is accessed on, or ``None`` for a class access.
747 :param objtype: Optional, the class the decorated member is accessed on.
748 :returns: The property's value, a bound wrapper around the method, or the decorator itself.
749 """
750 if obj is None:
751 return self
753 # 1. Thread-safe state check
754 with obj.__lazy_lock__:
755 if obj.__lazy_state__ < self._requiredState:
756 obj.__lazy_loader__(self._requiredState)
758 # 2. Determine if we are wrapping a property or a method
759 if isinstance(self._wrapped, property):
760 # If it's a property, call its __get__ to return the value
761 return self._wrapped.__get__(obj, objtype)
763 # 3. Otherwise, treat as a method and return a bound wrapper
764 @wraps(self._wrapped)
765 def wrapper(*args, **kwargs):
766 """
767 Nested function binding the decorated method to the object it was accessed on.
769 :param args: Positional parameters passed to the decorated method.
770 :param kwargs: Named parameters passed to the decorated method.
771 :returns: Whatever the decorated method returns.
772 """
773 return self._wrapped(obj, *args, **kwargs)
775 return wrapper
778@export
779class LazyLoadableMixin(metaclass=ExtendedType, mixin=True):
780 """
781 Mixin-class for objects whose details are fetched on first use.
783 The object is created from what its creator knows - often little more than a name - and everything else is loaded
784 when it is needed. The mixin records how far the object is loaded (:attr:`__lazy_state__`) and serializes
785 concurrent loading (:attr:`__lazy_lock__`); the deriving class implements a ``__lazy_loader__`` method and decides what
786 loading means.
787 """
788 __lazy_state__: LazyLoaderState #: State of the lazy loading process for this object.
789 __lazy_lock__: RLock #: Lock serializing concurrent lazy loading of this object.
791 def __init__(self, targetLevel: LazyLoaderState = LazyLoaderState.Initialized) -> None:
792 """
793 Initialize the lazy-loading state of an object.
795 :param targetLevel: Optional, state the object should be loaded to immediately; by default nothing is loaded.
796 """
797 self.__lazy_state__ = LazyLoaderState.Initialized
798 self.__lazy_lock__ = RLock()
800 if targetLevel > self.__lazy_state__:
801 with self.__lazy_lock__:
802 self.__lazy_loader__(targetLevel)
804 @abstractmethod
805 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
806 """
807 Load the object's details up to the given state.
809 :param targetLevel: Optional, state the object needs to be loaded to.
810 """
811 pass
814@export
815class Distribution(metaclass=ExtendedType, slots=True):
816 """
817 A single downloadable file of a release - a wheel or a source archive.
818 """
819 _filename: str #: Filename of the distribution's file.
820 _url: URL #: URL to download the distribution's file from.
821 _uploadTime: datetime #: Time when the distribution was uploaded to the package index.
823 def __init__(self, filename: str, url: Union[str, URL], uploadTime: datetime) -> None:
824 """
825 Initialize a distribution with the data the package index reports for it.
827 :param filename: Filename of the distribution's file.
828 :param url: URL to download the file from, as a string or a parsed URL.
829 :param uploadTime: Time the distribution was uploaded to the package index.
830 :raises TypeError: If a parameter is not of the expected type.
831 """
832 if not isinstance(filename, str): 832 ↛ 833line 832 didn't jump to line 833 because the condition on line 832 was never true
833 ex = TypeError("Parameter 'filename' is not of type 'str'.")
834 ex.add_note(f"Got type '{getFullyQualifiedName(filename)}'.")
835 raise ex
837 self._filename = filename
839 if isinstance(url, str): 839 ↛ 841line 839 didn't jump to line 841 because the condition on line 839 was always true
840 url = URL.Parse(url)
841 elif not isinstance(url, URL):
842 ex = TypeError("Parameter 'url' is not of type 'URL'.")
843 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
844 raise ex
846 self._url = url
848 if not isinstance(uploadTime, datetime): 848 ↛ 849line 848 didn't jump to line 849 because the condition on line 848 was never true
849 ex = TypeError("Parameter 'uploadTime' is not of type 'str'.")
850 ex.add_note(f"Got type '{getFullyQualifiedName(uploadTime)}'.")
851 raise ex
853 self._uploadTime = uploadTime
855 @readonly
856 def Filename(self) -> str:
857 """
858 Read-only property to access the distribution's filename (:attr:`_filename`).
860 :returns: Filename of the distribution.
861 """
862 return self._filename
864 @readonly
865 def URL(self) -> URL:
866 """
867 Read-only property to access the URL this distribution can be downloaded from (:attr:`_url`).
869 :returns: Download URL of the distribution.
870 """
871 return self._url
873 @readonly
874 def UploadTime(self) -> datetime:
875 """
876 Read-only property to access the time this distribution was uploaded (:attr:`_uploadTime`).
878 :returns: Upload time of the distribution.
879 """
880 return self._uploadTime
882 def __repr__(self) -> str:
883 """
884 Return a detailed string representation of this distribution.
886 :returns: The distribution's filename, prefixed by its kind.
887 """
888 return f"Distribution: {self._filename}"
890 def __str__(self) -> str:
891 """
892 Return a string representation of this distribution.
894 :returns: The distribution's filename.
895 """
896 return f"{self._filename}"
899@export
900class Release(PackageVersion, LazyLoadableMixin):
901 """
902 One released version of a project on a Python package index.
904 A release knows its distributions (the files that can be downloaded) and its requirements, sorted into the extras
905 they belong to. Both are fetched from the index on first use.
906 """
907 _files: list[Distribution] #: Distributions (wheels, source archives) of this release.
908 _requirements: dict[Union[str, None], list[Requirement]] #: Requirements per extra; ``None`` collects the unconditional ones.
910 _api: Nullable[URL] #: URL of the package index's API, used to load the release's details.
911 _session: Nullable[Session] #: HTTP session reused for the API requests.
913 def __init__(
914 self,
915 version: PythonVersion,
916 timestamp: datetime,
917 files: Nullable[Iterable[Distribution]] = None,
918 requirements: Nullable[Mapping[str, list[Requirement]]] = None,
919 project: Nullable[Project] = None,
920 lazy: LazyLoaderState = LazyLoaderState.Initialized
921 ) -> None:
922 """
923 Initialize a release of a project.
925 The API endpoint and the HTTP session are taken from the project's package index, so a release created from a
926 project can fetch its own details.
928 :param version: Version number of this release.
929 :param timestamp: Time this version was released.
930 :param files: Optional, distributions of this release.
931 :param requirements: Optional, requirements of this release, by extra.
932 :param project: Optional, project this release belongs to.
933 :param lazy: Optional, state the release should be loaded to immediately.
934 """
935 if project is not None and (storage := project._storage) is not None: 935 ↛ 939line 935 didn't jump to line 939 because the condition on line 935 was always true
936 self._api = storage._api
937 self._session = storage._session
938 else:
939 self._api = None
940 self._session = None
942 super().__init__(version, project, timestamp)
943 LazyLoadableMixin.__init__(self, lazy)
945 self._files = [file for file in files] if files is not None else []
946 self._requirements = {k: v for k, v in requirements} if requirements is not None else {None: []}
948 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
949 """
950 Download the release's details and post-process them, as far as the target state demands.
952 :param targetLevel: Optional, state the release needs to be loaded to.
953 """
954 if targetLevel >= LazyLoaderState.PartiallyLoaded: 954 ↛ 957line 954 didn't jump to line 957 because the condition on line 954 was always true
955 self.DownloadDetails()
957 if targetLevel >= LazyLoaderState.PostProcessed: 957 ↛ 958line 957 didn't jump to line 958 because the condition on line 957 was never true
958 self.PostProcess()
960 @lazy(LazyLoaderState.PostProcessed)
961 @PackageVersion.DependsOn.getter
962 def DependsOn(self) -> dict[Package, dict[SemanticVersion, PackageVersion]]:
963 """
964 Read-only property to access the packages this release depends on.
966 :returns: Dictionary of packages and their versions this release depends on.
967 """
968 return super().DependsOn
970 @readonly
971 def Project(self) -> Project:
972 """
973 Read-only property to access the project this release belongs to (:attr:`_package`).
975 :returns: The project this release belongs to.
976 """
977 return self._package
979 @lazy(LazyLoaderState.PartiallyLoaded)
980 @readonly
981 def Files(self) -> list[Distribution]:
982 """
983 Read-only property to access the distributions published for this release (:attr:`_files`).
985 :returns: List of distributions.
986 """
987 return self._files
989 @lazy(LazyLoaderState.PartiallyLoaded)
990 @readonly
991 def Requirements(self) -> dict[str, list[Requirement]]:
992 """
993 Read-only property to access the release's requirements, grouped by extra (:attr:`_requirements`).
995 :returns: Dictionary of extras and their requirements. Requirements without an extra are stored under ``None``.
996 """
997 return self._requirements
999 def _GetPyPIEndpoint(self) -> str:
1000 """
1001 Return the API endpoint describing this release.
1003 :returns: The endpoint's path, relative to the index's API URL.
1004 """
1005 return f"{self._package._name.lower()}/{self._version}/json"
1007 def DownloadDetails(self) -> None:
1008 """
1009 Download this release's details from the package index and load the projects it requires.
1011 :raises NoSessionAvailableError: If the release wasn't created by a package index, so it has no session. |br|
1012 A session is opened by the package index and handed to the objects it
1013 creates.
1014 :raises ReleaseNotFoundError: If the index doesn't know this release.
1015 """
1016 if self._session is None: 1016 ↛ 1017line 1016 didn't jump to line 1017 because the condition on line 1016 was never true
1017 ex = NoSessionAvailableError(f"No session available to download release '{self._version}' of package '{self._package._name}'.")
1018 ex.add_note("A session is opened by the package index and handed to the objects it creates.")
1019 raise ex
1021 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
1022 try:
1023 response.raise_for_status()
1024 except HTTPError as ex:
1025 if ex.response is not None and ex.response.status_code == 404:
1026 raise ReleaseNotFoundError(f"Release '{self._version}' of package '{self._package._name}' not found.") from ex
1028 raise ex
1030 self.UpdateDetailsFromPyPIJSON(response.json())
1032 index: PythonPackageIndex = self._package._storage
1033 for requirement in self._requirements[None]:
1034 packageName = requirement.name
1035 index.DownloadProject(packageName, True)
1037 def UpdateDetailsFromPyPIJSON(self, json) -> None:
1038 """
1039 Fill this release from the JSON document the package index returned.
1041 The requirements are sorted into the extras they belong to. A requirement lands under ``None`` when it has no
1042 marker at all, and also when its marker conditions it on the *environment* rather than on an extra -
1043 ``importlib-resources; python_version < "3.7"`` is required unconditionally, just not everywhere. A
1044 requirement naming an extra the release does not declare is reported as a :class:`BrokenRequirementWarning`.
1046 Metadata older than core-metadata 2.1 has no ``provides_extra`` field. Its extras are recovered from the
1047 markers naming them, because dropping every conditional requirement of an old release would empty exactly the
1048 releases a version-aware dependency graph is built to look at. A declared extra keeps the spelling it was
1049 declared with; a recovered one has no such spelling, so it keeps the canonical one - ``theme_furo`` written in
1050 a marker is recovered as ``theme-furo``.
1052 :param json: The parsed JSON document describing this release.
1053 """
1054 infoNode = json["info"]
1055 self._ResolveLicense(infoNode)
1056 self._ResolveURLs(infoNode)
1058 requirements = [Requirement(requirement) for requirement in (infoNode["requires_dist"] or ())]
1060 # The declared spelling is kept as the key, while the canonical one - 'code_style' and 'code-style' are the
1061 # same extra - is what a marker is matched against.
1062 if (declaredExtras := infoNode["provides_extra"]) is not None:
1063 extras = {canonicalize_name(extra): extra for extra in declaredExtras}
1064 else:
1065 extras = {}
1066 for requirement in requirements:
1067 if requirement.marker is not None:
1068 for name in _EXTRA_MARKER.findall(str(requirement.marker)):
1069 extras.setdefault(canonicalize_name(name), name)
1071 self._requirements = {extra: [] for extra in extras.values()}
1072 self._requirements[None] = []
1074 if len(requirements) > 0:
1075 brokenRequirements = []
1076 for req in requirements:
1077 # A marker naming no extra conditions the requirement on the interpreter or the platform, so the
1078 # requirement is unconditional as far as the extras are concerned.
1079 if req.marker is None or len(namedExtras := _EXTRA_MARKER.findall(str(req.marker))) == 0:
1080 self._requirements[None].append(req)
1081 continue
1083 for name in namedExtras:
1084 if (extra := extras.get(canonicalize_name(name))) is not None:
1085 self._requirements[extra].append(req)
1086 break
1087 else:
1088 brokenRequirements.append(req)
1090 if len(brokenRequirements) > 0:
1091 WarningCollector.Raise(
1092 BrokenRequirementWarning(f"Package '{self._package._name}' has {len(brokenRequirements)} requirement(s) whose marker matches no declared extra."),
1093 notes=[f"Broken requirement: {req}" for req in brokenRequirements]
1094 )
1095 # Preserving the broken requirements under the special index 0 makes 'Requirements' a dictionary of mixed
1096 # key types (str, None and int), which no consumer expects.
1097 # self._requirements[0] = brokenRequirements
1099 self.__lazy_state__ = LazyLoaderState.FullyLoaded
1101 def _ResolveLicense(self, infoNode: Mapping[str, Any]) -> None:
1102 """
1103 Resolve this release's license from what was published, and from what was stated by hand.
1105 The sources are consulted in order of how much they can be trusted, and the first one that answers wins:
1107 1. the :class:`LicenseOverrides` of the package index - an explicit statement always wins,
1108 2. ``license_expression``, the PEP 639 field, which is an SPDX expression by definition,
1109 3. ``license``, the legacy free-text field - it is handed to the parser like any other candidate, and a field
1110 holding the license's full text simply doesn't parse,
1111 4. a license classifier, but only when it means exactly one license - ``License :: OSI Approved :: BSD
1112 License`` means either ``BSD-2-Clause`` or ``BSD-3-Clause`` and is never guessed at.
1114 ``License :: Other/Proprietary License`` is the classifier that resolves without parsing anything: SPDX has
1115 no identifier for a license that isn't published, so it becomes a
1116 :class:`~pyTooling.Licensing.ProprietaryLicense` and the classifier itself is what
1117 :attr:`~pyTooling.Dependency.PackageVersion.PublishedLicense` reports.
1119 Whatever was found is parsed into a :class:`~pyTooling.Licensing.LicenseExpression` and kept verbatim in
1120 :attr:`~pyTooling.Dependency.PackageVersion.PublishedLicense`, even when it doesn't parse. A release whose
1121 license stays unresolved is reported as an
1122 :class:`~pyTooling.Dependency.UnknownLicenseWarning` naming what was published, because that is the list of
1123 packages the override file has to answer for. A release publishing ``NOASSERTION`` or ``NONE`` is on that
1124 list too - the *statement* resolved, the license is still unknown.
1126 :param infoNode: The ``info`` node of the JSON document describing this release.
1127 """
1128 index: PythonPackageIndex = self._package._storage
1129 overrides = index._licenseOverrides
1130 published = []
1131 candidates = []
1132 proprietaryClassifier = None
1134 if (override := overrides.LicenseOf(self._package._name, self._version)) is not None:
1135 candidates.append(override)
1136 else:
1137 if (licenseExpression := infoNode.get("license_expression", None)) is not None:
1138 published.append(f"license_expression: {licenseExpression}")
1139 candidates.append(licenseExpression)
1140 elif (licenseText := (infoNode.get("license", None) or "").strip()) != "":
1141 published.append(f"license: {licenseText[:_LICENSE_NOTE_LENGTH]}")
1142 candidates.append(licenseText)
1144 for classifier in infoNode.get("classifiers", None) or ():
1145 if not classifier.startswith("License ::"):
1146 continue
1148 published.append(f"classifier: {classifier}")
1149 if classifier == _PROPRIETARY_CLASSIFIER:
1150 proprietaryClassifier = classifier
1151 elif len(matches := LICENSES_BY_CLASSIFIER.get(classifier, ())) == 1:
1152 candidates.append(matches[0].SPDXIdentifier)
1154 break
1156 for candidate in candidates:
1157 try:
1158 self._licenseExpression = LicenseExpression.Parse(candidate)
1159 except (LicenseExpressionError, ValueError):
1160 continue
1162 # The expression keeps the candidate as its 'OriginalText', which is the only place it is held.
1163 break
1164 else:
1165 if proprietaryClassifier is not None:
1166 # SPDX has no identifier for a proprietary license, so there is nothing to parse - the node is built,
1167 # and it carries the classifier it was built from.
1168 self._licenseExpression = ProprietaryLicense(originalText=proprietaryClassifier)
1169 else:
1170 # Nothing resolved. 'NOASSERTION' is what SPDX says for that, and the node keeps what was published.
1171 self._licenseExpression = UnknownLicense(
1172 LicenseAbsence.NoAssertion,
1173 candidates[0] if len(candidates) > 0 else ""
1174 )
1176 if (licenseURL := overrides.LicenseURLOf(self._package._name)) is not None:
1177 self._licenseURL = URL.Parse(licenseURL)
1179 # 'UnknownLicense' covers both: the index said 'NOASSERTION' itself, and nothing resolved at all. Either way
1180 # the license is unknown, which is what this list is for.
1181 if isinstance(self._licenseExpression, UnknownLicense):
1182 WarningCollector.Raise(
1183 UnknownLicenseWarning(
1184 f"License of '{self._package._name}' {self._version} couldn't be resolved."
1185 ),
1186 notes=published if len(published) > 0 else ["The package index published no license information."]
1187 )
1189 def _ResolveURLs(self, infoNode: Mapping[str, Any]) -> None:
1190 """
1191 Resolve this release's project URLs from what the package index published.
1193 ``project_urls`` is a free-text mapping - one project writes ``Source``, another ``Source Code``,
1194 ``Repository`` or ``GitHub`` for the same thing - so its keys are matched case-insensitively against a list of
1195 aliases per URL and the first alias present wins. ``home_page`` is the fallback for the homepage, and the
1196 homepage is the last resort for the repository, which is what the project-level resolver used to do.
1198 These are resolved per release rather than per package because they move: a project migrating from Google
1199 Code or SourceForge to GitHub has one repository URL before the migration and another after it.
1200 :attr:`~pyTooling.Dependency.Package.RepositoryURL` and its siblings mirror the latest release, so the
1201 package still answers for the current state.
1203 :param infoNode: The ``info`` node of the JSON document describing this release.
1204 """
1205 projectURLs = {
1206 str(key).strip().lower(): value
1207 for key, value in (infoNode.get("project_urls", None) or {}).items()
1208 if value is not None
1209 }
1211 def urlFor(aliases: tuple[str, ...]) -> Nullable[URL]:
1212 """
1213 Return the first of the given aliases the package index published a URL for.
1215 :param aliases: The keys to look for, most specific first.
1216 :returns: The URL of the first alias that is present, or ``None`` if none of them is.
1217 """
1218 for alias in aliases:
1219 if (url := projectURLs.get(alias, None)) is not None:
1220 return URL.Parse(url)
1222 return None
1224 self._documentationURL = urlFor(_DOCUMENTATION_URL_ALIASES)
1225 self._issueTrackerURL = urlFor(_ISSUE_TRACKER_URL_ALIASES)
1226 self._changelogURL = urlFor(_CHANGELOG_URL_ALIASES)
1227 self._projectURL = urlFor(_PROJECT_URL_ALIASES)
1229 if self._projectURL is None and (homePage := infoNode.get("home_page", None)) is not None:
1230 self._projectURL = URL.Parse(homePage)
1232 index: PythonPackageIndex = self._package._storage
1233 if (repository := index._licenseOverrides.RepositoryOf(self._package._name)) is not None:
1234 self._repositoryURL = URL.Parse(repository)
1235 elif (repositoryURL := urlFor(_REPOSITORY_URL_ALIASES)) is not None:
1236 self._repositoryURL = repositoryURL
1237 else:
1238 self._repositoryURL = self._projectURL
1240 def PostProcess(self) -> None:
1241 """
1242 Resolve this release's requirements into dependencies on concrete releases.
1244 Every required project is downloaded and the releases matching the requirement's specifier are attached as
1245 dependencies of this release.
1246 """
1247 index: PythonPackageIndex = self._package._storage
1248 for requirement in self._requirements[None]:
1249 package = index.DownloadProject(requirement.name)
1251 for release in package:
1252 if str(release._version) in requirement.specifier:
1253 self.AddDependencyToPackageVersion(release)
1255 self.SortDependencies()
1256 self.__lazy_state__ = LazyLoaderState.PostProcessed
1258 @lazy(LazyLoaderState.PartiallyLoaded)
1259 def __repr__(self) -> str:
1260 """
1261 Return a detailed string representation of this release, loading its details if needed.
1263 :returns: Package name, version and the number of distributions.
1264 """
1265 return f"Release: {self._package._name}:{self._version} Files: {len(self._files)}"
1267 def __str__(self) -> str:
1268 """
1269 Return a string representation of this release.
1271 :returns: The release's version number.
1272 """
1273 return f"{self._version}"
1276@export
1277class Project(Package, LazyLoadableMixin):
1278 """
1279 A project (package) on a Python package index, with its releases.
1281 The list of releases and the project's details are fetched from the index on first use.
1282 """
1283 _url: Nullable[URL] #: URL of the project's page on the package index.
1285 _api: Nullable[URL] #: URL of the package index's API, used to load the project's details.
1286 _session: Nullable[Session] #: HTTP session reused for the API requests.
1288 def __init__(
1289 self,
1290 name: str,
1291 url: Union[str, URL],
1292 releases: Nullable[Iterable[Release]] = None,
1293 index: Nullable[PythonPackageIndex] = None,
1294 lazy: LazyLoaderState = LazyLoaderState.Initialized
1295 ) -> None:
1296 """
1297 Initialize a project on a package index.
1299 The API endpoint and the HTTP session are taken from the index, so the project can fetch its own details.
1301 :param name: Name of the project on the package index.
1302 :param url: URL of the project's page, as a string or a parsed URL.
1303 :param releases: Optional, releases of this project.
1304 :param index: Optional, package index this project is hosted on.
1305 :param lazy: Optional, state the project should be loaded to immediately.
1306 """
1307 if index is not None: 1307 ↛ 1311line 1307 didn't jump to line 1311 because the condition on line 1307 was always true
1308 self._api = index._api
1309 self._session = index._session
1310 else:
1311 self._api = None
1312 self._session = None
1314 super().__init__(name, storage=index)
1315 LazyLoadableMixin.__init__(self, lazy)
1317 # if isinstance(url, str):
1318 # url = URL.Parse(url)
1319 # elif not isinstance(url, URL):
1320 # ex = TypeError("Parameter 'url' is not of type 'URL'.")
1321 # ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
1322 # raise ex
1323 #
1324 # self._url = url
1325 # self._releases = {release.Version: release for release in sorted(releases, key=lambda r: r.Version)} if releases is not None else {}
1327 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
1328 """
1329 Download the project's details and its releases' details, as far as the target state demands.
1331 :param targetLevel: Optional, state the project needs to be loaded to.
1332 """
1333 if targetLevel >= LazyLoaderState.PartiallyLoaded: 1333 ↛ 1336line 1333 didn't jump to line 1336 because the condition on line 1333 was always true
1334 self.DownloadDetails()
1336 if targetLevel >= LazyLoaderState.PostProcessed: 1336 ↛ 1337line 1336 didn't jump to line 1337 because the condition on line 1336 was never true
1337 self.DownloadReleaseDetails()
1339 @readonly
1340 def PackageIndex(self) -> PythonPackageIndex:
1341 """
1342 Read-only property to access the package index this project was read from (:attr:`_storage`).
1344 :returns: The package index this project belongs to.
1345 """
1346 return self._storage
1348 @lazy(LazyLoaderState.PartiallyLoaded)
1349 @readonly
1350 def URL(self) -> URL:
1351 """
1352 Read-only property to access the project's URL in the package index (:attr:`_url`).
1354 :returns: URL of the project.
1355 """
1356 return self._url
1358 @lazy(LazyLoaderState.PartiallyLoaded)
1359 @readonly
1360 def Releases(self) -> dict[PythonVersion, Release]:
1361 """
1362 Read-only property to access all known releases of this project (:attr:`_versions`).
1364 :returns: Dictionary of versions and their releases.
1365 """
1366 return self._versions
1368 @lazy(LazyLoaderState.PartiallyLoaded)
1369 @readonly
1370 def ReleaseCount(self) -> int:
1371 """
1372 Read-only property to return the number of known releases.
1374 :returns: Number of releases.
1375 """
1376 return len(self._versions)
1378 @lazy(LazyLoaderState.PartiallyLoaded)
1379 @readonly
1380 def LatestRelease(self) -> Release:
1381 """
1382 Read-only property to return the most recent release of this project.
1384 :returns: The latest release.
1385 """
1386 return firstValue(self._versions)
1388 def _GetPyPIEndpoint(self) -> str:
1389 """
1390 Return the API endpoint describing this project.
1392 :returns: The endpoint's path, relative to the index's API URL.
1393 """
1394 return f"{self._name.lower()}/json"
1396 def DownloadDetails(self) -> None:
1397 """
1398 Download this project's details and its list of releases from the package index.
1400 :raises NoSessionAvailableError: If the project wasn't created by a package index, so it has no session. |br|
1401 A session is opened by the package index and handed to the objects it
1402 creates.
1403 :raises ProjectNotFoundError: If the index doesn't know this project.
1404 """
1405 if self._session is None: 1405 ↛ 1406line 1405 didn't jump to line 1406 because the condition on line 1405 was never true
1406 ex = NoSessionAvailableError(f"No session available to download details of package '{self._name}'.")
1407 ex.add_note("A session is opened by the package index and handed to the objects it creates.")
1408 raise ex
1410 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
1411 try:
1412 response.raise_for_status()
1413 except HTTPError as ex:
1414 if ex.response is not None and ex.response.status_code == 404:
1415 raise ProjectNotFoundError(f"Package '{self._name}' not found.") from ex
1417 self.UpdateDetailsFromPyPIJSON(response.json())
1419 def UpdateDetailsFromPyPIJSON(self, json) -> None:
1420 """
1421 Fill this project from the JSON document the package index returned.
1423 Releases without a distribution are skipped, and a version the parser doesn't understand is reported as a
1424 warning rather than failing the whole project.
1426 :param json: The parsed JSON document describing this project.
1427 """
1428 infoNode = json["info"]
1429 releasesNode = json["releases"]
1431 # Update project/package URL
1432 self._url = URL.Parse(infoNode["project_url"])
1434 # Convert key to Version number, skip empty releases
1435 convertedReleasesNode = {}
1436 for k, v in releasesNode.items():
1437 if len(v) == 0: 1437 ↛ 1438line 1437 didn't jump to line 1438 because the condition on line 1437 was never true
1438 continue
1440 try:
1441 version = PythonVersion.Parse(k)
1442 convertedReleasesNode[version] = v
1443 except ValueError as ex:
1444 print(f"Unsupported version format '{k}' - {ex}")
1446 for version, releaseNode in sorted(convertedReleasesNode.items(), key=lambda t: t[0]):
1447 if Parts.Postfix in version._parts: 1447 ↛ 1448line 1447 didn't jump to line 1448 because the condition on line 1447 was never true
1448 pass
1450 files = [Distribution(file["filename"], file["url"], datetime.fromisoformat(file["upload_time_iso_8601"]), ) for
1451 file in releaseNode]
1452 lazy = LazyLoaderState.PartiallyLoaded if LazyLoaderState.PartiallyLoaded <= self.__lazy_state__ <= LazyLoaderState.FullyLoaded else LazyLoaderState.Initialized
1453 Release(
1454 version,
1455 files[0]._uploadTime,
1456 files,
1457 project=self,
1458 lazy=lazy
1459 )
1461 self.SortVersions()
1462 self.__lazy_state__ = LazyLoaderState.FullyLoaded
1464 def DownloadReleaseDetails(self) -> None:
1465 """
1466 Download the details of every release of this project, in parallel.
1468 The requests run in one :mod:`asyncio` event loop over a shared session, because a project can easily have
1469 hundreds of releases.
1470 """
1471 async def ParallelDownloadReleaseDetails():
1472 """
1473 Nested coroutine downloading the details of every release over one shared session.
1474 """
1475 async def routine(session, release: Release):
1476 """
1477 Nested coroutine downloading the details of a single release.
1479 :param session: The HTTP session shared by all requests of this download.
1480 :param release: The release to download the details for.
1481 """
1482 if Parts.Postfix in release._version._parts: 1482 ↛ 1483line 1482 didn't jump to line 1483 because the condition on line 1482 was never true
1483 pass
1485 async with session.get(release._GetPyPIEndpoint()) as response:
1486 json = await response.json()
1487 response.raise_for_status()
1489 release.UpdateDetailsFromPyPIJSON(json)
1491 async with ClientSession(base_url=str(self._api), headers={"accept": "application/json"}) as session:
1492 tasks = []
1493 for release in self._versions.values(): # type: Release
1494 tasks.append(routine(session, release))
1496 results = await asyncio_gather(*tasks, return_exceptions=True)
1497 delList = []
1498 for release, result in zip(self.Releases.values(), results):
1499 if isinstance(result, Exception): 1499 ↛ 1500line 1499 didn't jump to line 1500 because the condition on line 1499 was never true
1500 delList.append((release, result))
1502 for release, ex in delList: 1502 ↛ 1503line 1502 didn't jump to line 1503 because the loop on line 1502 never started
1503 WarningCollector.Raise(
1504 ReleaseDetailsWarning(f"Dropping release '{release.Version}' of package '{release.Project._name}': details couldn't be downloaded."),
1505 ex
1506 )
1507 del self.Releases[release.Version]
1509 asyncio_run(ParallelDownloadReleaseDetails())
1510 self.__lazy_state__ = LazyLoaderState.PostProcessed
1512 def __repr__(self) -> str:
1513 """
1514 Return a detailed string representation of this project.
1516 :returns: The project's name and its latest release's version.
1517 """
1518 return f"Project: {self._name} latest: {self.LatestRelease._version}"
1520 def __str__(self) -> str:
1521 """
1522 Return a string representation of this project.
1524 :returns: The project's name.
1525 """
1526 return f"{self._name}"
1529@export
1530class PythonPackageIndex(PackageStorage):
1531 """
1532 A Python package index like PyPI, addressed through its JSON API.
1534 It is the entry point of the dependency graph: projects are looked up here, and every request to the index reuses
1535 the same HTTP session.
1536 """
1537 _url: URL #: URL of the package index's website.
1538 _api: URL #: URL of the package index's API.
1539 _session: Session #: HTTP session reused for every request to this index.
1540 _licenseOverrides: LicenseOverrides #: Licenses stated by hand, for what this index can't answer for.
1542 def __init__(
1543 self,
1544 name: str,
1545 url: Union[str, URL],
1546 api: Union[str, URL],
1547 graph: PackageDependencyGraph,
1548 licenseOverrides: Nullable[LicenseOverrides] = None
1549 ) -> None:
1550 """
1551 Initialize a package index and open the HTTP session used for every request to it.
1553 :param name: Name of the package index.
1554 :param url: URL of the index's website, as a string or a parsed URL.
1555 :param api: URL of the index's JSON API, as a string or a parsed URL.
1556 :param graph: Dependency graph this index belongs to.
1557 :param licenseOverrides: Optional, licenses stated by hand; an empty set of overrides if not given.
1558 :raises TypeError: If parameter 'url' is neither a string nor a :class:`~pyTooling.GenericPath.URL.URL`.
1559 :raises TypeError: If parameter 'api' is neither a string nor a :class:`~pyTooling.GenericPath.URL.URL`.
1560 """
1561 super().__init__(name, graph)
1563 self._licenseOverrides = licenseOverrides if licenseOverrides is not None else LicenseOverrides()
1565 if isinstance(url, str): 1565 ↛ 1567line 1565 didn't jump to line 1567 because the condition on line 1565 was always true
1566 url = URL.Parse(url)
1567 elif not isinstance(url, URL):
1568 ex = TypeError("Parameter 'url' is not of type 'URL'.")
1569 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
1570 raise ex
1572 self._url = url
1574 if isinstance(api, str): 1574 ↛ 1576line 1574 didn't jump to line 1576 because the condition on line 1574 was always true
1575 api = URL.Parse(api)
1576 elif not isinstance(api, URL):
1577 ex = TypeError("Parameter 'api' is not of type 'URL'.")
1578 ex.add_note(f"Got type '{getFullyQualifiedName(api)}'.")
1579 raise ex
1581 self._api = api
1583 self._session = Session()
1584 self._session.headers["accept"] = "application/json"
1586 # A package index is a third party reached over the Internet, and a documentation build that resolves a
1587 # hundred packages will meet a reset connection or a rate-limit eventually.
1588 adapter = HTTPAdapter(max_retries=Retry(
1589 total=RETRY_ATTEMPTS,
1590 backoff_factor=RETRY_BACKOFF,
1591 status_forcelist=RETRY_STATUS_CODES,
1592 ))
1593 self._session.mount("https://", adapter)
1594 self._session.mount("http://", adapter)
1596 @readonly
1597 def URL(self) -> URL:
1598 """
1599 Read-only property to access the package index' base URL (:attr:`_url`).
1601 :returns: Base URL of the package index.
1602 """
1603 return self._url
1605 @readonly
1606 def LicenseOverrides(self) -> LicenseOverrides:
1607 """
1608 Read-only property to access the licenses stated by hand for this index (:attr:`_licenseOverrides`).
1610 :returns: The index's license overrides.
1611 """
1612 return self._licenseOverrides
1614 @readonly
1615 def API(self) -> URL:
1616 """
1617 Read-only property to access the package index' API URL (:attr:`_api`).
1619 :returns: API URL of the package index.
1620 """
1621 return self._api
1623 @readonly
1624 def Projects(self) -> dict[str, Project]:
1625 """
1626 Read-only property to access all projects known to this package index (:attr:`_packages`).
1628 :returns: Dictionary of project names and projects.
1629 """
1630 return self._packages
1632 @readonly
1633 def ProjectCount(self) -> int:
1634 """
1635 Read-only property to return the number of known projects.
1637 :returns: Number of projects.
1638 """
1639 return len(self._packages)
1641 def _GetPyPIEndpoint(self, projectName: str) -> str:
1642 """
1643 Return the API endpoint describing a project.
1645 :param projectName: Name of the project on the package index.
1646 :returns: The endpoint's URL.
1647 """
1648 return f"{self._api}{projectName.lower()}/json"
1650 def DownloadProject(self, projectName: str, lazy: LazyLoaderState = LazyLoaderState.PartiallyLoaded) -> Project:
1651 """
1652 Look up a project on this package index.
1654 :param projectName: Name of the project on the package index.
1655 :param lazy: Optional, state the project should be loaded to immediately.
1656 :returns: The project, loaded as far as ``lazy`` demands.
1657 """
1658 project = Project(projectName, "", index=self, lazy=lazy)
1660 return project
1662 def __repr__(self) -> str:
1663 """
1664 Return a detailed string representation of this package index.
1666 :returns: The index's name.
1667 """
1668 return f"{self._name}"
1670 def __str__(self) -> str:
1671 """
1672 Return a string representation of this package index.
1674 :returns: The index's name.
1675 """
1676 return f"{self._name}"
1679@export
1680class PythonPackageDependencyGraph(PackageDependencyGraph):
1681 """
1682 A dependency graph of Python packages, whose vertices are projects and whose edges are requirements.
1683 """
1685 def __init__(self, name: str) -> None:
1686 """
1687 Initialize an empty dependency graph of Python packages.
1689 :param name: Name of the dependency graph.
1690 """
1691 super().__init__(name)