Coverage for pyTooling/Dependency/Python.py: 92%

1# ==================================================================================================================== #

2# _____ _ _ ____ _ #

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ | _ \ ___ _ __ ___ _ __ __| | ___ _ __ ___ _ _ #

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | | | |/ _ \ '_ \ / _ \ '_ \ / _` |/ _ \ '_ \ / __| | | | #

5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| |_| | __/ |_) | __/ | | | (_| | __/ | | | (__| |_| | #

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/ \___| .__/ \___|_| |_|\__,_|\___|_| |_|\___|\__, | #

7# |_| |___/ |___/ |_| |___/ #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32Implementation of package dependencies.

34Importing this module needs the ``dependencies`` extra, because it reads a package index over HTTP and parses PEP 440

35requirements:

37* :mod:`aiohttp`,

38* :mod:`packaging` and

39* :mod:`requests`

41are imported at module level and each is guarded, so a missing one names itself rather than failing as a bare

42:exc:`ImportError`.

44:raises MissingDependencyError: If the 'dependencies' extra isn't installed.

46.. hint::

48 See :ref:`high-level help <DEPENDENCIES>` for explanations and usage examples.

49"""

50from __future__ import annotations

52from asyncio import run as asyncio_run, gather as asyncio_gather

53from collections import deque

54from datetime import datetime, timedelta

55from enum import IntEnum

56from functools import wraps, update_wrapper

57from pathlib import Path

58from re import compile as re_compile, Pattern

59from threading import RLock

60from typing import Any, ClassVar, Deque, Optional as Nullable, Union, Iterable, Iterator, Mapping, \

61 Self

63from pyTooling.Configuration import Dictionary

64from pyTooling.Exceptions import MissingDependencyError

66try:

67 from aiohttp import ClientSession

68except ImportError as ex: # pragma: no cover

69 raise MissingDependencyError(dependency="aiohttp", extra="dependencies") from ex

71try:

72 from packaging.requirements import InvalidRequirement, Requirement

73 from packaging.utils import canonicalize_name

74except ImportError as ex: # pragma: no cover

75 raise MissingDependencyError(dependency="packaging", extra="dependencies") from ex

77try:

78 from requests import Session, HTTPError

79 from requests.adapters import HTTPAdapter

80 from urllib3.util.retry import Retry

81except ImportError as ex: # pragma: no cover

82 raise MissingDependencyError(dependency="requests", extra="dependencies") from ex

84from pyTooling.Decorators import export, readonly

85from pyTooling.MetaClasses import ExtendedType, abstractmethod

86from pyTooling.Common import getFullyQualifiedName, firstValue

87from pyTooling.Dependency import Package, PackageStorage, PackageVersion, PackageDependencyGraph, \

88 ReleaseDetailsWarning, UndeclaredExtraMarkerWarning, UnknownLicenseWarning, \

89 UnparsableRequirementWarning, UnsupportedVersionWarning, ProjectNotFoundError, \

90 DependencyError, NoSessionAvailableError, ReleaseNotFoundError, \

91 CircularRequirementsFileError, RequirementsFileNotFoundError, PackageIndexError, \

92 AlreadyInRequirementsTreeError

93from pyTooling.Licensing import LicenseExpression, LicenseExpressionError, LICENSES_BY_CLASSIFIER, \

94 LicenseAbsence, ProprietaryLicense, UnknownLicense

95from pyTooling.Stopwatch import Stopwatch

96from pyTooling.Warning import WarningCollector

97from pyTooling.GenericPath.URL import URL

98from pyTooling.Versioning import Parts, PythonVersion, PythonVersionExpression, SemanticVersion

100

101#: Longest prefix of a free-text ``license`` field quoted in a warning note, so a full license text doesn't flood it.

102_LICENSE_NOTE_LENGTH = 64

103

104#: PyPI's classifier for a license that isn't open source. SPDX can't name one, so it becomes a

105#: :class:`~pyTooling.Licensing.ProprietaryLicense` rather than an expression to parse.

106_PROPRIETARY_CLASSIFIER = "License :: Other/Proprietary License"

107

108#: Aliases matched against the free-text keys of ``project_urls``, lower-cased, most specific first.

109_REPOSITORY_URL_ALIASES = ("source code", "source", "code", "repository", "github", "gitlab")

110_DOCUMENTATION_URL_ALIASES = ("documentation", "docs", "read the docs")

111_ISSUE_TRACKER_URL_ALIASES = ("bug tracker", "issue tracker", "issues", "bug reports", "tracker")

112_PROJECT_URL_ALIASES = ("homepage", "home page", "home")

113_CHANGELOG_URL_ALIASES = ("changelog", "changes", "release notes", "whatsnew", "what's new")

114

115#: Pattern of an ``extra == "<name>"`` comparison in a requirement's marker.

116_EXTRA_MARKER = re_compile(r'''extra\s*==\s*["']([^"']+)["']''')

117

118#: How often a request to a package index is retried before it is reported as an error.

119RETRY_ATTEMPTS = 4

120

121#: Seconds the delay between two retries grows by, doubling per attempt: 0.5 s, 1 s, 2 s, 4 s.

122RETRY_BACKOFF = 0.5

123

124#: Status codes worth retrying: a rate-limit and the transient server-side failures.

125RETRY_STATUS_CODES = (429, 500, 502, 503, 504)

126

127#: What a requirements file states: a requirement, a referenced file, or a reference to a file read before.

128RequirementsFileEntryT = Union[Requirement, "RequirementsFile", "RequirementsFileReference"]

129

130

131@export

132class RequirementsFile(metaclass=ExtendedType, slots=True):

133 """

134 A ``requirements.txt`` file, together with the files it references.

135

136 A ``-r other.txt`` line references another file, and the tree of those references is kept rather than flattened:

137 ``tests/requirements.txt`` is nothing but four ``-r`` lines, so a flattened list would say nothing about which of

138 them a package came from - which is exactly what a table per entrypoint has to show.

139

140 It is a **tree**, so every file knows where it sits in one: :attr:`Parent` is the file that referenced it and

141 :attr:`Root` the entrypoint the whole tree was read from. The root alone keeps

142 :attr:`AnalyzedRequirementFiles`, every file of the tree by its resolved path, which is the answer to *where did

143 this package come from*.

144

145 :meth:`Parse` reads a file and the files it references and measures how long reading took

146 (:attr:`AnalysisDuration`); the initializer takes what a file states, so a tree can be built by hand, too.

147 **A cycle raises.** A file referencing itself, directly or through a chain, is a statement

148 nobody wrote on purpose; reading it once and continuing would hide it.

149

150 **A file referenced twice is read once.** Two files may reference the same third one, as pip accepts it -

151 ``unit/requirements.txt`` and ``typing/requirements.txt`` both referencing ``../../requirements.txt``. The first

152 ``-r`` line reads the file into the tree; every later one is a :class:`RequirementsFileReference` to it. So a file

153 is one object with one parent, and :attr:`AnalyzedRequirementFiles` keeps one per path. Only a file referenced

154 while it's still being read - one on the chain from the root down to the referencing file - is a cycle.

155 """

156

157 _root: RequirementsFile #: Entrypoint this tree was read from.

158 _parent: Nullable[RequirementsFile] #: Referencing file; ``None`` for a root.

159 _path: Path #: Path of this requirements file.

160 _entries: list[RequirementsFileEntryT] #: What this file states, in file order.

161 _analyzedRequirementFiles: Nullable[dict[Path, RequirementsFile]] #: Every file of the tree, by resolved path.

162 _analysisDuration: Nullable[timedelta] #: Time :meth:`Parse` took to read the file.

163

164 def __init__(

165 self,

166 path: Path,

167 entries: Nullable[Iterable[RequirementsFileEntryT]] = None,

168 *,

169 parent: Nullable[RequirementsFile] = None,

170 analysisDuration: Nullable[timedelta] = None

171 ) -> None:

172 """

173 Initializes a requirements file from its path and what it states.

174

175 A requirements file given in ``entries`` is referenced by this file: it and the files it references join this

176 file's tree. A :class:`RequirementsFileReference` given in ``entries`` is stated by this file; its target isn't

177 checked to be in the same tree, because :meth:`Parse` initializes a file before the root joining both exists.

178

179 :param path: Path of the requirements file.

180 :param entries: Optional, requirements, referenced files and references to files read

181 before, in the order the file states them. Default: ``None``.

182 :param parent: Optional, the file referencing this one, which appends this file to its

183 entries. Default: ``None``, a root.

184 :param analysisDuration: Optional, the time reading the file took, including the files it

185 references. Default: ``None``, not measured.

186 :raises ValueError: If parameter 'path' is ``None``.

187 :raises TypeError: If parameter 'path' is not of type :class:`~pathlib.Path`.

188 :raises TypeError: If parameter 'entries' is not iterable.

189 :raises TypeError: If parameter 'parent' is not of type :class:`RequirementsFile`.

190 :raises TypeError: If parameter 'analysisDuration' is not of type :class:`~datetime.timedelta`.

191 :raises ValueError: If an element of parameter 'entries' is ``None``.

192 :raises TypeError: If an element of parameter 'entries' is not of type

193 :class:`~packaging.requirements.Requirement`, :class:`RequirementsFile`

194 or :class:`RequirementsFileReference`.

195 :raises AlreadyInRequirementsTreeError: If an element of parameter 'entries' is referenced by another file

196 already.

197 :raises AlreadyInRequirementsTreeError: If an element of parameter 'entries' is a reference another file states

198 already.

199 """

200 if path is None:

201 raise ValueError("Parameter 'path' is None.")

202 elif not isinstance(path, Path):

203 ex = TypeError("Parameter 'path' is not of type 'Path'.")

204 ex.add_note(f"Got type '{getFullyQualifiedName(path)}'.")

205 raise ex

206

207 if entries is not None and not isinstance(entries, Iterable):

208 ex = TypeError("Parameter 'entries' is not iterable.")

209 ex.add_note(f"Got type '{getFullyQualifiedName(entries)}'.")

210 raise ex

211

212 if parent is not None and not isinstance(parent, RequirementsFile):

213 ex = TypeError("Parameter 'parent' is not of type 'RequirementsFile'.")

214 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")

215 raise ex

216

217 if analysisDuration is not None and not isinstance(analysisDuration, timedelta):

218 ex = TypeError("Parameter 'analysisDuration' is not of type 'timedelta'.")

219 ex.add_note(f"Got type '{getFullyQualifiedName(analysisDuration)}'.")

220 raise ex

221

222 self._path = path

223 self._parent = parent

224 self._root = self if parent is None else parent._root

225 self._entries = []

226 self._analysisDuration = analysisDuration

227

228 # resolved as the mapping's key only, not as '_path': a file reachable under two spellings is one key

229 if parent is None:

230 self._analyzedRequirementFiles = {

231 path.resolve(): self

232 }

233 else:

234 self._analyzedRequirementFiles = None

235 self._root._analyzedRequirementFiles[path.resolve()] = self

236

237 if entries is not None:

238 for entry in entries:

239 if entry is None:

240 raise ValueError("An element of parameter 'entries' is None.")

241 elif isinstance(entry, RequirementsFile):

242 if entry._parent is not None:

243 raise AlreadyInRequirementsTreeError(

244 f"Requirements file '{entry._path}' is referenced by '{entry._parent._path}' already."

245 )

246

247 entry._parent = self

248 for resolvedPath, requirementsFile in entry._analyzedRequirementFiles.items():

249 requirementsFile._root = self._root

250 self._root._analyzedRequirementFiles[resolvedPath] = requirementsFile

251

252 entry._analyzedRequirementFiles = None

253 elif isinstance(entry, RequirementsFileReference):

254 if entry._parent is not None:

255 raise AlreadyInRequirementsTreeError(

256 f"Reference to requirements file '{entry._path}' is stated by '{entry._parent._path}' already."

257 )

258

259 entry._parent = self

260 elif not isinstance(entry, Requirement):

261 ex = TypeError(

262 "An element of parameter 'entries' is not of type 'Requirement', 'RequirementsFile' or "

263 "'RequirementsFileReference'."

264 )

265 ex.add_note(f"Got type '{getFullyQualifiedName(entry)}'.")

266 raise ex

267

268 self._entries.append(entry)

269

270 if parent is not None:

271 parent._entries.append(self)

272

273 @classmethod

274 def Parse(cls, path: Path) -> Self:

275 """

276 Read a requirements file and the files it references.

277

278 Every ``-r`` line is read into a referenced file first, so a file is initialized after the files it references,

279 and the root is initialized last. A ``-r`` line referencing a file read before becomes a

280 :class:`RequirementsFileReference` to it; the file isn't read again. A line that can't be parsed as a requirement

281 is left out and reported as a critical :exc:`~pyTooling.Dependency.UnparsableRequirementWarning` to a

282 :class:`~pyTooling.Warning.WarningCollector`; without a collector,

283 :exc:`~pyTooling.Warning.UnhandledCriticalWarningError` is raised.

284

285 Each file measures how long reading it took with a :class:`~pyTooling.Stopwatch.Stopwatch`, the files it

286 references included (:attr:`AnalysisDuration`).

287

288 :param path: Path of the requirements file to read.

289 :returns: The requirements file, root of the tree of the files it references.

290 :raises ValueError: If parameter 'path' is ``None``.

291 :raises TypeError: If parameter 'path' is not of type :class:`~pathlib.Path`.

292 :raises RequirementsFileNotFoundError: If the requirements file doesn't exist.

293 :raises RequirementsFileNotFoundError: If a ``-r`` line references a file that doesn't exist.

294 :raises CircularRequirementsFileError: If a ``-r`` line references a file still being read, on the chain from the

295 root down to the referencing file.

296 :raises UnhandledCriticalWarningError: If a line can't be parsed as a requirement and no

297 :class:`~pyTooling.Warning.WarningCollector` collects the

298 :exc:`~pyTooling.Dependency.UnparsableRequirementWarning`.

299 """

300 if path is None:

301 raise ValueError("Parameter 'path' is None.")

302 elif not isinstance(path, Path):

303 ex = TypeError("Parameter 'path' is not of type 'Path'.")

304 ex.add_note(f"Got type '{getFullyQualifiedName(path)}'.")

305 raise ex

306

307 readFiles: dict[Path, RequirementsFile] = {} # by resolved path

308 readingFiles: set[Path] = set()

309 chain: list[Path] = []

310

311 def parseFile(file: Path) -> Self:

312 """

313 Nested function for recursion.

314

315 :param file: Path of the requirements file to read.

316 :returns: The requirements file, initialized after the files it references.

317 :raises RequirementsFileNotFoundError: If the requirements file doesn't exist.

318 :raises CircularRequirementsFileError: If a ``-r`` line references a file still being read.

319 """

320 if not file.exists():

321 raise RequirementsFileNotFoundError(

322 f"Requirements file '{file}' does not exist.",

323 filePath=file

324 ) from FileNotFoundError(file)

325

326 with Stopwatch() as sw:

327 readingFiles.add(file.resolve())

328 chain.append(file)

329 entries: list[RequirementsFileEntryT] = []

330

331 lines = file.read_text(encoding="utf-8").splitlines()

332 for lineNumber, line in enumerate(lines, start=1):

333 if (line := line.split("#")[0].strip()) == "":

334 continue

335

336 if line.startswith("-r"):

337 referenced = (file.parent / line[2:].strip()).resolve()

338

339 if referenced in readingFiles:

340 ex = CircularRequirementsFileError(

341 f"Requirements file '{referenced}' referenced in '{file}' line {lineNumber} is already being read.",

342 filePath=referenced,

343 referencedIn=file,

344 lineNumber=lineNumber

345 )

346 ex.add_note(f"Chain: {' → '.join(str(chainedFile) for chainedFile in chain)} → {referenced}")

347 raise ex

348 elif (target := readFiles.get(referenced, None)) is not None:

349 entries.append(RequirementsFileReference(referenced, target))

350 else:

351 entries.append(parseFile(referenced))

352

353 continue

354

355 # '--index-url', '-e' and a bare URL are instructions to the installer, not requirements

356 if line.startswith("-") or line.startswith("http"):

357 continue

358

359 try:

360 entries.append(Requirement(line))

361 except InvalidRequirement as ex:

362 WarningCollector.Raise(

363 UnparsableRequirementWarning(

364 f"Requirement '{line}' in '{file}' line {lineNumber} can't be parsed.",

365 line=line,

366 filePath=file,

367 lineNumber=lineNumber

368 ),

369 ex

370 )

371

372 chain.pop()

373 readingFiles.discard(file.resolve())

374

375 requirementsFile = cls(file, entries, analysisDuration=sw.Duration)

376 readFiles[file.resolve()] = requirementsFile

377

378 return requirementsFile

379

380 return parseFile(path)

381

382 @readonly

383 def Path(self) -> Path:

384 """

385 Read-only property to access this requirement file's path (:attr:`_path`).

386

387 :returns: Path of the requirements file, spelled the way it was handed in.

388 """

389 return self._path

390

391 @readonly

392 def Root(self) -> RequirementsFile:

393 """

394 Read-only property to access the entrypoint this tree was read from (:attr:`_root`).

395

396 :returns: The tree's root, which is this file itself when it is one.

397 """

398 return self._root

399

400 @readonly

401 def Parent(self) -> Nullable[RequirementsFile]:

402 """

403 Read-only property to access the file whose ``-r`` line referenced this one (:attr:`_parent`).

404

405 :returns: The referencing file, or ``None`` for a root.

406 """

407 return self._parent

408

409 @readonly

410 def Hierarchy(self) -> tuple[RequirementsFile, ...]:

411 """

412 Read-only property to return the path from the root down to this file as a tuple.

413

414 :returns: A tuple of requirements files.

415 """

416 hierarchy: Deque[RequirementsFile] = deque([self])

417 parentRequirementsFile: Nullable[RequirementsFile] = self

418 while (parentRequirementsFile := parentRequirementsFile._parent) is not None:

419 hierarchy.appendleft(parentRequirementsFile)

420

421 return tuple(hierarchy)

422

423 @readonly

424 def AnalyzedRequirementFiles(self) -> Nullable[dict[Path, RequirementsFile]]:

425 """

426 Read-only property to access every file of this tree, by its resolved path.

427

428 **Only the root's is filled**; ask :attr:`Root` for it. It answers which files a tree was read from - the list

429 a documentation build registers so a change to any of them rebuilds the page.

430

431 :returns: Every file of the tree by resolved path, or ``None`` for a referenced file.

432 """

433 return self._analyzedRequirementFiles

434

435 @readonly

436 def AnalysisDuration(self) -> Nullable[timedelta]:

437 """

438 Read-only property to access the time :meth:`Parse` took to read this file (:attr:`_analysisDuration`).

439

440 It includes reading the files this file references, so the root's duration is the whole :meth:`Parse` call.

441

442 :returns: Duration of reading this file and the files it references, or ``None`` if the file wasn't read by

443 :meth:`Parse`.

444 """

445 return self._analysisDuration

446

447 @readonly

448 def Entries(self) -> list[RequirementsFileEntryT]:

449 """

450 Read-only property to access what this file states, in the order it states it (:attr:`_entries`).

451

452 Requirements, referenced files and references to files read before are kept in one list, because a file states

453 them interleaved. :attr:`Requirements`, :attr:`ReferencedFiles` and :attr:`FileReferences` are the filtered

454 views of it.

455

456 :returns: This file's requirements, referenced files and references.

457 """

458 return self._entries

459

460 @readonly

461 def Requirements(self) -> Iterator[Requirement]:

462 """

463 Read-only property to iterate the requirements stated in this file.

464

465 This is what *this* file states; what the files it references state is reachable through

466 :attr:`ReferencedFiles`.

467

468 :returns: An iterator of this file's requirements, in the order they are written.

469 """

470 return (entry for entry in self._entries if isinstance(entry, Requirement))

471

472 @readonly

473 def ReferencedFiles(self) -> Iterator[RequirementsFile]:

474 """

475 Read-only property to iterate the files referenced with ``-r`` and read here.

476

477 These are the children of this file in the tree. A file read before, where another file referenced it first, is in

478 :attr:`FileReferences` instead.

479

480 :returns: An iterator of the referenced files, in the order they are referenced.

481 """

482 return (entry for entry in self._entries if isinstance(entry, RequirementsFile))

483

484 @readonly

485 def FileReferences(self) -> Iterator[RequirementsFileReference]:

486 """

487 Read-only property to iterate the ``-r`` lines referencing a file read before.

488

489 The file itself is a child of the file referencing it first (:attr:`ReferencedFiles`); here it's a

490 :class:`RequirementsFileReference`, whose :attr:`~RequirementsFileReference.Target` is that file.

491

492 :returns: An iterator of the references, in the order they are written.

493 """

494 return (entry for entry in self._entries if isinstance(entry, RequirementsFileReference))

495

496 def IterateTree(self) -> Iterator[RequirementsFile]:

497 """

498 Iterate this file and every file it references, depth first.

499

500 :returns: A generator of requirements files, this one first.

501 """

502 yield self

503 for referenced in self.ReferencedFiles:

504 yield from referenced.IterateTree()

505

506 @readonly

507 def AllRequirements(self) -> Iterator[Requirement]:

508 """

509 Read-only property to iterate the requirements of this file and of every file it references.

510

511 **The file's order is kept**, and **the nearer statement wins**: a requirement stated in this file overrides

512 the same package required by a referenced file, wherever the two stand, because that is the constraint the

513 entrypoint was written for. Overriding keeps the position, so every package is yielded exactly once.

514

515 .. code-block:: text

516

517 # base.txt # requirements.txt AllRequirements

518 pytest ~= 8.0 pytest ~= 9.1 pytest ~= 9.1

519 sphinx ~= 9.1 -r base.txt sphinx ~= 9.1

520 colorama ~= 0.4.6 colorama ~= 0.4.6

521

522 A :class:`RequirementsFileReference` contributes its target's requirements, unless the target was collected

523 already in this call: a file referenced twice is collected where it's referenced first. So the tree's root yields

524 a shared file once, and a referencing file asked on its own still yields what it references.

525

526 :returns: A generator of requirements, deduplicated by canonical package name, in the order stated.

527 """

528 collected: set[RequirementsFile] = set()

529

530 def collect(requirementsFile: RequirementsFile) -> dict[str, Requirement]:

531 """

532 Nested function for recursion.

533

534 :param requirementsFile: The file to collect the requirements of.

535 :returns: The file's requirements and the ones it references, by canonical package name.

536 """

537 collected.add(requirementsFile)

538 requirements: dict[str, Requirement] = {}

539 stated: dict[str, Requirement] = {}

540

541 for entry in requirementsFile._entries:

542 if isinstance(entry, Requirement):

543 name = canonicalize_name(entry.name)

544 requirements[name] = stated[name] = entry

545 else:

546 referenced = entry if isinstance(entry, RequirementsFile) else entry._target

547 if referenced not in collected:

548 requirements.update(collect(referenced))

549

550 # a key keeps the position of its first insertion, so this overrides the value without moving the package

551 requirements.update(stated)

552

553 return requirements

554

555 yield from collect(self).values()

556

557 def __len__(self) -> int:

558 """

559 Return the number of requirements this file states, not counting the files it references.

560

561 :returns: Number of requirements stated in this file.

562 """

563 return sum(1 for _ in self.Requirements)

564

565 def __iter__(self) -> Iterator[Requirement]:

566 """

567 Iterate the requirements this file states, not the ones it references.

568

569 :returns: An iterator of this file's requirements.

570 """

571 return self.Requirements

572

573 def __str__(self) -> str:

574 """

575 Return this file's path and how much it states.

576

577 :returns: A string representation of this requirements file.

578 """

579 referenced = sum(1 for entry in self._entries if not isinstance(entry, Requirement))

580

581 return f"{self._path}: {len(self)} requirement(s), {referenced} referenced file(s)"

582

583

584@export

585class RequirementsFileReference(metaclass=ExtendedType, slots=True):

586 """

587 A ``-r`` line referencing a requirements file read before, where another file referenced it first.

588

589 Two files may reference the same third one - ``unit/requirements.txt`` and ``typing/requirements.txt`` both

590 referencing ``../../requirements.txt``. The file is read once: the first ``-r`` line makes it a

591 :class:`RequirementsFile` in the tree, every later one is this reference to it. A reference is no node of the tree:

592 it has a :attr:`Parent`, the file stating the line, and its :attr:`Target` is the file read before.

593 """

594

595 _parent: Nullable[RequirementsFile] #: File stating the ``-r`` line; ``None`` until a file states it.

596 _path: Path #: Path of the referenced file.

597 _target: RequirementsFile #: The referenced file, read where it was referenced first.

598

599 def __init__(self, path: Path, target: RequirementsFile, *, parent: Nullable[RequirementsFile] = None) -> None:

600 """

601 Initializes a reference to a requirements file read before.

602

603 :param path: Path of the referenced file.

604 :param target: The referenced file, read where it was referenced first.

605 :param parent: Optional, the file stating the ``-r`` line, which appends this reference to its entries.

606 Default: ``None``.

607 :raises ValueError: If parameter 'path' is ``None``.

608 :raises TypeError: If parameter 'path' is not of type :class:`~pathlib.Path`.

609 :raises ValueError: If parameter 'target' is ``None``.

610 :raises TypeError: If parameter 'target' is not of type :class:`RequirementsFile`.

611 :raises TypeError: If parameter 'parent' is not of type :class:`RequirementsFile`.

612 """

613 if path is None:

614 raise ValueError("Parameter 'path' is None.")

615 elif not isinstance(path, Path):

616 ex = TypeError("Parameter 'path' is not of type 'Path'.")

617 ex.add_note(f"Got type '{getFullyQualifiedName(path)}'.")

618 raise ex

619

620 if target is None:

621 raise ValueError("Parameter 'target' is None.")

622 elif not isinstance(target, RequirementsFile):

623 ex = TypeError("Parameter 'target' is not of type 'RequirementsFile'.")

624 ex.add_note(f"Got type '{getFullyQualifiedName(target)}'.")

625 raise ex

626

627 if parent is not None and not isinstance(parent, RequirementsFile):

628 ex = TypeError("Parameter 'parent' is not of type 'RequirementsFile'.")

629 ex.add_note(f"Got type '{getFullyQualifiedName(parent)}'.")

630 raise ex

631

632 self._path = path

633 self._target = target

634 self._parent = parent

635

636 if parent is not None:

637 parent._entries.append(self)

638

639 @readonly

640 def Path(self) -> Path:

641 """

642 Read-only property to access the referenced file's path (:attr:`_path`).

643

644 :returns: Path of the referenced file, spelled the way it was handed in.

645 """

646 return self._path

647

648 @readonly

649 def Parent(self) -> Nullable[RequirementsFile]:

650 """

651 Read-only property to access the file stating the ``-r`` line (:attr:`_parent`).

652

653 :returns: The referencing file, or ``None`` if no file states the reference yet.

654 """

655 return self._parent

656

657 @readonly

658 def Target(self) -> RequirementsFile:

659 """

660 Read-only property to access the referenced file, read where it was referenced first (:attr:`_target`).

661

662 :returns: The referenced requirements file.

663 """

664 return self._target

665

666 def __str__(self) -> str:

667 """

668 Return the referenced file's path, marked as a reference.

669

670 :returns: A string representation of this reference.

671 """

672 return f"{self._path} (referenced before)"

673

674

675@export

676class LicenseOverrides(metaclass=ExtendedType, slots=True):

677 """

678 Licenses stated by hand, for the packages a package index can't answer for.

679

680 A package index is not a reliable source of license information: roughly half of a typical dependency set

681 publishes a PEP 639 ``license_expression``, some state a license *name* where an identifier is expected, and the

682 classifier ``License :: OSI Approved :: BSD License`` means either ``BSD-2-Clause`` or ``BSD-3-Clause`` with no

683 way to tell which. Those packages are answered here instead of guessed at.

684

685 The file is YAML, and a license may be stated for the package as a whole or per version - a package that

686 relicensed has one license before the switch and another after it:

687

688 .. code-block:: yaml

689

690 version: "0.1"

691 analysedAt: 2026-09-02

692

693 packages:

694 colorama:

695 license: BSD-3-Clause

696 licenseURL: https://GitHub.com/tartley/colorama/blob/master/LICENSE.txt

697 repository: https://GitHub.com/tartley/colorama

698 "igraph >=0.10":

699 license: GPL-2.0-or-later

700 "igraph <0.10":

701 license: GPL-2.0-only

702 "igraph 0.9.10":

703 license: GPL-2.0-only

704

705 ``version`` states the structure this file is written for and is checked against

706 :attr:`SCHEMA_VERSION`; ``analysedAt`` is the day a human last checked the statements - see :attr:`AnalysedAt`.

707

708 **A key is a package name, optionally followed by a version expression** - the shape a requirement line has.

709 The expression is read by :class:`~pyTooling.Versioning.PythonVersionExpression`, so it is the same language a

710 requirement file writes, and two of its rules are what make one key form enough:

711

712 * **a bare version is an equality**, so ``igraph 0.9.10`` and ``igraph ==0.9.10`` state the same thing, and

713 * **an expression with no constraints matches every version**, so ``igraph`` on its own is the statement for

714 every version rather than a special case.

715

716 Keys are matched in the order the file writes them and the first one a version satisfies wins, so a narrower

717 statement goes above the bare name it refines. Asking without a version answers from the bare-name key only.

718

719 A key states a whole entry, so ``licenseURL`` and ``repository`` can differ per version too - a project that

720 moved forge between releases has two ``repository`` statements and no special case for it.

721 """

722

723 #: Structure this parser reads. A file states the one it was written for as its ``version`` field, and a file

724 #: stating a different one is rejected rather than read on the chance that it still fits.

725 SCHEMA_VERSION: ClassVar[SemanticVersion] = SemanticVersion(0, 1)

726

727 #: Splits a key into the package name and whatever follows it. A name stops at the first character an operator

728 #: can start with, so ``igraph>=0.10`` splits the same way ``igraph >=0.10`` does.

729 _PACKAGE_KEY: ClassVar[Pattern[str]] = re_compile(r"^\s*(?P<name>[^\s<>=!~]+)\s*(?P<expression>.*?)\s*$")

730

731 _analysedAt: Nullable[datetime] #: When the statements were last checked by a human.

732 _path: Nullable[Path] #: Path of the file the statements were read from.

733 #: License expression per package, by the version expression its key states, in the file's order.

734 _licenses: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]

735 #: URL of the license's text per package, by the version expression its key states, in the file's order.

736 _licenseURLs: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]

737 #: URL of the source repository per package, by the version expression its key states, in the file's order.

738 _repositories: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]]

739

740 def __init__(self, analysedAt: Nullable[datetime] = None, path: Nullable[Path] = None) -> None:

741 """

742 Initialize an empty set of overrides.

743

744 :param analysedAt: Optional, the day these statements were last checked by a human.

745 :param path: Optional, path of the file the statements were read from.

746 """

747 self._analysedAt = analysedAt

748 self._path = path

749 self._licenses = {}

750 self._licenseURLs = {}

751 self._repositories = {}

752

753 @readonly

754 def AnalysedAt(self) -> Nullable[datetime]:

755 """

756 Read-only property to access when these statements were last checked (:attr:`_analysedAt`).

757

758 A package index answers for itself every time it is asked, so what it says is as old as the request. These

759 statements are written by hand and are as old as whoever last looked, which nothing else records - so a

760 report can mark an entry *overridden*, and *stale* once this date is far enough back.

761

762 :meth:`FromFile` requires it; :meth:`FromDictionary` doesn't, because overrides assembled in code are as old

763 as the code.

764

765 :returns: The day of the last analysis, or ``None`` if the overrides were built without one.

766 """

767 return self._analysedAt

768

769 @readonly

770 def Path(self) -> Nullable[Path]:

771 """

772 Read-only property to access the path of the file the statements were read from (:attr:`_path`).

773

774 A warning about a statement names the file to correct.

775

776 :returns: The path :meth:`FromFile` read, or ``None`` if the overrides were built without a file.

777 """

778 return self._path

779

780 @classmethod

781 def FromFile(cls, path: Path) -> Self:

782 """

783 Read overrides from a YAML file.

784

785 The file is read through :class:`pyTooling.Configuration.YAML.Configuration`, and the ``packages`` node is

786 handed to :meth:`FromDictionary` **as it is** - a configuration node answers ``items()`` and ``get()``, so

787 there is nothing to convert and no second constructor for the node tree.

788

789 ``version`` is **required** and is the first thing checked: it states which structure the file was written

790 for, and is compared against :attr:`SCHEMA_VERSION`. **Quote it** - unquoted, YAML reads ``0.1`` as a float,

791 and a float loses a trailing zero, so ``1.10`` would arrive as ``1.1``.

792

793 ``analysedAt`` is **required** too, and is the day a human last checked these statements. It is an ISO-8601

794 date, written either way round - ``analysedAt: 2026-09-02`` reads as YAML's own date type, and

795 ``analysedAt: "2026-09-02"`` as a string. A configuration hands both over in the same ISO-8601 spelling.

796

797 :param path: Path of the YAML file to read.

798 :returns: The overrides the file states.

799 :raises MissingDependencyError: If ``ruamel.yaml`` isn't installed.

800 :raises FileNotFoundError: If the file doesn't exist.

801 :raises ConfigurationError: If the file isn't a YAML document describing a mapping. |br|

802 An empty file is one, and states no overrides - but it states no

803 ``version`` either, so it is rejected by the next check.

804 :raises DependencyError: If ``version`` is missing, isn't a version, or isn't

805 :attr:`SCHEMA_VERSION`.

806 :raises DependencyError: If ``analysedAt`` is missing or isn't an ISO-8601 date.

807 :raises DependencyError: If ``packages`` isn't a mapping. |br|

808 A file stating no packages is fine and gives no overrides.

809 :raises DependencyError: If a version expression in the file can't be parsed.

810 """

811 # Imported here rather than at module level, so a missing 'ruamel.yaml' is reported when the overrides are

812 # read instead of when 'pyTooling.Dependency.Python' is imported.

813 from pyTooling.Configuration.YAML import Configuration

814

815 # 'Configuration' reports a missing file too, as a 'ConfigurationError' naming the *format*. This says which

816 # file of ours is missing, and is what the signature promises, so it stays.

817 if not path.exists():

818 raise FileNotFoundError(f"License override file '{path}' not found.")

819

820 configuration = Configuration(path)

821

822 if (schemaVersion := configuration.get("version", None)) is None:

823 ex = DependencyError(f"License override file '{path}' states no 'version'.")

824 ex.add_note("It says which structure the file is written for, so a later one can be told apart.")

825 ex.add_note(f'Add it as the first field: version: "{cls.SCHEMA_VERSION}"')

826 raise ex

827

828 try:

829 fileVersion = SemanticVersion.Parse(str(schemaVersion))

830 except ValueError as cause:

831 ex = DependencyError(f"License override file '{path}' states a 'version' that isn't a version number.")

832 ex.add_note(f"Got '{schemaVersion}'.")

833 raise ex from cause

834

835 if fileVersion != cls.SCHEMA_VERSION:

836 ex = DependencyError(f"License override file '{path}' is written for structure '{fileVersion}'.")

837 ex.add_note(f"This reads '{cls.SCHEMA_VERSION}'.")

838 raise ex

839

840 if (analysedMoment := configuration.get("analysedAt", None)) is None:

841 ex = DependencyError(f"License override file '{path}' states no 'analysedAt' timestamp.")

842 ex.add_note("These statements are written by hand, so nothing else records how old they are.")

843 ex.add_note("Add an ISO-8601 timestamp, for example: analysedAt: 2026-09-04T21:45:00+00:00")

844 raise ex

845

846 try:

847 analysedAt = datetime.fromisoformat(str(analysedMoment))

848 except ValueError as cause:

849 ex = DependencyError(

850 f"License override file '{path}' states an 'analysedAt' that isn't an ISO-8601 timestamp."

851 )

852 ex.add_note(f"Got '{analysedMoment}'.")

853 ex.add_note("Write it as an ISO-8601 timestamp: analysedAt: 2026-09-04T21:45:00+00:00")

854 raise ex from cause

855

856 packages = configuration.get("packages", None)

857 if packages is None:

858 return cls(analysedAt, path)

859 elif not isinstance(packages, Dictionary):

860 ex = DependencyError(f"License override file '{path}' states a 'packages' node that isn't a mapping.")

861 ex.add_note(f"Got '{packages}'.")

862 raise ex

863

864 return cls.FromDictionary(packages, analysedAt, path)

865

866 @classmethod

867 def FromDictionary(

868 cls,

869 packages: Union[Mapping[str, Any], Dictionary],

870 analysedAt: Nullable[datetime] = None,

871 path: Nullable[Path] = None

872 ) -> Self:

873 """

874 Build overrides from an already parsed mapping.

875

876 Keeping this apart from :meth:`FromFile` is what lets the overrides be assembled in code, and tested, without

877 a file and without YAML.

878

879 A :class:`~pyTooling.Configuration.Dictionary` is accepted beside a plain :class:`dict`, which is what lets

880 :meth:`FromFile` hand over a configuration node without flattening it first. Only ``items()`` and ``get()``

881 are read, so a configuration node of either backend fits - it is **not** a :class:`~typing.Mapping`, because

882 its bare iteration yields values rather than keys.

883

884 :param packages: Mapping of a package's name to what is stated for it.

885 :param analysedAt: Optional, the day these statements were last checked. :meth:`FromFile` requires one;

886 overrides assembled in code are as old as the code and need none.

887 :param path: Optional, path of the file the mapping was read from.

888 :returns: The overrides the mapping states.

889 :raises DependencyError: If a version specifier can't be parsed.

890 """

891 overrides = cls(analysedAt, path)

892

893 # A configuration node types its values as the whole 'ValueT' union, which every '.get' below would then

894 # have to be narrowed against. What is read here is a document either way, so it is read as one.

895 statement: Any

896 for packageKey, statement in packages.items():

897 name, versionExpression = cls._SplitKey(str(packageKey))

898

899 for field, table in (

900 ("license", overrides._licenses),

901 ("licenseURL", overrides._licenseURLs),

902 ("repository", overrides._repositories),

903 ):

904 if (value := statement.get(field, None)) is not None:

905 table.setdefault(name, []).append((versionExpression, str(value)))

906

907 return overrides

908

909 @classmethod

910 def _SplitKey(cls, packageKey: str) -> tuple[str, PythonVersionExpression[SemanticVersion]]:

911 """

912 Split a key into the package it names and the version expression it restricts that package to.

913

914 A key is the shape a requirement line has - a name, then optionally a version expression, with or without a

915 space between them. A key naming only a package gives an expression with no constraints, which matches every

916 version.

917

918 :param packageKey: The key as the file writes it.

919 :returns: The canonical package name, and the version expression.

920 :raises DependencyError: If what follows the name isn't a version expression.

921 """

922 match = cls._PACKAGE_KEY.match(packageKey)

923 if match is None: # pragma: no cover

924 ex = DependencyError(f"Key '{packageKey}' doesn't name a package.")

925 raise ex

926

927 try:

928 versionExpression: PythonVersionExpression[SemanticVersion] = PythonVersionExpression.Parse(match["expression"])

929 except (LicenseExpressionError, ValueError) as cause:

930 ex = DependencyError(f"Key '{packageKey}' states a version expression that can't be parsed.")

931 ex.add_note(f"Got '{match['expression']}'.")

932 ex.add_note("A bare version is an equality, so 'igraph 0.10' and 'igraph ==0.10' state the same thing.")

933 raise ex from cause

934

935 return canonicalize_name(match["name"]), versionExpression

936

937 def LicenseOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:

938 """

939 Return the license expression stated for a package, or for one of its versions.

940

941 Keys are tried in the order the file writes them and the first one this version satisfies wins, so a

942 narrower statement answers before the bare name it refines.

943

944 :param packageName: Name of the package.

945 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.

946 :returns: The license expression stated, or ``None`` if the package isn't overridden.

947 """

948 return self._Lookup(self._licenses, packageName, version)

949

950 def LicenseURLOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:

951 """

952 Return the URL of a package's license text, where one was stated.

953

954 :param packageName: Name of the package.

955 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.

956 :returns: The URL stated, or ``None``.

957 """

958 return self._Lookup(self._licenseURLs, packageName, version)

959

960 @staticmethod

961 def _Lookup(

962 table: dict[str, list[tuple[PythonVersionExpression[SemanticVersion], str]]],

963 packageName: str,

964 version: Nullable[SemanticVersion]

965 ) -> Nullable[str]:

966 """

967 Return the first statement in one table whose key applies to a version.

968

969 :param table: One of the per-package tables.

970 :param packageName: Name of the package.

971 :param version: The version to answer for. Without one, only an unrestricted key answers.

972 :returns: What that key states, or ``None`` if none applies.

973 """

974 if (entries := table.get(canonicalize_name(packageName), None)) is None:

975 return None

976

977 for versionExpression, value in entries:

978 # Without a version, only a key that restricts nothing can be said to apply.

979 if version in versionExpression if version is not None else len(versionExpression) == 0:

980 return value

981

982 return None

983

984 def RepositoryOf(self, packageName: str, version: Nullable[SemanticVersion] = None) -> Nullable[str]:

985 """

986 Return the URL of a package's source repository, where one was stated.

987

988 :param packageName: Name of the package.

989 :param version: Optional, the version to answer for. Without one, only a key naming no version answers.

990 :returns: The URL stated, or ``None``.

991 """

992 return self._Lookup(self._repositories, packageName, version)

993

994 def __len__(self) -> int:

995 """

996 Return the number of packages that are overridden.

997

998 :returns: Number of overridden packages.

999 """

1000 return len(set(self._licenses) | set(self._licenseURLs) | set(self._repositories))

1001

1002 def __str__(self) -> str:

1003 """

1004 Return a string representation of these overrides.

1005

1006 :returns: The number of packages that are overridden.

1007 """

1008 return f"LicenseOverrides({len(self)} packages)"

1009

1010

1011@export

1012class LazyLoaderState(IntEnum):

1013 """

1014 Loading states of a lazy-loadable object, in the order they are reached.

1015

1016 The states are ordered, so a loader can be asked for *at least* a given state and does nothing when the object is

1017 already loaded that far.

1018 """

1019 Uninitialized = 0 #: No data or minimal data like ID or name.

1020 Initialized = 1 #: Initialized by some __init__ parameters.

1021 PartiallyLoaded = 2 #: Some additional data was loaded.

1022 FullyLoaded = 3 #: All data is loaded.

1023 PostProcessed = 4 #: Loaded data triggered further processing.

1024

1025

1026@export

1027class lazy:

1028 """

1029 Decorator loading an object far enough, before a method or property of it is used.

1030

1031 The object is a :class:`LazyLoadableMixin`; the decorator passes the state given to it to the object's

1032 ``__lazy_loader__``, unless the object reached that state already. It is applied to a method, or on top of a property:

1033

1034 .. code-block:: Python

1035

1036 @lazy(LazyLoaderState.PartiallyLoaded)

1037 def Method(self) -> None:

1038 ...

1039

1040 @lazy(LazyLoaderState.PartiallyLoaded)

1041 @readonly

1042 def Property(self) -> int:

1043 ...

1044 """

1045

1046 def __init__(self, _requiredState: LazyLoaderState = LazyLoaderState.PartiallyLoaded):

1047 """

1048 Initialize the decorator with the loading state its member needs.

1049

1050 :param _requiredState: Optional, state the object has to be loaded to before the decorated member is used.

1051 """

1052 self._requiredState = _requiredState

1053 self._wrapped = None

1054

1055 def __call__(self, wrapped):

1056 """

1057 Apply the decorator to a method or property.

1058

1059 :param wrapped: The method or property to load lazily.

1060 :returns: The decorator itself, which acts as the descriptor of the decorated member.

1061 """

1062 self._wrapped = wrapped

1063 # If it's a function, we update metadata.

1064 # If it's a property, it doesn't support update_wrapper directly.

1065 if hasattr(wrapped, "__name__"):

1066 update_wrapper(self, wrapped)

1067

1068 return self

1069

1070 def __get__(self, obj, objtype=None):

1071 """

1072 Load the object far enough, then hand out the decorated property's value or a bound method.

1073

1074 :param obj: The object the decorated member is accessed on, or ``None`` for a class access.

1075 :param objtype: Optional, the class the decorated member is accessed on.

1076 :returns: The property's value, a bound wrapper around the method, or the decorator itself.

1077 """

1078 if obj is None:

1079 return self

1080

1081 # 1. Thread-safe state check

1082 with obj.__lazy_lock__:

1083 if obj.__lazy_state__ < self._requiredState:

1084 obj.__lazy_loader__(self._requiredState)

1085

1086 # 2. Determine if we are wrapping a property or a method

1087 if isinstance(self._wrapped, property):

1088 # If it's a property, call its __get__ to return the value

1089 return self._wrapped.__get__(obj, objtype)

1090

1091 # 3. Otherwise, treat as a method and return a bound wrapper

1092 @wraps(self._wrapped)

1093 def wrapper(*args, **kwargs):

1094 """

1095 Nested function binding the decorated method to the object it was accessed on.

1096

1097 :param args: Positional parameters passed to the decorated method.

1098 :param kwargs: Named parameters passed to the decorated method.

1099 :returns: Whatever the decorated method returns.

1100 """

1101 return self._wrapped(obj, *args, **kwargs)

1102

1103 return wrapper

1104

1105

1106@export

1107class LazyLoadableMixin(metaclass=ExtendedType, mixin=True):

1108 """

1109 Mixin-class for objects whose details are fetched on first use.

1110

1111 The object is created from what its creator knows - often little more than a name - and everything else is loaded

1112 when it is needed. The mixin records how far the object is loaded (:attr:`__lazy_state__`) and serializes

1113 concurrent loading (:attr:`__lazy_lock__`); the deriving class implements a ``__lazy_loader__`` method and decides what

1114 loading means.

1115 """

1116 __lazy_state__: LazyLoaderState #: State of the lazy loading process for this object.

1117 __lazy_lock__: RLock #: Lock serializing concurrent lazy loading of this object.

1118

1119 def __init__(self, targetLevel: LazyLoaderState = LazyLoaderState.Initialized) -> None:

1120 """

1121 Initialize the lazy-loading state of an object.

1122

1123 :param targetLevel: Optional, state the object should be loaded to immediately; by default nothing is loaded.

1124 """

1125 self.__lazy_state__ = LazyLoaderState.Initialized

1126 self.__lazy_lock__ = RLock()

1127

1128 if targetLevel > self.__lazy_state__:

1129 with self.__lazy_lock__:

1130 self.__lazy_loader__(targetLevel)

1131

1132 @abstractmethod

1133 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:

1134 """

1135 Load the object's details up to the given state.

1136

1137 :param targetLevel: State the object needs to be loaded to.

1138 """

1139 pass

1140

1141

1142@export

1143class Distribution(metaclass=ExtendedType, slots=True):

1144 """

1145 A single downloadable file of a release - a wheel or a source archive.

1146 """

1147 _filename: str #: Filename of the distribution's file.

1148 _url: URL #: URL to download the distribution's file from.

1149 _uploadTime: datetime #: Time when the distribution was uploaded to the package index.

1150

1151 def __init__(self, filename: str, url: Union[str, URL], uploadTime: datetime) -> None:

1152 """

1153 Initialize a distribution with the data the package index reports for it.

1154

1155 :param filename: Filename of the distribution's file.

1156 :param url: URL to download the file from, as a string or a parsed URL.

1157 :param uploadTime: Time the distribution was uploaded to the package index.

1158 :raises TypeError: If a parameter is not of the expected type.

1159 """

1160 if not isinstance(filename, str): 1160 ↛ 1161line 1160 didn't jump to line 1161 because the condition on line 1160 was never true

1161 ex = TypeError("Parameter 'filename' is not of type 'str'.")

1162 ex.add_note(f"Got type '{getFullyQualifiedName(filename)}'.")

1163 raise ex

1164

1165 self._filename = filename

1166

1167 if isinstance(url, str): 1167 ↛ 1169line 1167 didn't jump to line 1169 because the condition on line 1167 was always true

1168 url = URL.Parse(url)

1169 elif not isinstance(url, URL):

1170 ex = TypeError("Parameter 'url' is not of type 'URL'.")

1171 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")

1172 raise ex

1173

1174 self._url = url

1175

1176 if not isinstance(uploadTime, datetime): 1176 ↛ 1177line 1176 didn't jump to line 1177 because the condition on line 1176 was never true

1177 ex = TypeError("Parameter 'uploadTime' is not of type 'str'.")

1178 ex.add_note(f"Got type '{getFullyQualifiedName(uploadTime)}'.")

1179 raise ex

1180

1181 self._uploadTime = uploadTime

1182

1183 @readonly

1184 def Filename(self) -> str:

1185 """

1186 Read-only property to access the distribution's filename (:attr:`_filename`).

1187

1188 :returns: Filename of the distribution.

1189 """

1190 return self._filename

1191

1192 @readonly

1193 def URL(self) -> URL:

1194 """

1195 Read-only property to access the URL this distribution can be downloaded from (:attr:`_url`).

1196

1197 :returns: Download URL of the distribution.

1198 """

1199 return self._url

1200

1201 @readonly

1202 def UploadTime(self) -> datetime:

1203 """

1204 Read-only property to access the time this distribution was uploaded (:attr:`_uploadTime`).

1205

1206 :returns: Upload time of the distribution.

1207 """

1208 return self._uploadTime

1209

1210 def __repr__(self) -> str:

1211 """

1212 Return a detailed string representation of this distribution.

1213

1214 :returns: The distribution's filename, prefixed by its kind.

1215 """

1216 return f"Distribution: {self._filename}"

1217

1218 def __str__(self) -> str:

1219 """

1220 Return a string representation of this distribution.

1221

1222 :returns: The distribution's filename.

1223 """

1224 return f"{self._filename}"

1225

1226

1227@export

1228class Release(PackageVersion, LazyLoadableMixin):

1229 """

1230 One released version of a project on a Python package index.

1231

1232 A release knows its distributions (the files that can be downloaded) and its requirements, sorted into the extras

1233 they belong to. Both are fetched from the index on first use, which reports the warnings of

1234 :meth:`UpdateDetailsFromPyPIJSON`.

1235 """

1236 _files: list[Distribution] #: Distributions (wheels, source archives) of this release.

1237 _requirements: dict[Nullable[str], list[Requirement]] #: Requirements per extra; ``None`` collects the unconditional ones.

1238

1239 _api: Nullable[URL] #: URL of the package index's API, used to load the release's details.

1240 _session: Nullable[Session] #: HTTP session reused for the API requests.

1241

1242 def __init__(

1243 self,

1244 version: PythonVersion,

1245 timestamp: datetime,

1246 files: Nullable[Iterable[Distribution]] = None,

1247 requirements: Nullable[Mapping[str, list[Requirement]]] = None,

1248 project: Nullable[Project] = None,

1249 lazy: LazyLoaderState = LazyLoaderState.Initialized

1250 ) -> None:

1251 """

1252 Initialize a release of a project.

1253

1254 The API endpoint and the HTTP session are taken from the project's package index, so a release created from a

1255 project can fetch its own details. If ``lazy`` asks for them, they're fetched right away, which reports the

1256 warnings of :meth:`UpdateDetailsFromPyPIJSON`.

1257

1258 :param version: Version number of this release.

1259 :param timestamp: Time this version was released.

1260 :param files: Optional, distributions of this release.

1261 :param requirements: Optional, requirements of this release, by extra.

1262 :param project: Optional, project this release belongs to.

1263 :param lazy: Optional, state the release should be loaded to immediately.

1264 """

1265 if project is not None and (storage := project._storage) is not None: 1265 ↛ 1269line 1265 didn't jump to line 1269 because the condition on line 1265 was always true

1266 self._api = storage._api

1267 self._session = storage._session

1268 else:

1269 self._api = None

1270 self._session = None

1271

1272 super().__init__(version, project, timestamp)

1273 LazyLoadableMixin.__init__(self, lazy)

1274

1275 self._files = list(files) if files is not None else []

1276 self._requirements = dict(requirements) if requirements is not None else {None: []}

1277

1278 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:

1279 """

1280 Download the release's details and post-process them, as far as the target state demands.

1281

1282 A step reached already isn't repeated: a release with its details asked to be post-processed doesn't download

1283 them again.

1284

1285 :param targetLevel: State the release needs to be loaded to.

1286 """

1287 if targetLevel >= LazyLoaderState.PartiallyLoaded and self.__lazy_state__ < LazyLoaderState.PartiallyLoaded: 1287 ↛ 1290line 1287 didn't jump to line 1290 because the condition on line 1287 was always true

1288 self.DownloadDetails()

1289

1290 if targetLevel >= LazyLoaderState.PostProcessed: 1290 ↛ 1291line 1290 didn't jump to line 1291 because the condition on line 1290 was never true

1291 self.PostProcess()

1292

1293 @lazy(LazyLoaderState.PostProcessed)

1294 @PackageVersion.DependsOn.getter

1295 def DependsOn(self) -> dict[Package, dict[SemanticVersion, PackageVersion]]:

1296 """

1297 Read-only property to access the packages this release depends on.

1298

1299 :returns: Dictionary of packages and their versions this release depends on.

1300 """

1301 return super().DependsOn

1302

1303 @readonly

1304 def Project(self) -> Project:

1305 """

1306 Read-only property to access the project this release belongs to (:attr:`_package`).

1307

1308 :returns: The project this release belongs to.

1309 """

1310 return self._package

1311

1312 @lazy(LazyLoaderState.PartiallyLoaded)

1313 @readonly

1314 def Files(self) -> list[Distribution]:

1315 """

1316 Read-only property to access the distributions published for this release (:attr:`_files`).

1317

1318 :returns: List of distributions.

1319 """

1320 return self._files

1321

1322 @lazy(LazyLoaderState.PartiallyLoaded)

1323 @readonly

1324 def Requirements(self) -> dict[str, list[Requirement]]:

1325 """

1326 Read-only property to access the release's requirements, grouped by extra (:attr:`_requirements`).

1327

1328 :returns: Dictionary of extras and their requirements. Requirements without an extra are stored under ``None``.

1329 """

1330 return self._requirements

1331

1332 def _GetPyPIEndpoint(self) -> str:

1333 """

1334 Return the API endpoint describing this release.

1335

1336 :returns: The endpoint's path, relative to the index's API URL.

1337 """

1338 return f"{self._package._name.lower()}/{self._version}/json"

1339

1340 def DownloadDetails(self) -> None:

1341 """

1342 Download this release's details from the package index and load the projects it requires.

1343

1344 Reading the details reports an :exc:`~pyTooling.Dependency.UndeclaredExtraMarkerWarning` or an

1345 :exc:`~pyTooling.Dependency.UnknownLicenseWarning` to a :class:`~pyTooling.Warning.WarningCollector`, see

1346 :meth:`UpdateDetailsFromPyPIJSON`; without a collector, they're dropped.

1347

1348 :raises NoSessionAvailableError: If the release wasn't created by a package index, so it has no session. |br|

1349 A session is opened by the package index and handed to the objects it

1350 creates.

1351 :raises ReleaseNotFoundError: If the index doesn't know this release.

1352 :raises PackageIndexError: If the index responds with another HTTP error status.

1353 """

1354 if self._session is None:

1355 ex = NoSessionAvailableError(

1356 f"No session available to download release '{self._version}' of package '{self._package._name}'.",

1357 projectName=self._package._name,

1358 version=self._version

1359 )

1360 ex.add_note("A session is opened by the package index and handed to the objects it creates.")

1361 raise ex

1362

1363 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")

1364 try:

1365 response.raise_for_status()

1366 except HTTPError as cause:

1367 if cause.response is not None and cause.response.status_code == 404:

1368 raise ReleaseNotFoundError(

1369 f"Release '{self._version}' of package '{self._package._name}' not found.",

1370 projectName=self._package._name,

1371 version=self._version

1372 ) from cause

1373

1374 raise PackageIndexError(

1375 f"Package index responded {response.status_code} {response.reason} for release '{self._version}' of "

1376 f"package '{self._package._name}'.",

1377 projectName=self._package._name,

1378 version=self._version

1379 ) from cause

1380

1381 self.UpdateDetailsFromPyPIJSON(response.json())

1382

1383 index: PythonPackageIndex = self._package._storage

1384 for requirement in self._requirements[None]:

1385 # a project the index reported as unknown before stays a requirement of this release; it isn't registered

1386 if canonicalize_name(requirement.name) not in index._unknownProjects:

1387 index.DownloadProject(requirement.name, LazyLoaderState.Initialized)

1388

1389 def UpdateDetailsFromPyPIJSON(self, json) -> None:

1390 """

1391 Fill this release from the JSON document the package index returned.

1392

1393 The requirements are sorted into the extras they belong to. A requirement lands under ``None`` when it has no

1394 marker at all, and also when its marker conditions it on the *environment* rather than on an extra -

1395 ``importlib-resources; python_version < "3.7"`` is required unconditionally, just not everywhere. A

1396 requirement naming an extra the release does not declare is reported as an

1397 :exc:`~pyTooling.Dependency.UndeclaredExtraMarkerWarning`, and a license that can't be resolved as an

1398 :exc:`~pyTooling.Dependency.UnknownLicenseWarning` (see :meth:`_ResolveLicense`), each to a

1399 :class:`~pyTooling.Warning.WarningCollector`; without a collector, they're dropped. pip doesn't install such a

1400 requirement either: it installs the requirements of the extras a release declares only.

1401

1402 Metadata older than core-metadata 2.1 has no ``provides_extra`` field. Its extras are recovered from the

1403 markers naming them, because dropping every conditional requirement of an old release would empty exactly the

1404 releases a version-aware dependency graph is built to look at. A declared extra keeps the spelling it was

1405 declared with; a recovered one has no such spelling, so it keeps the canonical one - ``theme_furo`` written in

1406 a marker is recovered as ``theme-furo``.

1407

1408 :param json: The parsed JSON document describing this release.

1409 """

1410 infoNode = json["info"]

1411 self._ResolveLicense(infoNode)

1412 self._ResolveURLs(infoNode)

1413

1414 requirements = [Requirement(requirement) for requirement in (infoNode["requires_dist"] or ())]

1415

1416 # The declared spelling is kept as the key, while the canonical one - 'code_style' and 'code-style' are the

1417 # same extra - is what a marker is matched against.

1418 if (declaredExtras := infoNode["provides_extra"]) is not None:

1419 extras = {canonicalize_name(extra): extra for extra in declaredExtras}

1420 else:

1421 extras = {}

1422 for requirement in requirements:

1423 if requirement.marker is not None:

1424 for name in _EXTRA_MARKER.findall(str(requirement.marker)):

1425 extras.setdefault(canonicalize_name(name), name)

1426

1427 self._requirements = {extra: [] for extra in extras.values()}

1428 self._requirements[None] = []

1429

1430 if len(requirements) > 0:

1431 brokenRequirements = []

1432 for req in requirements:

1433 # A marker naming no extra conditions the requirement on the interpreter or the platform, so the

1434 # requirement is unconditional as far as the extras are concerned.

1435 if req.marker is None or len(namedExtras := _EXTRA_MARKER.findall(str(req.marker))) == 0:

1436 self._requirements[None].append(req)

1437 continue

1438

1439 for name in namedExtras:

1440 if (extra := extras.get(canonicalize_name(name))) is not None:

1441 self._requirements[extra].append(req)

1442 break

1443 else:

1444 brokenRequirements.append(req)

1445

1446 if len(brokenRequirements) > 0:

1447 WarningCollector.Raise(

1448 UndeclaredExtraMarkerWarning(

1449 f"Package '{self._package._name}' has {len(brokenRequirements)} requirement(s) whose marker matches no "

1450 "declared extra.",

1451 release=self,

1452 requirements=brokenRequirements

1453 ),

1454 notes=[f"Broken requirement: {req}" for req in brokenRequirements]

1455 )

1456 # Preserving the broken requirements under the special index 0 makes 'Requirements' a dictionary of mixed

1457 # key types (str, None and int), which no consumer expects.

1458 # self._requirements[0] = brokenRequirements

1459

1460 self.__lazy_state__ = LazyLoaderState.FullyLoaded

1461

1462 def _ResolveLicense(self, infoNode: Mapping[str, Any]) -> None:

1463 """

1464 Resolve this release's license from what was published, and from what was stated by hand.

1465

1466 The sources are consulted in order of how much they can be trusted, and the first one that answers wins:

1467

1468 1. the :class:`LicenseOverrides` of the package index - an explicit statement always wins,

1469 2. ``license_expression``, the PEP 639 field, which is an SPDX expression by definition,

1470 3. ``license``, the legacy free-text field - it is handed to the parser like any other candidate, and a field

1471 holding the license's full text simply doesn't parse,

1472 4. a license classifier, but only when it means exactly one license - ``License :: OSI Approved :: BSD

1473 License`` means either ``BSD-2-Clause`` or ``BSD-3-Clause`` and is never guessed at.

1474

1475 ``License :: Other/Proprietary License`` is the classifier that resolves without parsing anything: SPDX has

1476 no identifier for a license that isn't published, so it becomes a

1477 :class:`~pyTooling.Licensing.ProprietaryLicense` and the classifier itself is what

1478 :attr:`~pyTooling.Dependency.PackageVersion.PublishedLicense` reports.

1479

1480 Whatever was found is parsed into a :class:`~pyTooling.Licensing.LicenseExpression` and kept verbatim in

1481 :attr:`~pyTooling.Dependency.PackageVersion.PublishedLicense`, even when it doesn't parse. A release whose

1482 license stays unresolved is reported as an

1483 :class:`~pyTooling.Dependency.UnknownLicenseWarning` naming what was published, because that is the list of

1484 packages the override file has to answer for; without a :class:`~pyTooling.Warning.WarningCollector`, it's

1485 dropped. A release publishing ``NOASSERTION`` or ``NONE`` is on that list too - the *statement* resolved, the

1486 license is still unknown. An override that doesn't parse is reported the same way, naming the override file and

1487 its statement, because that is the file to correct; the published metadata isn't read in its place.

1488

1489 :param infoNode: The ``info`` node of the JSON document describing this release.

1490 """

1491 index: PythonPackageIndex = self._package._storage

1492 overrides = index._licenseOverrides

1493 published = []

1494 candidates = []

1495 proprietaryClassifier = None

1496

1497 overrideRejected = False

1498 if (override := overrides.LicenseOf(self._package._name, self._version)) is not None:

1499 candidates.append(override)

1500 else:

1501 if (licenseExpression := infoNode.get("license_expression", None)) is not None:

1502 published.append(f"license_expression: {licenseExpression}")

1503 candidates.append(licenseExpression)

1504 elif (licenseText := (infoNode.get("license", None) or "").strip()) != "":

1505 published.append(f"license: {licenseText[:_LICENSE_NOTE_LENGTH]}")

1506 candidates.append(licenseText)

1507

1508 for classifier in infoNode.get("classifiers", None) or ():

1509 if not classifier.startswith("License ::"):

1510 continue

1511

1512 published.append(f"classifier: {classifier}")

1513 if classifier == _PROPRIETARY_CLASSIFIER:

1514 proprietaryClassifier = classifier

1515 elif len(matches := LICENSES_BY_CLASSIFIER.get(classifier, ())) == 1:

1516 candidates.append(matches[0].SPDXIdentifier)

1517

1518 break

1519

1520 for candidate in candidates:

1521 try:

1522 self._licenseExpression = LicenseExpression.Parse(candidate)

1523 except (LicenseExpressionError, ValueError):

1524 continue

1525

1526 # The expression keeps the candidate as its 'OriginalText', which is the only place it is held.

1527 break

1528 else:

1529 # An override is the only candidate, so nothing resolving means the override was rejected.

1530 overrideRejected = override is not None

1531 if proprietaryClassifier is not None:

1532 # SPDX has no identifier for a proprietary license, so there is nothing to parse - the node is built,

1533 # and it carries the classifier it was built from.

1534 self._licenseExpression = ProprietaryLicense(originalText=proprietaryClassifier)

1535 else:

1536 # Nothing resolved. 'NOASSERTION' is what SPDX says for that, and the node keeps what was published.

1537 self._licenseExpression = UnknownLicense(

1538 LicenseAbsence.NoAssertion,

1539 candidates[0] if len(candidates) > 0 else ""

1540 )

1541

1542 if (licenseURL := overrides.LicenseURLOf(self._package._name)) is not None:

1543 self._licenseURL = URL.Parse(licenseURL)

1544

1545 # 'UnknownLicense' covers both: the index said 'NOASSERTION' itself, and nothing resolved at all. Either way

1546 # the license is unknown, which is what this list is for.

1547 if isinstance(self._licenseExpression, UnknownLicense):

1548 if overrideRejected:

1549 source = f"license override file '{overrides._path}'" if overrides._path is not None else "license overrides"

1550 notes = [f"'{override}', stated by the {source}, isn't a license expression 'pyTooling.Licensing' knows."]

1551 elif len(published) > 0:

1552 notes = published

1553 else:

1554 notes = ["The package index published no license information."]

1555

1556 WarningCollector.Raise(

1557 UnknownLicenseWarning(

1558 f"License of '{self._package._name}' {self._version} couldn't be resolved.",

1559 projectName=self._package._name,

1560 version=self._version

1561 ),

1562 notes=notes

1563 )

1564

1565 def _ResolveURLs(self, infoNode: Mapping[str, Any]) -> None:

1566 """

1567 Resolve this release's project URLs from what the package index published.

1568

1569 ``project_urls`` is a free-text mapping - one project writes ``Source``, another ``Source Code``,

1570 ``Repository`` or ``GitHub`` for the same thing - so its keys are matched case-insensitively against a list of

1571 aliases per URL and the first alias present wins. ``home_page`` is the fallback for the homepage, and the

1572 homepage is the last resort for the repository, which is what the project-level resolver used to do.

1573

1574 These are resolved per release rather than per package because they move: a project migrating from Google

1575 Code or SourceForge to GitHub has one repository URL before the migration and another after it.

1576 :attr:`~pyTooling.Dependency.Package.RepositoryURL` and its siblings mirror the latest release, so the

1577 package still answers for the current state.

1578

1579 :param infoNode: The ``info`` node of the JSON document describing this release.

1580 """

1581 projectURLs = {

1582 str(key).strip().lower(): value

1583 for key, value in (infoNode.get("project_urls", None) or {}).items()

1584 if value is not None

1585 }

1586

1587 def urlFor(aliases: tuple[str, ...]) -> Nullable[URL]:

1588 """

1589 Return the first of the given aliases the package index published a URL for.

1590

1591 :param aliases: The keys to look for, most specific first.

1592 :returns: The URL of the first alias that is present, or ``None`` if none of them is.

1593 """

1594 for alias in aliases:

1595 if (url := projectURLs.get(alias, None)) is not None:

1596 return URL.Parse(url)

1597

1598 return None

1599

1600 self._documentationURL = urlFor(_DOCUMENTATION_URL_ALIASES)

1601 self._issueTrackerURL = urlFor(_ISSUE_TRACKER_URL_ALIASES)

1602 self._changelogURL = urlFor(_CHANGELOG_URL_ALIASES)

1603 self._projectURL = urlFor(_PROJECT_URL_ALIASES)

1604

1605 if self._projectURL is None and (homePage := infoNode.get("home_page", None)) is not None:

1606 self._projectURL = URL.Parse(homePage)

1607

1608 index: PythonPackageIndex = self._package._storage

1609 if (repository := index._licenseOverrides.RepositoryOf(self._package._name)) is not None:

1610 self._repositoryURL = URL.Parse(repository)

1611 elif (repositoryURL := urlFor(_REPOSITORY_URL_ALIASES)) is not None:

1612 self._repositoryURL = repositoryURL

1613 else:

1614 self._repositoryURL = self._projectURL

1615

1616 def PostProcess(self) -> None:

1617 """

1618 Resolve this release's requirements into dependencies on concrete releases.

1619

1620 Every required project is downloaded and the releases matching the requirement's specifier are attached as

1621 dependencies of this release.

1622 """

1623 index: PythonPackageIndex = self._package._storage

1624 for requirement in self._requirements[None]:

1625 package = index.DownloadProject(requirement.name)

1626

1627 for release in package:

1628 if str(release._version) in requirement.specifier:

1629 self.AddDependencyToPackageVersion(release)

1630

1631 self.SortDependencies()

1632 self.__lazy_state__ = LazyLoaderState.PostProcessed

1633

1634 @lazy(LazyLoaderState.PartiallyLoaded)

1635 def __repr__(self) -> str:

1636 """

1637 Return a detailed string representation of this release, loading its details if needed.

1638

1639 :returns: Package name, version and the number of distributions.

1640 """

1641 return f"Release: {self._package._name}:{self._version} Files: {len(self._files)}"

1642

1643 def __str__(self) -> str:

1644 """

1645 Return a string representation of this release.

1646

1647 :returns: The release's version number.

1648 """

1649 return f"{self._version}"

1650

1651

1652@export

1653class Project(Package, LazyLoadableMixin):

1654 """

1655 A project (package) on a Python package index, with its releases.

1656

1657 The list of releases and the project's details are fetched from the index on first use.

1658 """

1659 _url: Nullable[URL] #: URL of the project's page on the package index.

1660

1661 _api: Nullable[URL] #: URL of the package index's API, used to load the project's details.

1662 _session: Nullable[Session] #: HTTP session reused for the API requests.

1663

1664 def __init__(

1665 self,

1666 name: str,

1667 url: Union[str, URL],

1668 releases: Nullable[Iterable[Release]] = None,

1669 index: Nullable[PythonPackageIndex] = None,

1670 lazy: LazyLoaderState = LazyLoaderState.Initialized

1671 ) -> None:

1672 """

1673 Initialize a project on a package index.

1674

1675 The API endpoint and the HTTP session are taken from the index, so the project can fetch its own details. If

1676 ``lazy`` asks for :attr:`~pyTooling.Dependency.Python.LazyLoaderState.PostProcessed`, the details of every

1677 release are fetched right away, which reports the warnings of :meth:`DownloadReleaseDetails`.

1678

1679 :param name: Name of the project on the package index.

1680 :param url: URL of the project's page, as a string or a parsed URL.

1681 :param releases: Optional, releases of this project.

1682 :param index: Optional, package index this project is hosted on.

1683 :param lazy: Optional, state the project should be loaded to immediately.

1684 :raises UnhandledCriticalWarningError: If ``lazy`` asks for

1685 :attr:`~pyTooling.Dependency.Python.LazyLoaderState.PostProcessed`, a

1686 release's details can't be downloaded or read, and no

1687 :class:`~pyTooling.Warning.WarningCollector` collects the

1688 :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`.

1689 """

1690 if index is not None: 1690 ↛ 1694line 1690 didn't jump to line 1694 because the condition on line 1690 was always true

1691 self._api = index._api

1692 self._session = index._session

1693 else:

1694 self._api = None

1695 self._session = None

1696

1697 super().__init__(name, storage=index)

1698 LazyLoadableMixin.__init__(self, lazy)

1699

1700 # if isinstance(url, str):

1701 # url = URL.Parse(url)

1702 # elif not isinstance(url, URL):

1703 # ex = TypeError("Parameter 'url' is not of type 'URL'.")

1704 # ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")

1705 # raise ex

1706 #

1707 # self._url = url

1708 # self._releases = {release.Version: release for release in sorted(releases, key=lambda r: r.Version)} if releases is not None else {}

1709

1710 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:

1711 """

1712 Download the project's details and its releases' details, as far as the target state demands.

1713

1714 A step reached already isn't repeated: a project with its list of releases asked for its releases' details

1715 doesn't download the list again.

1716

1717 :param targetLevel: State the project needs to be loaded to.

1718 :raises UnhandledCriticalWarningError: If the target state is

1719 :attr:`~pyTooling.Dependency.Python.LazyLoaderState.PostProcessed`, a

1720 release's details can't be downloaded or read, and no

1721 :class:`~pyTooling.Warning.WarningCollector` collects the

1722 :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`.

1723 """

1724 if targetLevel >= LazyLoaderState.PartiallyLoaded and self.__lazy_state__ < LazyLoaderState.PartiallyLoaded:

1725 self.DownloadDetails()

1726

1727 if targetLevel >= LazyLoaderState.PostProcessed: 1727 ↛ 1728line 1727 didn't jump to line 1728 because the condition on line 1727 was never true

1728 self.DownloadReleaseDetails()

1729

1730 @readonly

1731 def PackageIndex(self) -> PythonPackageIndex:

1732 """

1733 Read-only property to access the package index this project was read from (:attr:`_storage`).

1734

1735 :returns: The package index this project belongs to.

1736 """

1737 return self._storage

1738

1739 @lazy(LazyLoaderState.PartiallyLoaded)

1740 @readonly

1741 def URL(self) -> URL:

1742 """

1743 Read-only property to access the project's URL in the package index (:attr:`_url`).

1744

1745 :returns: URL of the project.

1746 """

1747 return self._url

1748

1749 @lazy(LazyLoaderState.PartiallyLoaded)

1750 @readonly

1751 def Releases(self) -> dict[PythonVersion, Release]:

1752 """

1753 Read-only property to access all known releases of this project (:attr:`_versions`).

1754

1755 :returns: Dictionary of versions and their releases.

1756 """

1757 return self._versions

1758

1759 @lazy(LazyLoaderState.PartiallyLoaded)

1760 @readonly

1761 def ReleaseCount(self) -> int:

1762 """

1763 Read-only property to return the number of known releases.

1764

1765 :returns: Number of releases.

1766 """

1767 return len(self._versions)

1768

1769 @lazy(LazyLoaderState.PartiallyLoaded)

1770 @readonly

1771 def LatestRelease(self) -> Release:

1772 """

1773 Read-only property to return the most recent release of this project.

1774

1775 :returns: The latest release.

1776 """

1777 return firstValue(self._versions)

1778

1779 def _GetPyPIEndpoint(self) -> str:

1780 """

1781 Return the API endpoint describing this project.

1782

1783 :returns: The endpoint's path, relative to the index's API URL.

1784 """

1785 return f"{self._name.lower()}/json"

1786

1787 def DownloadDetails(self) -> None:

1788 """

1789 Download this project's details and its list of releases from the package index.

1790

1791 A release whose version the parser doesn't support is skipped and reported as a

1792 :exc:`~pyTooling.Dependency.UnsupportedVersionWarning`, see :meth:`UpdateDetailsFromPyPIJSON`. If this project is

1793 loaded already, the releases fetch their details right away, which reports the warnings of

1794 :meth:`Release.UpdateDetailsFromPyPIJSON`.

1795

1796 :raises NoSessionAvailableError: If the project wasn't created by a package index, so it has no session. |br|

1797 A session is opened by the package index and handed to the objects it

1798 creates.

1799 :raises ProjectNotFoundError: If the index doesn't know this project.

1800 :raises PackageIndexError: If the index responds with another HTTP error status.

1801 :raises UnhandledCriticalWarningError: If a release's version isn't supported and no

1802 :class:`~pyTooling.Warning.WarningCollector` collects the warning.

1803 """

1804 if self._session is None:

1805 ex = NoSessionAvailableError(

1806 f"No session available to download details of package '{self._name}'.",

1807 projectName=self._name

1808 )

1809 ex.add_note("A session is opened by the package index and handed to the objects it creates.")

1810 raise ex

1811

1812 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")

1813 try:

1814 response.raise_for_status()

1815 except HTTPError as cause:

1816 if cause.response is not None and cause.response.status_code == 404:

1817 raise ProjectNotFoundError(f"Package '{self._name}' not found.", projectName=self._name) from cause

1818

1819 raise PackageIndexError(

1820 f"Package index responded {response.status_code} {response.reason} for package '{self._name}'.",

1821 projectName=self._name

1822 ) from cause

1823

1824 self.UpdateDetailsFromPyPIJSON(response.json())

1825

1826 def UpdateDetailsFromPyPIJSON(self, json) -> None:

1827 """

1828 Fill this project from the JSON document the package index returned.

1829

1830 Releases without a distribution are skipped. A release whose version the parser doesn't support is skipped too,

1831 rather than failing the whole project, and reported as a :exc:`~pyTooling.Dependency.UnsupportedVersionWarning`

1832 to a :class:`~pyTooling.Warning.WarningCollector`. It's a critical warning: without a collector,

1833 :exc:`~pyTooling.Warning.UnhandledCriticalWarningError` is raised. If this project is loaded already, the releases

1834 fetch their details right away, which reports the warnings of :meth:`Release.UpdateDetailsFromPyPIJSON`.

1835

1836 :param json: The parsed JSON document describing this project.

1837 :raises UnhandledCriticalWarningError: If a release's version isn't supported and no

1838 :class:`~pyTooling.Warning.WarningCollector` collects the warning.

1839 """

1840 infoNode = json["info"]

1841 releasesNode = json["releases"]

1842

1843 # Update project/package URL

1844 self._url = URL.Parse(infoNode["project_url"])

1845

1846 # Convert key to Version number, skip empty releases

1847 convertedReleasesNode = {}

1848 for k, v in releasesNode.items():

1849 if len(v) == 0: 1849 ↛ 1850line 1849 didn't jump to line 1850 because the condition on line 1849 was never true

1850 continue

1851

1852 try:

1853 version = PythonVersion.Parse(k)

1854 convertedReleasesNode[version] = v

1855 except ValueError as ex:

1856 WarningCollector.Raise(

1857 UnsupportedVersionWarning(

1858 f"Skipping release '{k}' of package '{self._name}': version format isn't supported.",

1859 projectName=self._name,

1860 version=k

1861 ),

1862 ex

1863 )

1864

1865 for version, releaseNode in sorted(convertedReleasesNode.items(), key=lambda t: t[0]):

1866 if Parts.Postfix in version._parts: 1866 ↛ 1867line 1866 didn't jump to line 1867 because the condition on line 1866 was never true

1867 pass

1868

1869 files = [Distribution(file["filename"], file["url"], datetime.fromisoformat(file["upload_time_iso_8601"]), ) for

1870 file in releaseNode]

1871 lazy = LazyLoaderState.PartiallyLoaded if LazyLoaderState.PartiallyLoaded <= self.__lazy_state__ <= LazyLoaderState.FullyLoaded else LazyLoaderState.Initialized

1872 Release(

1873 version,

1874 files[0]._uploadTime,

1875 files,

1876 project=self,

1877 lazy=lazy

1878 )

1879

1880 self.SortVersions()

1881 self.__lazy_state__ = LazyLoaderState.FullyLoaded

1882

1883 def DownloadReleaseDetails(self) -> None:

1884 """

1885 Download the details of every release of this project, in parallel.

1886

1887 The requests run in one :mod:`asyncio` event loop over a shared session, because a project can easily have

1888 hundreds of releases.

1889

1890 The event loop runs in the calling thread, so the warnings of :meth:`Release.UpdateDetailsFromPyPIJSON` reach the

1891 caller's :class:`~pyTooling.Warning.WarningCollector`. A release whose details can't be downloaded or read is

1892 dropped and reported as a critical :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`. An exception raised while

1893 reading a release's details doesn't propagate: the release is dropped and the exception becomes the cause of its

1894 :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`. That's also the case for an

1895 :exc:`~pyTooling.Warning.EscalatedWarningError` from a collector's handler.

1896

1897 Without a collector, the first :exc:`~pyTooling.Dependency.ReleaseDetailsWarning` raises

1898 :exc:`~pyTooling.Warning.UnhandledCriticalWarningError`, after every download finished. No release is dropped

1899 then.

1900

1901 :raises UnhandledCriticalWarningError: If a release's details can't be downloaded or read, and no

1902 :class:`~pyTooling.Warning.WarningCollector` collects the

1903 :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`.

1904 """

1905 async def ParallelDownloadReleaseDetails():

1906 """

1907 Nested coroutine downloading the details of every release over one shared session.

1908 """

1909 async def routine(session, release: Release):

1910 """

1911 Nested coroutine downloading the details of a single release.

1912

1913 :param session: The HTTP session shared by all requests of this download.

1914 :param release: The release to download the details for.

1915 """

1916 if Parts.Postfix in release._version._parts: 1916 ↛ 1917line 1916 didn't jump to line 1917 because the condition on line 1916 was never true

1917 pass

1918

1919 async with session.get(release._GetPyPIEndpoint()) as response:

1920 json = await response.json()

1921 response.raise_for_status()

1922

1923 release.UpdateDetailsFromPyPIJSON(json)

1924

1925 async with ClientSession(base_url=str(self._api), headers={"accept": "application/json"}) as session:

1926 tasks = []

1927 for release in self._versions.values(): # type: Release

1928 tasks.append(routine(session, release))

1929

1930 results = await asyncio_gather(*tasks, return_exceptions=True)

1931 delList = []

1932 for release, result in zip(self.Releases.values(), results):

1933 if isinstance(result, Exception):

1934 delList.append((release, result))

1935

1936 for release, ex in delList:

1937 WarningCollector.Raise(

1938 ReleaseDetailsWarning(

1939 f"Dropping release '{release.Version}' of package '{release.Project._name}': details couldn't be "

1940 "downloaded.",

1941 projectName=release.Project._name,

1942 version=release.Version

1943 ),

1944 ex

1945 )

1946 del self.Releases[release.Version]

1947

1948 asyncio_run(ParallelDownloadReleaseDetails())

1949 self.__lazy_state__ = LazyLoaderState.PostProcessed

1950

1951 def __repr__(self) -> str:

1952 """

1953 Return a detailed string representation of this project.

1954

1955 :returns: The project's name and its latest release's version.

1956 """

1957 return f"Project: {self._name} latest: {self.LatestRelease._version}"

1958

1959 def __str__(self) -> str:

1960 """

1961 Return a string representation of this project.

1962

1963 :returns: The project's name.

1964 """

1965 return f"{self._name}"

1966

1967

1968@export

1969class PythonPackageIndex(PackageStorage):

1970 """

1971 A Python package index like PyPI, addressed through its JSON API.

1972

1973 It is the entry point of the dependency graph: projects are looked up here, and every request to the index reuses

1974 the same HTTP session.

1975 """

1976 _url: URL #: URL of the package index's website.

1977 _api: URL #: URL of the package index's API.

1978 _session: Session #: HTTP session reused for every request to this index.

1979 _licenseOverrides: LicenseOverrides #: Licenses stated by hand, for what this index can't answer for.

1980 _projectCache: dict[str, Project] #: Projects looked up on this index, by canonical name.

1981 _unknownProjects: set[str] #: Canonical names of the projects this index doesn't know.

1982

1983 def __init__(

1984 self,

1985 name: str,

1986 url: Union[str, URL],

1987 api: Union[str, URL],

1988 graph: PackageDependencyGraph,

1989 licenseOverrides: Nullable[LicenseOverrides] = None

1990 ) -> None:

1991 """

1992 Initialize a package index and open the HTTP session used for every request to it.

1993

1994 :param name: Name of the package index.

1995 :param url: URL of the index's website, as a string or a parsed URL.

1996 :param api: URL of the index's JSON API, as a string or a parsed URL.

1997 :param graph: Dependency graph this index belongs to.

1998 :param licenseOverrides: Optional, licenses stated by hand; an empty set of overrides if not given.

1999 :raises TypeError: If parameter 'url' is neither a string nor a :class:`~pyTooling.GenericPath.URL.URL`.

2000 :raises TypeError: If parameter 'api' is neither a string nor a :class:`~pyTooling.GenericPath.URL.URL`.

2001 """

2002 super().__init__(name, graph)

2003

2004 self._licenseOverrides = licenseOverrides if licenseOverrides is not None else LicenseOverrides()

2005 self._projectCache = {}

2006 self._unknownProjects = set()

2007

2008 if isinstance(url, str): 2008 ↛ 2010line 2008 didn't jump to line 2010 because the condition on line 2008 was always true

2009 url = URL.Parse(url)

2010 elif not isinstance(url, URL):

2011 ex = TypeError("Parameter 'url' is not of type 'URL'.")

2012 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")

2013 raise ex

2014

2015 self._url = url

2016

2017 if isinstance(api, str): 2017 ↛ 2019line 2017 didn't jump to line 2019 because the condition on line 2017 was always true

2018 api = URL.Parse(api)

2019 elif not isinstance(api, URL):

2020 ex = TypeError("Parameter 'api' is not of type 'URL'.")

2021 ex.add_note(f"Got type '{getFullyQualifiedName(api)}'.")

2022 raise ex

2023

2024 self._api = api

2025

2026 self._session = Session()

2027 self._session.headers["accept"] = "application/json"

2028

2029 # A package index is a third party reached over the Internet, and a documentation build that resolves a

2030 # hundred packages will meet a reset connection or a rate-limit eventually.

2031 adapter = HTTPAdapter(max_retries=Retry(

2032 total=RETRY_ATTEMPTS,

2033 backoff_factor=RETRY_BACKOFF,

2034 status_forcelist=RETRY_STATUS_CODES,

2035 ))

2036 self._session.mount("https://", adapter)

2037 self._session.mount("http://", adapter)

2038

2039 @readonly

2040 def URL(self) -> URL:

2041 """

2042 Read-only property to access the package index' base URL (:attr:`_url`).

2043

2044 :returns: Base URL of the package index.

2045 """

2046 return self._url

2047

2048 @readonly

2049 def LicenseOverrides(self) -> LicenseOverrides:

2050 """

2051 Read-only property to access the licenses stated by hand for this index (:attr:`_licenseOverrides`).

2052

2053 :returns: The index's license overrides.

2054 """

2055 return self._licenseOverrides

2056

2057 @readonly

2058 def API(self) -> URL:

2059 """

2060 Read-only property to access the package index' API URL (:attr:`_api`).

2061

2062 :returns: API URL of the package index.

2063 """

2064 return self._api

2065

2066 @readonly

2067 def Projects(self) -> dict[str, Project]:

2068 """

2069 Read-only property to access all projects known to this package index (:attr:`_packages`).

2070

2071 :returns: Dictionary of project names and projects.

2072 """

2073 return self._packages

2074

2075 @readonly

2076 def ProjectCount(self) -> int:

2077 """

2078 Read-only property to return the number of known projects.

2079

2080 :returns: Number of projects.

2081 """

2082 return len(self._packages)

2083

2084 @readonly

2085 def UnknownProjects(self) -> tuple[str, ...]:

2086 """

2087 Read-only property to return the canonical names of the projects this index answered ``404 Not Found`` for.

2088

2089 :meth:`DownloadProject` doesn't ask the index for them again.

2090

2091 :returns: Tuple of canonical project names.

2092 """

2093 return tuple(self._unknownProjects)

2094

2095 def _GetPyPIEndpoint(self, projectName: str) -> str:

2096 """

2097 Return the API endpoint describing a project.

2098

2099 :param projectName: Name of the project on the package index.

2100 :returns: The endpoint's URL.

2101 """

2102 return f"{self._api}{projectName.lower()}/json"

2103

2104 def DownloadProject(self, projectName: str, lazy: LazyLoaderState = LazyLoaderState.PartiallyLoaded) -> Project:

2105 """

2106 Look up a project on this package index.

2107

2108 The index downloads a project once: a later call, also with another spelling of the name (``Sphinx_RTD_Theme``

2109 and ``sphinx-rtd-theme``), returns the same project, loaded further if ``lazy`` asks for more than it has. A

2110 project the index answers ``404 Not Found`` for is remembered (:attr:`UnknownProjects`) and raises again

2111 without a request.

2112

2113 Loading the project's details skips a release whose version the parser doesn't support and reports it as a

2114 :exc:`~pyTooling.Dependency.UnsupportedVersionWarning`, see :meth:`Project.UpdateDetailsFromPyPIJSON`. If

2115 ``lazy`` asks for :attr:`~pyTooling.Dependency.Python.LazyLoaderState.PostProcessed`, the details of every

2116 release are fetched right away, which reports the warnings of :meth:`Project.DownloadReleaseDetails`.

2117

2118 :param projectName: Name of the project on the package index.

2119 :param lazy: Optional, state the project should be loaded to immediately.

2120 :returns: The project, loaded as far as ``lazy`` demands.

2121 :raises UnhandledCriticalWarningError: If ``lazy`` asks for the project's details, a release's version isn't

2122 supported and no :class:`~pyTooling.Warning.WarningCollector` collects

2123 the warning.

2124 :raises ProjectNotFoundError: If the index doesn't know the project - also when it answered so before,

2125 without asking it again.

2126 :raises UnhandledCriticalWarningError: If ``lazy`` asks for

2127 :attr:`~pyTooling.Dependency.Python.LazyLoaderState.PostProcessed`, a

2128 release's details can't be downloaded or read, and no

2129 :class:`~pyTooling.Warning.WarningCollector` collects the

2130 :exc:`~pyTooling.Dependency.ReleaseDetailsWarning`.

2131 """

2132 key = canonicalize_name(projectName)

2133 if key in self._unknownProjects:

2134 ex = ProjectNotFoundError(f"Package '{projectName}' not found.", projectName=projectName)

2135 ex.add_note("The package index answered '404 Not Found' for it before; it isn't asked again.")

2136 raise ex

2137

2138 if (project := self._projectCache.get(key, None)) is None:

2139 try:

2140 project = Project(projectName, "", index=self, lazy=lazy)

2141 except ProjectNotFoundError:

2142 # the project registered itself in the index before its download failed

2143 self._packages.pop(projectName, None)

2144 self._unknownProjects.add(key)

2145 raise

2146

2147 self._projectCache[key] = project

2148 else:

2149 try:

2150 with project.__lazy_lock__:

2151 if project.__lazy_state__ < lazy:

2152 project.__lazy_loader__(lazy)

2153 except ProjectNotFoundError:

2154 self._ForgetProject(project)

2155 raise

2156

2157 return project

2158

2159 def _ForgetProject(self, project: Project) -> None:

2160 """

2161 Remove a project the index answered ``404 Not Found`` for, and remember its name as unknown.

2162

2163 :param project: The project, which registered itself in this index before its download failed.

2164 """

2165 key = canonicalize_name(project._name)

2166 self._packages.pop(project._name, None)

2167 self._projectCache.pop(key, None)

2168 self._unknownProjects.add(key)

2169

2170 def __repr__(self) -> str:

2171 """

2172 Return a detailed string representation of this package index.

2173

2174 :returns: The index's name.

2175 """

2176 return f"{self._name}"

2177

2178 def __str__(self) -> str:

2179 """

2180 Return a string representation of this package index.

2181

2182 :returns: The index's name.

2183 """

2184 return f"{self._name}"

2185

2186

2187@export

2188class PythonPackageDependencyGraph(PackageDependencyGraph):

2189 """

2190 A dependency graph of Python packages, whose vertices are projects and whose edges are requirements.

2191 """

2192

2193 def __init__(self, name: str) -> None:

2194 """

2195 Initialize an empty dependency graph of Python packages.

2196

2197 :param name: Name of the dependency graph.

2198 """

2199 super().__init__(name)