Coverage for pyTooling/Dependency/Python.py: 67%
320 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-31 07:24 +0000
1# ==================================================================================================================== #
2# _____ _ _ ____ _ #
3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ | _ \ ___ _ __ ___ _ __ __| | ___ _ __ ___ _ _ #
4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | | | |/ _ \ '_ \ / _ \ '_ \ / _` |/ _ \ '_ \ / __| | | | #
5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| |_| | __/ |_) | __/ | | | (_| | __/ | | | (__| |_| | #
6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/ \___| .__/ \___|_| |_|\__,_|\___|_| |_|\___|\__, | #
7# |_| |___/ |___/ |_| |___/ #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2025-2026 Patrick Lehmann - Bötzingen, Germany #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""
32Implementation of package dependencies.
34.. hint::
36 See :ref:`high-level help <DEPENDENCIES>` for explanations and usage examples.
37"""
38from asyncio import run as asyncio_run, gather as asyncio_gather
39from datetime import datetime
40from enum import IntEnum
41from functools import wraps, update_wrapper
42from threading import RLock
43from typing import Optional as Nullable, List, Dict, Union, Iterable, Mapping
45try:
46 from aiohttp import ClientSession
47except ImportError as ex: # pragma: no cover
48 raise Exception(f"Optional dependency 'aiohttp' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'aiohttp' directly.") from ex
50try:
51 from packaging.requirements import Requirement
52except ImportError as ex: # pragma: no cover
53 raise Exception(f"Optional dependency 'packaging' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'packaging' directly.") from ex
55try:
56 from requests import Session, HTTPError
57except ImportError as ex: # pragma: no cover
58 raise Exception(f"Optional dependency 'requests' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'requests' directly.") from ex
60from pyTooling.Decorators import export, readonly
61from pyTooling.MetaClasses import ExtendedType, abstractmethod
62from pyTooling.Common import getFullyQualifiedName, firstValue
63from pyTooling.Dependency import Package, PackageStorage, PackageVersion, PackageDependencyGraph
64from pyTooling.Dependency import BrokenRequirementWarning, NoSessionAvailableException, ProjectNotFoundException
65from pyTooling.Dependency import ReleaseDetailsWarning, ReleaseNotFoundException
66from pyTooling.Warning import WarningCollector
67from pyTooling.GenericPath.URL import URL
68from pyTooling.Versioning import SemanticVersion, PythonVersion, Parts
71@export
72class LazyLoaderState(IntEnum):
73 Uninitialized = 0 #: No data or minimal data like ID or name.
74 Initialized = 1 #: Initialized by some __init__ parameters.
75 PartiallyLoaded = 2 #: Some additional data was loaded.
76 FullyLoaded = 3 #: All data is loaded.
77 PostProcessed = 4 #: Loaded data triggered further processing.
80@export
81class lazy:
82 """
83 Unified decorator that supports:
84 1. @lazy(state) def method()
85 2. @lazy(state) @property def prop()
86 """
88 def __init__(self, _requiredState: LazyLoaderState = LazyLoaderState.PartiallyLoaded):
89 self._requiredState = _requiredState
90 self._wrapped = None
92 def __call__(self, wrapped):
93 self._wrapped = wrapped
94 # If it's a function, we update metadata.
95 # If it's a property, it doesn't support update_wrapper directly.
96 if hasattr(wrapped, "__name__"):
97 update_wrapper(self, wrapped)
99 return self
101 def __get__(self, obj, objtype=None):
102 if obj is None: 102 ↛ 103line 102 didn't jump to line 103 because the condition on line 102 was never true
103 return self
105 # 1. Thread-safe state check
106 with obj.__lazy_lock__:
107 if obj.__lazy_state__ < self._requiredState:
108 obj.__lazy_loader__(self._requiredState)
110 # 2. Determine if we are wrapping a property or a method
111 if isinstance(self._wrapped, property):
112 # If it's a property, call its __get__ to return the value
113 return self._wrapped.__get__(obj, objtype)
115 # 3. Otherwise, treat as a method and return a bound wrapper
116 @wraps(self._wrapped)
117 def wrapper(*args, **kwargs):
118 return self._wrapped(obj, *args, **kwargs)
120 return wrapper
123@export
124class LazyLoadableMixin(metaclass=ExtendedType, mixin=True):
125 __lazy_state__: LazyLoaderState
126 __lazy_lock__: RLock
128 def __init__(self, targetLevel: LazyLoaderState = LazyLoaderState.Initialized) -> None:
129 self.__lazy_state__ = LazyLoaderState.Initialized
130 self.__lazy_lock__ = RLock()
132 if targetLevel > self.__lazy_state__:
133 with self.__lazy_lock__:
134 self.__lazy_loader__(targetLevel)
136 @abstractmethod
137 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
138 pass
141@export
142class Distribution(metaclass=ExtendedType, slots=True):
143 _filename: str
144 _url: URL
145 _uploadTime: datetime
147 def __init__(self, filename: str, url: Union[str, URL], uploadTime: datetime) -> None:
148 if not isinstance(filename, str): 148 ↛ 149line 148 didn't jump to line 149 because the condition on line 148 was never true
149 ex = TypeError("Parameter 'filename' is not of type 'str'.")
150 ex.add_note(f"Got type '{getFullyQualifiedName(filename)}'.")
151 raise ex
153 self._filename = filename
155 if isinstance(url, str): 155 ↛ 157line 155 didn't jump to line 157 because the condition on line 155 was always true
156 url = URL.Parse(url)
157 elif not isinstance(url, URL):
158 ex = TypeError("Parameter 'url' is not of type 'URL'.")
159 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
160 raise ex
162 self._url = url
164 if not isinstance(uploadTime, datetime): 164 ↛ 165line 164 didn't jump to line 165 because the condition on line 164 was never true
165 ex = TypeError("Parameter 'uploadTime' is not of type 'str'.")
166 ex.add_note(f"Got type '{getFullyQualifiedName(uploadTime)}'.")
167 raise ex
169 self._uploadTime = uploadTime
171 @readonly
172 def Filename(self) -> str:
173 """
174 Read-only property to access the distribution's filename (:attr:`_filename`).
176 :returns: Filename of the distribution.
177 """
178 return self._filename
180 @readonly
181 def URL(self) -> URL:
182 """
183 Read-only property to access the URL this distribution can be downloaded from (:attr:`_url`).
185 :returns: Download URL of the distribution.
186 """
187 return self._url
189 @readonly
190 def UploadTime(self) -> datetime:
191 """
192 Read-only property to access the time this distribution was uploaded (:attr:`_uploadTime`).
194 :returns: Upload time of the distribution.
195 """
196 return self._uploadTime
198 def __repr__(self) -> str:
199 return f"Distribution: {self._filename}"
201 def __str__(self) -> str:
202 return f"{self._filename}"
205@export
206class Release(PackageVersion, LazyLoadableMixin):
207 _files: List[Distribution]
208 _requirements: Dict[Union[str, None], List[Requirement]]
210 _api: Nullable[URL]
211 _session: Nullable[Session]
213 def __init__(
214 self,
215 version: PythonVersion,
216 timestamp: datetime,
217 files: Nullable[Iterable[Distribution]] = None,
218 requirements: Nullable[Mapping[str, List[Requirement]]] = None,
219 project: Nullable["Project"] = None,
220 lazy: LazyLoaderState = LazyLoaderState.Initialized
221 ) -> None:
222 if project is not None and (storage := project._storage) is not None: 222 ↛ 226line 222 didn't jump to line 226 because the condition on line 222 was always true
223 self._api = storage._api
224 self._session = storage._session
225 else:
226 self._api = None
227 self._session = None
229 super().__init__(version, project, timestamp)
230 LazyLoadableMixin.__init__(self, lazy)
232 self._files = [file for file in files] if files is not None else []
233 self._requirements = {k: v for k, v in requirements} if requirements is not None else {None: []}
235 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
236 if targetLevel >= LazyLoaderState.PartiallyLoaded: 236 ↛ 238line 236 didn't jump to line 238 because the condition on line 236 was always true
237 self.DownloadDetails()
238 if targetLevel >= LazyLoaderState.PostProcessed: 238 ↛ 239line 238 didn't jump to line 239 because the condition on line 238 was never true
239 self.PostProcess()
241 @lazy(LazyLoaderState.PostProcessed)
242 @PackageVersion.DependsOn.getter
243 def DependsOn(self) -> Dict["Package", Dict[SemanticVersion, "PackageVersion"]]:
244 return super().DependsOn
246 @readonly
247 def Project(self) -> "Project":
248 """
249 Read-only property to access the project this release belongs to (:attr:`_package`).
251 :returns: The project this release belongs to.
252 """
253 return self._package
255 @lazy(LazyLoaderState.PartiallyLoaded)
256 @readonly
257 def Files(self) -> List[Distribution]:
258 """
259 Read-only property to access the distributions published for this release (:attr:`_files`).
261 :returns: List of distributions.
262 """
263 return self._files
265 @lazy(LazyLoaderState.PartiallyLoaded)
266 @readonly
267 def Requirements(self) -> Dict[str, List[Requirement]]:
268 """
269 Read-only property to access the release's requirements, grouped by extra (:attr:`_requirements`).
271 :returns: Dictionary of extras and their requirements. Requirements without an extra are stored under ``None``.
272 """
273 return self._requirements
275 def _GetPyPIEndpoint(self) -> str:
276 return f"{self._package._name.lower()}/{self._version}/json"
278 def DownloadDetails(self) -> None:
279 if self._session is None: 279 ↛ 280line 279 didn't jump to line 280 because the condition on line 279 was never true
280 ex = NoSessionAvailableException(f"No session available to download release '{self._version}' of package '{self._package._name}'.")
281 ex.add_note(f"A session is opened by the package index and handed to the objects it creates.")
282 raise ex
284 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
285 try:
286 response.raise_for_status()
287 except HTTPError as ex:
288 if ex.response.status_code == 404:
289 raise ReleaseNotFoundException(f"Release '{self._version}' of package '{self._package._name}' not found.") from ex
291 self.UpdateDetailsFromPyPIJSON(response.json())
293 index: PythonPackageIndex = self._package._storage
294 for requirement in self._requirements[None]:
295 packageName = requirement.name
296 index.DownloadProject(packageName, True)
298 def UpdateDetailsFromPyPIJSON(self, json) -> None:
299 infoNode = json["info"]
300 if (extras := infoNode["provides_extra"]) is not None:
301 self._requirements = {extra: [] for extra in extras}
302 self._requirements[None] = []
304 if (requirements := infoNode["requires_dist"]) is not None: 304 ↛ 328line 304 didn't jump to line 328 because the condition on line 304 was always true
305 brokenRequirements = []
306 for requirement in requirements:
307 req = Requirement(requirement)
309 # Handle requirements without an extra marker
310 if req.marker is None:
311 self._requirements[None].append(req)
312 continue
314 for extra in self._requirements.keys():
315 if extra is not None and req.marker.evaluate({"extra": extra}):
316 self._requirements[extra].append(req)
317 break
318 else:
319 brokenRequirements.append(req)
321 if len(brokenRequirements) > 0:
322 WarningCollector.Raise(
323 BrokenRequirementWarning(f"Package '{self._package._name}' has {len(brokenRequirements)} requirement(s) whose marker matches no declared extra."),
324 notes=[f"Broken requirement: {req}" for req in brokenRequirements]
325 )
326 self._requirements[0] = brokenRequirements
328 self.__lazy_state__ = LazyLoaderState.FullyLoaded
330 def PostProcess(self) -> None:
331 index: PythonPackageIndex = self._package._storage
332 for requirement in self._requirements[None]:
333 package = index.DownloadProject(requirement.name)
335 for release in package:
336 if str(release._version) in requirement.specifier:
337 self.AddDependencyToPackageVersion(release)
339 self.SortDependencies()
340 self.__lazy_state__ = LazyLoaderState.PostProcessed
342 @lazy(LazyLoaderState.PartiallyLoaded)
343 def __repr__(self) -> str:
344 return f"Release: {self._package._name}:{self._version} Files: {len(self._files)}"
346 def __str__(self) -> str:
347 return f"{self._version}"
350@export
351class Project(Package, LazyLoadableMixin):
352 _url: Nullable[URL]
354 _api: Nullable[URL]
355 _session: Nullable[Session]
357 def __init__(
358 self,
359 name: str,
360 url: Union[str, URL],
361 releases: Nullable[Iterable[Release]] = None,
362 index: Nullable["PythonPackageIndex"] = None,
363 lazy: LazyLoaderState = LazyLoaderState.Initialized
364 ) -> None:
365 if index is not None: 365 ↛ 369line 365 didn't jump to line 369 because the condition on line 365 was always true
366 self._api = index._api
367 self._session = index._session
368 else:
369 self._api = None
370 self._session = None
372 super().__init__(name, storage=index)
373 LazyLoadableMixin.__init__(self, lazy)
375 # if isinstance(url, str):
376 # url = URL.Parse(url)
377 # elif not isinstance(url, URL):
378 # ex = TypeError("Parameter 'url' is not of type 'URL'.")
379 # ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
380 # raise ex
381 #
382 # self._url = url
383 # self._releases = {release.Version: release for release in sorted(releases, key=lambda r: r.Version)} if releases is not None else {}
385 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
386 if targetLevel >= LazyLoaderState.PartiallyLoaded: 386 ↛ 388line 386 didn't jump to line 388 because the condition on line 386 was always true
387 self.DownloadDetails()
388 if targetLevel >= LazyLoaderState.PostProcessed: 388 ↛ 389line 388 didn't jump to line 389 because the condition on line 388 was never true
389 self.DownloadReleaseDetails()
391 @readonly
392 def PackageIndex(self) -> "PythonPackageIndex":
393 """
394 Read-only property to access the package index this project was read from (:attr:`_storage`).
396 :returns: The package index this project belongs to.
397 """
398 return self._storage
400 @lazy(LazyLoaderState.PartiallyLoaded)
401 @readonly
402 def URL(self) -> URL:
403 """
404 Read-only property to access the project's URL in the package index (:attr:`_url`).
406 :returns: URL of the project.
407 """
408 return self._url
410 @lazy(LazyLoaderState.PartiallyLoaded)
411 @readonly
412 def Releases(self) -> Dict[PythonVersion, Release]:
413 """
414 Read-only property to access all known releases of this project (:attr:`_versions`).
416 :returns: Dictionary of versions and their releases.
417 """
418 return self._versions
420 @lazy(LazyLoaderState.PartiallyLoaded)
421 @readonly
422 def ReleaseCount(self) -> int:
423 """
424 Read-only property to return the number of known releases.
426 :returns: Number of releases.
427 """
428 return len(self._versions)
430 @lazy(LazyLoaderState.PartiallyLoaded)
431 @readonly
432 def LatestRelease(self) -> Release:
433 """
434 Read-only property to return the most recent release of this project.
436 :returns: The latest release.
437 """
438 return firstValue(self._versions)
440 def _GetPyPIEndpoint(self) -> str:
441 return f"{self._name.lower()}/json"
443 def DownloadDetails(self) -> None:
444 if self._session is None: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true
445 ex = NoSessionAvailableException(f"No session available to download details of package '{self._name}'.")
446 ex.add_note(f"A session is opened by the package index and handed to the objects it creates.")
447 raise ex
449 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
450 try:
451 response.raise_for_status()
452 except HTTPError as ex:
453 if ex.response.status_code == 404:
454 raise ProjectNotFoundException(f"Package '{self._name}' not found.") from ex
456 self.UpdateDetailsFromPyPIJSON(response.json())
458 def UpdateDetailsFromPyPIJSON(self, json) -> None:
459 infoNode = json["info"]
460 releasesNode = json["releases"]
462 # Update project/package URL
463 self._url = URL.Parse(infoNode["project_url"])
465 # Convert key to Version number, skip empty releases
466 convertedReleasesNode = {}
467 for k, v in releasesNode.items():
468 if len(v) == 0: 468 ↛ 469line 468 didn't jump to line 469 because the condition on line 468 was never true
469 continue
471 try:
472 version = PythonVersion.Parse(k)
473 convertedReleasesNode[version] = v
474 except ValueError as ex:
475 print(f"Unsupported version format '{k}' - {ex}")
477 for version, releaseNode in sorted(convertedReleasesNode.items(), key=lambda t: t[0]):
478 if Parts.Postfix in version._parts: 478 ↛ 479line 478 didn't jump to line 479 because the condition on line 478 was never true
479 pass
481 files = [Distribution(file["filename"], file["url"], datetime.fromisoformat(file["upload_time_iso_8601"]), ) for
482 file in releaseNode]
483 lazy = LazyLoaderState.PartiallyLoaded if LazyLoaderState.PartiallyLoaded <= self.__lazy_state__ <= LazyLoaderState.FullyLoaded else LazyLoaderState.Initialized
484 Release(
485 version,
486 files[0]._uploadTime,
487 files,
488 project=self,
489 lazy=lazy
490 )
492 self.SortVersions()
493 self.__lazy_state__ = LazyLoaderState.FullyLoaded
495 def DownloadReleaseDetails(self) -> None:
496 async def ParallelDownloadReleaseDetails():
497 async def routine(session, release: Release):
498 if Parts.Postfix in release._version._parts:
499 pass
501 async with session.get(self._GetPyPIEndpoint()) as response:
502 json = await response.json()
503 response.raise_for_status()
505 release.UpdateDetailsFromPyPIJSON(json)
507 async with ClientSession(base_url=str(self._api), headers={"accept": "application/json"}) as session:
508 tasks = []
509 for release in self._versions.values(): # type: Release
510 tasks.append(routine(session, release))
512 results = await asyncio_gather(*tasks, return_exceptions=True)
513 delList = []
514 for release, result in zip(self.Releases.values(), results):
515 if isinstance(result, Exception):
516 delList.append((release, result))
518 for release, ex in delList:
519 WarningCollector.Raise(
520 ReleaseDetailsWarning(f"Dropping release '{release.Version}' of package '{release.Project._name}': details couldn't be downloaded."),
521 ex
522 )
523 del self.Releases[release.Version]
525 asyncio_run(ParallelDownloadReleaseDetails())
526 self.__lazy_state__ = LazyLoaderState.PostProcessed
528 def __repr__(self) -> str:
529 return f"Project: {self._name} latest: {self.LatestRelease._version}"
531 def __str__(self) -> str:
532 return f"{self._name}"
535@export
536class PythonPackageIndex(PackageStorage):
537 _url: URL
539 _api: URL
540 _session: Session
542 def __init__(self, name: str, url: Union[str, URL], api: Union[str, URL], graph: "PackageDependencyGraph") -> None:
543 super().__init__(name, graph)
545 if isinstance(url, str): 545 ↛ 547line 545 didn't jump to line 547 because the condition on line 545 was always true
546 url = URL.Parse(url)
547 elif not isinstance(url, URL):
548 ex = TypeError("Parameter 'url' is not of type 'URL'.")
549 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
550 raise ex
552 self._url = url
554 if isinstance(api, str): 554 ↛ 556line 554 didn't jump to line 556 because the condition on line 554 was always true
555 api = URL.Parse(api)
556 elif not isinstance(api, URL):
557 ex = TypeError("Parameter 'api' is not of type 'URL'.")
558 ex.add_note(f"Got type '{getFullyQualifiedName(api)}'.")
559 raise ex
561 self._api = api
563 self._session = Session()
564 self._session.headers["accept"] = "application/json"
566 @readonly
567 def URL(self) -> URL:
568 """
569 Read-only property to access the package index' base URL (:attr:`_url`).
571 :returns: Base URL of the package index.
572 """
573 return self._url
575 @readonly
576 def API(self) -> URL:
577 """
578 Read-only property to access the package index' API URL (:attr:`_api`).
580 :returns: API URL of the package index.
581 """
582 return self._api
584 @readonly
585 def Projects(self) -> Dict[str, Project]:
586 """
587 Read-only property to access all projects known to this package index (:attr:`_packages`).
589 :returns: Dictionary of project names and projects.
590 """
591 return self._packages
593 @readonly
594 def ProjectCount(self) -> int:
595 """
596 Read-only property to return the number of known projects.
598 :returns: Number of projects.
599 """
600 return len(self._packages)
602 def _GetPyPIEndpoint(self, projectName: str) -> str:
603 return f"{self._api}{projectName.lower()}/json"
605 def DownloadProject(self, projectName: str, lazy: LazyLoaderState = LazyLoaderState.PartiallyLoaded) -> Project:
606 project = Project(projectName, "", index=self, lazy=lazy)
608 return project
610 def __repr__(self) -> str:
611 return f"{self._name}"
613 def __str__(self) -> str:
614 return f"{self._name}"
617@export
618class PythonPackageDependencyGraph(PackageDependencyGraph):
619 def __init__(self, name: str) -> None:
620 super().__init__(name)