Coverage for pyTooling / Dependency / Python.py: 68%
329 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-08 23:46 +0000
1# ==================================================================================================================== #
2# _____ _ _ ____ _ #
3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ | _ \ ___ _ __ ___ _ __ __| | ___ _ __ ___ _ _ #
4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | | | | |/ _ \ '_ \ / _ \ '_ \ / _` |/ _ \ '_ \ / __| | | | #
5# | |_) | |_| || | (_) | (_) | | | | | | (_| |_| |_| | __/ |_) | __/ | | | (_| | __/ | | | (__| |_| | #
6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/ \___| .__/ \___|_| |_|\__,_|\___|_| |_|\___|\__, | #
7# |_| |___/ |___/ |_| |___/ #
8# ==================================================================================================================== #
9# Authors: #
10# Patrick Lehmann #
11# #
12# License: #
13# ==================================================================================================================== #
14# Copyright 2025-2026 Patrick Lehmann - Bötzingen, Germany #
15# #
16# Licensed under the Apache License, Version 2.0 (the "License"); #
17# you may not use this file except in compliance with the License. #
18# You may obtain a copy of the License at #
19# #
20# http://www.apache.org/licenses/LICENSE-2.0 #
21# #
22# Unless required by applicable law or agreed to in writing, software #
23# distributed under the License is distributed on an "AS IS" BASIS, #
24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
25# See the License for the specific language governing permissions and #
26# limitations under the License. #
27# #
28# SPDX-License-Identifier: Apache-2.0 #
29# ==================================================================================================================== #
30#
31"""
32Implementation of package dependencies.
34.. hint::
36 See :ref:`high-level help <DEPENDENCIES>` for explanations and usage examples.
37"""
38from asyncio import run as asyncio_run, gather as asyncio_gather
39from datetime import datetime
40from enum import IntEnum
41from functools import wraps, update_wrapper
42from threading import RLock
43from typing import Optional as Nullable, List, Dict, Union, Iterable, Mapping, Callable, Iterator
45try:
46 from aiohttp import ClientSession
47except ImportError as ex: # pragma: no cover
48 raise Exception(f"Optional dependency 'aiohttp' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'aiohttp' directly.") from ex
50try:
51 from packaging.requirements import Requirement
52except ImportError as ex: # pragma: no cover
53 raise Exception(f"Optional dependency 'packaging' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'packaging' directly.") from ex
55try:
56 from requests import Session, HTTPError
57except ImportError as ex: # pragma: no cover
58 raise Exception(f"Optional dependency 'requests' not installed. Either install pyTooling with extra dependencies 'pyTooling[pypi]' or install 'requests' directly.") from ex
60try:
61 from pyTooling.Decorators import export, readonly
62 from pyTooling.MetaClasses import ExtendedType, abstractmethod, mustoverride
63 from pyTooling.Exceptions import ToolingException
64 from pyTooling.Common import getFullyQualifiedName, firstKey, firstValue
65 from pyTooling.Dependency import Package, PackageStorage, PackageVersion, PackageDependencyGraph
66 from pyTooling.GenericPath.URL import URL
67 from pyTooling.Versioning import SemanticVersion, PythonVersion, Parts
68except (ImportError, ModuleNotFoundError): # pragma: no cover
69 print("[pyTooling.Dependency] Could not import from 'pyTooling.*'!")
71 try:
72 from Decorators import export, readonly
73 from MetaClasses import ExtendedType, abstractmethod, mustoverride
74 from Exceptions import ToolingException
75 from Common import getFullyQualifiedName, firstKey, firstValue
76 from Dependency import Package, PackageStorage, PackageVersion, PackageDependencyGraph
77 from GenericPath.URL import URL
78 from Versioning import SemanticVersion, PythonVersion, Parts
79 except (ImportError, ModuleNotFoundError) as ex: # pragma: no cover
80 print("[pyTooling.Dependency] Could not import directly!")
81 raise ex
84@export
85class LazyLoaderState(IntEnum):
86 Uninitialized = 0 #: No data or minimal data like ID or name.
87 Initialized = 1 #: Initialized by some __init__ parameters.
88 PartiallyLoaded = 2 #: Some additional data was loaded.
89 FullyLoaded = 3 #: All data is loaded.
90 PostProcessed = 4 #: Loaded data triggered further processing.
93@export
94class lazy:
95 """
96 Unified decorator that supports:
97 1. @lazy(state) def method()
98 2. @lazy(state) @property def prop()
99 """
101 def __init__(self, _requiredState: LazyLoaderState = LazyLoaderState.PartiallyLoaded):
102 self._requiredState = _requiredState
103 self._wrapped = None
105 def __call__(self, wrapped):
106 self._wrapped = wrapped
107 # If it's a function, we update metadata.
108 # If it's a property, it doesn't support update_wrapper directly.
109 if hasattr(wrapped, "__name__"):
110 update_wrapper(self, wrapped)
112 return self
114 def __get__(self, obj, objtype=None):
115 if obj is None: 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true
116 return self
118 # 1. Thread-safe state check
119 with obj.__lazy_lock__:
120 if obj.__lazy_state__ < self._requiredState:
121 obj.__lazy_loader__(self._requiredState)
123 # 2. Determine if we are wrapping a property or a method
124 if isinstance(self._wrapped, property):
125 # If it's a property, call its __get__ to return the value
126 return self._wrapped.__get__(obj, objtype)
128 # 3. Otherwise, treat as a method and return a bound wrapper
129 @wraps(self._wrapped)
130 def wrapper(*args, **kwargs):
131 return self._wrapped(obj, *args, **kwargs)
133 return wrapper
136@export
137class LazyLoadableMixin(metaclass=ExtendedType, mixin=True):
138 __lazy_state__: LazyLoaderState
139 __lazy_lock__: RLock
141 def __init__(self, targetLevel: LazyLoaderState = LazyLoaderState.Initialized) -> None:
142 self.__lazy_state__ = LazyLoaderState.Initialized
143 self.__lazy_lock__ = RLock()
145 if targetLevel > self.__lazy_state__:
146 with self.__lazy_lock__:
147 self.__lazy_loader__(targetLevel)
149 @abstractmethod
150 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
151 pass
154@export
155class Distribution(metaclass=ExtendedType, slots=True):
156 _filename: str
157 _url: URL
158 _uploadTime: datetime
160 def __init__(self, filename: str, url: Union[str, URL], uploadTime: datetime) -> None:
161 if not isinstance(filename, str): 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 ex = TypeError("Parameter 'filename' is not of type 'str'.")
163 ex.add_note(f"Got type '{getFullyQualifiedName(filename)}'.")
164 raise ex
166 self._filename = filename
168 if isinstance(url, str): 168 ↛ 170line 168 didn't jump to line 170 because the condition on line 168 was always true
169 url = URL.Parse(url)
170 elif not isinstance(url, URL):
171 ex = TypeError("Parameter 'url' is not of type 'URL'.")
172 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
173 raise ex
175 self._url = url
177 if not isinstance(uploadTime, datetime): 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 ex = TypeError("Parameter 'uploadTime' is not of type 'str'.")
179 ex.add_note(f"Got type '{getFullyQualifiedName(uploadTime)}'.")
180 raise ex
182 self._uploadTime = uploadTime
184 @readonly
185 def Filename(self) -> str:
186 return self._filename
188 @readonly
189 def URL(self) -> URL:
190 return self._url
192 @readonly
193 def UploadTime(self) -> datetime:
194 return self._uploadTime
196 def __repr__(self) -> str:
197 return f"Distribution: {self._filename}"
199 def __str__(self) -> str:
200 return f"{self._filename}"
203@export
204class Release(PackageVersion, LazyLoadableMixin):
205 _files: List[Distribution]
206 _requirements: Dict[Union[str, None], List[Requirement]]
208 _api: Nullable[URL]
209 _session: Nullable[Session]
211 def __init__(
212 self,
213 version: PythonVersion,
214 timestamp: datetime,
215 files: Nullable[Iterable[Distribution]] = None,
216 requirements: Nullable[Mapping[str, List[Requirement]]] = None,
217 project: Nullable["Project"] = None,
218 lazy: LazyLoaderState = LazyLoaderState.Initialized
219 ) -> None:
220 if project is not None and (storage := project._storage) is not None: 220 ↛ 224line 220 didn't jump to line 224 because the condition on line 220 was always true
221 self._api = storage._api
222 self._session = storage._session
223 else:
224 self._api = None
225 self._session = None
227 super().__init__(version, project, timestamp)
228 LazyLoadableMixin.__init__(self, lazy)
230 self._files = [file for file in files] if files is not None else []
231 self._requirements = {k: v for k, v in requirements} if requirements is not None else {None: []}
233 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
234 if targetLevel >= LazyLoaderState.PartiallyLoaded: 234 ↛ 236line 234 didn't jump to line 236 because the condition on line 234 was always true
235 self.DownloadDetails()
236 if targetLevel >= LazyLoaderState.PostProcessed: 236 ↛ 237line 236 didn't jump to line 237 because the condition on line 236 was never true
237 self.PostProcess()
239 @lazy(LazyLoaderState.PostProcessed)
240 @PackageVersion.DependsOn.getter
241 def DependsOn(self) -> Dict["Package", Dict[SemanticVersion, "PackageVersion"]]:
242 return super().DependsOn
244 @readonly
245 def Project(self) -> "Project":
246 return self._package
248 @lazy(LazyLoaderState.PartiallyLoaded)
249 @readonly
250 def Files(self) -> List[Distribution]:
251 return self._files
253 @lazy(LazyLoaderState.PartiallyLoaded)
254 @readonly
255 def Requirements(self) -> Dict[str, List[Requirement]]:
256 return self._requirements
258 def _GetPyPIEndpoint(self) -> str:
259 return f"{self._package._name.lower()}/{self._version}/json"
261 def DownloadDetails(self) -> None:
262 if self._session is None: 262 ↛ 264line 262 didn't jump to line 264 because the condition on line 262 was never true
263 # TODO: NoSessionAvailableException
264 raise ToolingException(f"No session available.")
266 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
267 try:
268 response.raise_for_status()
269 except HTTPError as ex:
270 if ex.response.status_code == 404:
271 # TODO: ReleaseNotFoundException
272 raise ToolingException(f"Release '{self._version}' of package '{self._package._name}' not found.")
274 self.UpdateDetailsFromPyPIJSON(response.json())
276 index: PythonPackageIndex = self._package._storage
277 for requirement in self._requirements[None]:
278 packageName = requirement.name
279 index.DownloadProject(packageName, True)
281 def UpdateDetailsFromPyPIJSON(self, json) -> None:
282 infoNode = json["info"]
283 if (extras := infoNode["provides_extra"]) is not None:
284 self._requirements = {extra: [] for extra in extras}
285 self._requirements[None] = []
287 if (requirements := infoNode["requires_dist"]) is not None: 287 ↛ 308line 287 didn't jump to line 308 because the condition on line 287 was always true
288 brokenRequirements = []
289 for requirement in requirements:
290 req = Requirement(requirement)
292 # Handle requirements without an extra marker
293 if req.marker is None:
294 self._requirements[None].append(req)
295 continue
297 for extra in self._requirements.keys():
298 if extra is not None and req.marker.evaluate({"extra": extra}):
299 self._requirements[extra].append(req)
300 break
301 else:
302 brokenRequirements.append(req)
304 # TODO: raise a warning
305 if len(brokenRequirements) > 0:
306 self._requirements[0] = brokenRequirements
308 self.__lazy_state__ = LazyLoaderState.FullyLoaded
310 def PostProcess(self) -> None:
311 index: PythonPackageIndex = self._package._storage
312 for requirement in self._requirements[None]:
313 package = index.DownloadProject(requirement.name)
315 for release in package:
316 if str(release._version) in requirement.specifier:
317 self.AddDependencyToPackageVersion(release)
319 self.SortDependencies()
320 self.__lazy_state__ = LazyLoaderState.PostProcessed
322 @lazy(LazyLoaderState.PartiallyLoaded)
323 def __repr__(self) -> str:
324 return f"Release: {self._package._name}:{self._version} Files: {len(self._files)}"
326 def __str__(self) -> str:
327 return f"{self._version}"
330@export
331class Project(Package, LazyLoadableMixin):
332 _url: Nullable[URL]
334 _api: Nullable[URL]
335 _session: Nullable[Session]
337 def __init__(
338 self,
339 name: str,
340 url: Union[str, URL],
341 releases: Nullable[Iterable[Release]] = None,
342 index: Nullable["PythonPackageIndex"] = None,
343 lazy: LazyLoaderState = LazyLoaderState.Initialized
344 ) -> None:
345 if index is not None: 345 ↛ 349line 345 didn't jump to line 349 because the condition on line 345 was always true
346 self._api = index._api
347 self._session = index._session
348 else:
349 self._api = None
350 self._session = None
352 super().__init__(name, storage=index)
353 LazyLoadableMixin.__init__(self, lazy)
355 # if isinstance(url, str):
356 # url = URL.Parse(url)
357 # elif not isinstance(url, URL):
358 # ex = TypeError("Parameter 'url' is not of type 'URL'.")
359 # ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
360 # raise ex
361 #
362 # self._url = url
363 # self._releases = {release.Version: release for release in sorted(releases, key=lambda r: r.Version)} if releases is not None else {}
365 def __lazy_loader__(self, targetLevel: LazyLoaderState) -> None:
366 if targetLevel >= LazyLoaderState.PartiallyLoaded: 366 ↛ 368line 366 didn't jump to line 368 because the condition on line 366 was always true
367 self.DownloadDetails()
368 if targetLevel >= LazyLoaderState.PostProcessed: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true
369 self.DownloadReleaseDetails()
371 @readonly
372 def PackageIndex(self) -> "PythonPackageIndex":
373 return self._storage
375 @lazy(LazyLoaderState.PartiallyLoaded)
376 @readonly
377 def URL(self) -> URL:
378 return self._url
380 @lazy(LazyLoaderState.PartiallyLoaded)
381 @readonly
382 def Releases(self) -> Dict[PythonVersion, Release]:
383 return self._versions
385 @lazy(LazyLoaderState.PartiallyLoaded)
386 @readonly
387 def ReleaseCount(self) -> int:
388 return len(self._versions)
390 @lazy(LazyLoaderState.PartiallyLoaded)
391 @readonly
392 def LatestRelease(self) -> Release:
393 return firstValue(self._versions)
395 def _GetPyPIEndpoint(self) -> str:
396 return f"{self._name.lower()}/json"
398 def DownloadDetails(self) -> None:
399 if self._session is None: 399 ↛ 401line 399 didn't jump to line 401 because the condition on line 399 was never true
400 # TODO: NoSessionAvailableException
401 raise ToolingException(f"No session available.")
403 response = self._session.get(url=f"{self._api}{self._GetPyPIEndpoint()}")
404 try:
405 response.raise_for_status()
406 except HTTPError as ex:
407 if ex.response.status_code == 404:
408 # TODO: ReleaseNotFoundException
409 raise ToolingException(f"Package '{self._name}' not found.")
411 self.UpdateDetailsFromPyPIJSON(response.json())
413 def UpdateDetailsFromPyPIJSON(self, json) -> None:
414 infoNode = json["info"]
415 releasesNode = json["releases"]
417 # Update project/package URL
418 self._url = URL.Parse(infoNode["project_url"])
420 # Convert key to Version number, skip empty releases
421 convertedReleasesNode = {}
422 for k, v in releasesNode.items():
423 if len(v) == 0: 423 ↛ 424line 423 didn't jump to line 424 because the condition on line 423 was never true
424 continue
426 try:
427 version = PythonVersion.Parse(k)
428 convertedReleasesNode[version] = v
429 except ValueError as ex:
430 print(f"Unsupported version format '{k}' - {ex}")
432 for version, releaseNode in sorted(convertedReleasesNode.items(), key=lambda t: t[0]):
433 if Parts.Postfix in version._parts: 433 ↛ 434line 433 didn't jump to line 434 because the condition on line 433 was never true
434 pass
436 files = [Distribution(file["filename"], file["url"], datetime.fromisoformat(file["upload_time_iso_8601"]), ) for
437 file in releaseNode]
438 lazy = LazyLoaderState.PartiallyLoaded if LazyLoaderState.PartiallyLoaded <= self.__lazy_state__ <= LazyLoaderState.FullyLoaded else LazyLoaderState.Initialized
439 Release(
440 version,
441 files[0]._uploadTime,
442 files,
443 project=self,
444 lazy=lazy
445 )
447 self.SortVersions()
448 self.__lazy_state__ = LazyLoaderState.FullyLoaded
450 def DownloadReleaseDetails(self) -> None:
451 async def ParallelDownloadReleaseDetails():
452 async def routine(session, release: Release):
453 if Parts.Postfix in release._version._parts:
454 pass
456 async with session.get(self._GetPyPIEndpoint()) as response:
457 json = await response.json()
458 response.raise_for_status()
460 release.UpdateDetailsFromPyPIJSON(json)
462 async with ClientSession(base_url=str(self._api), headers={"accept": "application/json"}) as session:
463 tasks = []
464 for release in self._versions.values(): # type: Release
465 tasks.append(routine(session, release))
467 results = await asyncio_gather(*tasks, return_exceptions=True)
468 delList = []
469 for release, result in zip(self.Releases.values(), results):
470 if isinstance(result, Exception):
471 delList.append((release, result))
473 # TODO: raise a warning
474 for release, ex in delList:
475 print(f" Removing {release.Project._name} {release.Version} - {ex}")
476 del self.Releases[release.Version]
478 asyncio_run(ParallelDownloadReleaseDetails())
479 self.__lazy_state__ = LazyLoaderState.PostProcessed
481 def __repr__(self) -> str:
482 return f"Project: {self._name} latest: {self.LatestRelease._version}"
484 def __str__(self) -> str:
485 return f"{self._name}"
488@export
489class PythonPackageIndex(PackageStorage):
490 _url: URL
492 _api: URL
493 _session: Session
495 def __init__(self, name: str, url: Union[str, URL], api: Union[str, URL], graph: "PackageDependencyGraph") -> None:
496 super().__init__(name, graph)
498 if isinstance(url, str): 498 ↛ 500line 498 didn't jump to line 500 because the condition on line 498 was always true
499 url = URL.Parse(url)
500 elif not isinstance(url, URL):
501 ex = TypeError("Parameter 'url' is not of type 'URL'.")
502 ex.add_note(f"Got type '{getFullyQualifiedName(url)}'.")
503 raise ex
505 self._url = url
507 if isinstance(api, str): 507 ↛ 509line 507 didn't jump to line 509 because the condition on line 507 was always true
508 api = URL.Parse(api)
509 elif not isinstance(api, URL):
510 ex = TypeError("Parameter 'api' is not of type 'URL'.")
511 ex.add_note(f"Got type '{getFullyQualifiedName(api)}'.")
512 raise ex
514 self._api = api
516 self._session = Session()
517 self._session.headers["accept"] = "application/json"
519 @readonly
520 def URL(self) -> URL:
521 return self._url
523 @readonly
524 def API(self) -> URL:
525 return self._api
527 @readonly
528 def Projects(self) -> Dict[str, Project]:
529 return self._packages
531 @readonly
532 def ProjectCount(self) -> int:
533 return len(self._packages)
535 def _GetPyPIEndpoint(self, projectName: str) -> str:
536 return f"{self._api}{projectName.lower()}/json"
538 def DownloadProject(self, projectName: str, lazy: LazyLoaderState = LazyLoaderState.PartiallyLoaded) -> Project:
539 project = Project(projectName, "", index=self, lazy=lazy)
541 return project
543 def __repr__(self) -> str:
544 return f"{self._name}"
546 def __str__(self) -> str:
547 return f"{self._name}"
550@export
551class PythonPackageDependencyGraph(PackageDependencyGraph):
552 def __init__(self, name: str) -> None:
553 super().__init__(name)