1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
|
# ==================================================================================================================== #
# _____ _ _ ____ _ _ #
# _ __ _ |_ _|__ ___ | (_)_ __ __ _ / ___| _ __ | |__ (_)_ __ __ __ #
# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` | \___ \| '_ \| '_ \| | '_ \\ \/ / #
# | |_) | |_| || | (_) | (_) | | | | | | (_| |_ ___) | |_) | | | | | | | |> < #
# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____/| .__/|_| |_|_|_| |_/_/\_\ #
# |_| |___/ |___/ |_| #
# ==================================================================================================================== #
# Authors: #
# Patrick Lehmann #
# #
# License: #
# ==================================================================================================================== #
# Copyright 2026-2026 Patrick Lehmann - Bötzingen, Germany #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); #
# you may not use this file except in compliance with the License. #
# You may obtain a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# #
# SPDX-License-Identifier: Apache-2.0 #
# ==================================================================================================================== #
#
"""
A Sphinx directive rendering a project's dependencies as a table, from the requirements rather than by hand.
A dependency table states, per package, which version is required, what it is licensed under, and what it drags in.
Written by hand it is wrong within a release or two: pyTooling's own documentation table was missing four of the
packages :file:`doc/requirements.txt` requires, listed one that isn't required at all, and called Sphinx BSD-3-Clause
when it is BSD-2-Clause.
**The entrypoints are declared in** :file:`conf.py` **and named by the documents**, the way
:mod:`sphinx_reports` declares its reports:
.. code-block:: Python
# doc/conf.py
pyTooling_Dependency_PackageOverrides = "Dependency.PackageOverrides.yaml"
pyTooling_Dependency_Requirements = {
"package": {"file": "../requirements.txt"},
"documentation": {"file": "requirements.txt"},
"yaml": {"package": "pyTooling[yaml]"}
}
.. code-block:: rest
.. dependency-table:: documentation
:caption: Documentation dependencies
A requirements file is read - with its ``-r`` includes followed - while :file:`conf.py` is being processed, so a
path that doesn't exist ends the build with one clear message instead of an error box in the middle of a page.
**The data is fetched live** from the package index, once per build and shared between every table of that build:
:file:`requirements.txt`, :file:`tests/requirements.txt` and :file:`doc/requirements.txt` overlap heavily, and a
package they share is downloaded once. That still costs real time, so every table reports what it spent, measured
with a :class:`~pyTooling.Stopwatch.Stopwatch`, and the build ends with the total.
"""
from __future__ import annotations
from enum import Enum, auto
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, Iterable, Literal, Mapping, Optional as Nullable
from typing import TypeVar, cast
from pyTooling.Common import getFullyQualifiedName
from pyTooling.Decorators import export, readonly
from pyTooling.Dependency import UnknownLicenseWarning
from pyTooling.Exceptions import ConfigurationError, MissingDependencyError
from pyTooling.MetaClasses import ExtendedType
from pyTooling.Stopwatch import Stopwatch
from pyTooling.Warning import WarningCollector
from docutils import nodes
from docutils.parsers.rst import directives
from sphinx.application import Sphinx
from sphinx.util import logging
if TYPE_CHECKING: # pragma: no cover
# Only this directive needs a package index, so the model is imported when the configuration declares an
# entrypoint rather than when the extension is loaded - otherwise every documentation build using any of these
# roles would need the 'pypi' extra.
from sphinx.config import Config
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
from pyTooling.Dependency.Python import LicenseOverrides, Project, PythonPackageDependencyGraph
from pyTooling.Dependency.Python import PythonPackageIndex, Release, RequirementsFile
from pyTooling.Sphinx import BaseDirective, SphinxExtensionError, strip
from pyTooling.Sphinx import stripAndNormalize
#: URL of the package index the tables are built from, unless :file:`conf.py` names another.
DEFAULT_INDEX_URL = "https://pypi.org"
#: URL of that index's JSON API.
DEFAULT_API_URL = "https://pypi.org/pypi/"
#: Levels of sub-dependencies rendered when nothing says otherwise; ``0`` expands until the tree ends.
DEFAULT_DEPTH = 0
#: Whether a version constraint is reduced to its lower bound when the document doesn't say.
DEFAULT_SIMPLIFIED_VERSIONS = True
@export
class VersionFormat(Enum):
"""How many parts of a version number a dependency table prints."""
Major = auto() #: The major part alone, ``≥9``.
MajorMinor = auto() #: Major and minor, ``≥9.1`` - the default.
MajorMinorPatch = auto() #: Major, minor and patch, ``≥9.1.2``.
All = auto() #: Every part the constraint states, ``≥9.1.2.dev3``.
def __str__(self) -> str:
"""
Return this format's name, as a document writes it.
:returns: The enum member's name.
"""
return self.name
@export
class DependencyFormat(Enum):
"""What a line of a dependency tree states about a package."""
Package = auto() #: The name alone.
PackageVersion = auto() #: Name and version constraint.
PackageLicense = auto() #: Name and license.
PackageVersionLicense = auto() #: Name, version constraint and license - the default.
@readonly
def ShowsVersion(self) -> bool:
"""
Whether this format states a version constraint.
:returns: ``True`` if the version is printed.
"""
return self in (DependencyFormat.PackageVersion, DependencyFormat.PackageVersionLicense)
@readonly
def ShowsLicense(self) -> bool:
"""
Whether this format states a license.
:returns: ``True`` if the license is printed.
"""
return self in (DependencyFormat.PackageLicense, DependencyFormat.PackageVersionLicense)
def __str__(self) -> str:
"""
Return this format's name, as a document writes it.
:returns: The enum member's name.
"""
return self.name
#: How many parts of a version number a table prints when the document doesn't say.
DEFAULT_VERSION_FORMAT = VersionFormat.MajorMinor
#: What a line of a dependency tree states when the document doesn't say.
DEFAULT_DEPENDENCY_FORMAT = DependencyFormat.PackageVersionLicense
#: Comparison operators as a reader writes them. Order matters - the two-character forms have to be tried first.
OPERATOR_SYMBOLS = (("<=", "≤"), (">=", "≥"), ("!=", "≠"), ("==", "="))
#: Operators a simplified constraint drops: an upper bound and an exclusion say what is *not* required.
_DROPPED_OPERATORS = ("<", "<=", "!=")
#: The table's columns, as ``(title, relative width)``.
TABLE_COLUMNS = (("Package", 3), ("Version", 1), ("License", 2), ("Dependencies", 4))
#: The fields one entrypoint may state in :file:`conf.py`, exactly one of them.
#:
#: The singular forms take a string and the plural forms an iterable of strings; they are otherwise the same
#: statement, and a project writes whichever reads better where it stands.
ENTRYPOINT_FIELDS = ("file", "files", "package", "packages")
#: Prefix every configuration value of this extension carries in :file:`conf.py`.
CONFIG_PREFIX = "pyTooling_Dependency"
#: What Sphinx accepts as the rebuild condition of a configuration value - the one this extension uses.
_ConfigRebuild = Literal["env"]
__all__ = [
"DEFAULT_INDEX_URL", "DEFAULT_API_URL", "DEFAULT_DEPTH", "DEFAULT_SIMPLIFIED_VERSIONS", "OPERATOR_SYMBOLS",
"TABLE_COLUMNS", "ENTRYPOINT_FIELDS", "CONFIG_PREFIX",
"DEFAULT_VERSION_FORMAT", "DEFAULT_DEPENDENCY_FORMAT"
]
_logger = logging.getLogger(__name__)
#: The two option enumerations, so one parser serves both.
_FormatType = TypeVar("_FormatType", VersionFormat, DependencyFormat)
def _OneLevelDown(depth: Nullable[int]) -> Nullable[int]:
"""
Return the depth one level deeper, leaving an unlimited depth unlimited.
:param depth: Levels still to expand, or ``None`` for unlimited.
:returns: One level fewer, or ``None``.
"""
return None if depth is None else depth - 1
#: The collector of each running build, by the id of its Sphinx application.
#:
#: It can't live on the build environment, which Sphinx pickles between runs - an open HTTP session and the locks
#: guarding lazy loading are not picklable, and a cached view of a package index would be stale anyway.
_COLLECTORS: dict[int, "DependencyCollector"] = {}
@export
class Entrypoint(metaclass=ExtendedType, slots=True):
"""
One entry of ``pyTooling_Dependency_Requirements``: an identifier and the requirements it stands for.
A file entrypoint is read while :file:`conf.py` is being processed and carries its requirements from then on.
A package entrypoint can only be resolved by asking the package index, so it carries the package's name and
extra and is resolved the first time a table names it.
"""
_identifier: str #: Name the documents refer to this entrypoint by.
_files: tuple[Path, ...] #: Every requirements file read, references included.
_packages: tuple[tuple[str, Nullable[str]], ...] #: The packages to read, as name and extra.
_requirements: Nullable[dict[str, Requirement]] #: The resolved requirements, by canonical package name.
def __init__(
self,
identifier: str,
files: tuple[Path, ...] = (),
packages: tuple[tuple[str, Nullable[str]], ...] = (),
requirements: Nullable[dict[str, Requirement]] = None
) -> None:
"""
Describe one entrypoint.
:param identifier: Name the documents refer to this entrypoint by.
:param files: Optional, every requirements file read, for a file entrypoint. Default: ``()``.
:param packages: Optional, the packages and their extras, for a package entrypoint. Default: ``()``.
:param requirements: Optional, the requirements, if they are known already. Default: ``None``.
"""
self._identifier = identifier
self._files = files
self._packages = packages
self._requirements = requirements
@readonly
def Identifier(self) -> str:
"""
Name the documents refer to this entrypoint by.
:returns: The identifier.
"""
return self._identifier
@readonly
def Files(self) -> tuple[Path, ...]:
"""
The requirements file and every file it includes.
:returns: The files this entrypoint was read from; empty for a package entrypoint.
"""
return self._files
@readonly
def Packages(self) -> tuple[tuple[str, Nullable[str]], ...]:
"""
The packages this entrypoint reads the requirements of, as ``(name, extra)`` pairs.
:returns: The packages, or an empty tuple for a file entrypoint.
"""
return self._packages
@readonly
def Requirements(self) -> Nullable[dict[str, Requirement]]:
"""
The requirements this entrypoint stands for.
:returns: Every required package by its canonical name, or ``None`` if they weren't resolved yet.
"""
return self._requirements
def CacheRequirements(self, requirements: dict[str, Requirement]) -> None:
"""
Remember the requirements the package index answered with.
A package entrypoint can only be resolved by asking the index; remembering the answer is what keeps a second
table naming the same entrypoint from asking again.
:param requirements: Every required package, by its canonical name.
"""
self._requirements = requirements
def __repr__(self) -> str:
"""
Return a representation naming what this entrypoint reads.
:returns: The identifier and its source.
"""
if len(self._files) > 0:
source = ", ".join(str(file) for file in self._files)
else:
source = ", ".join(name if extra is None else f"{name}[{extra}]" for name, extra in self._packages)
return f"<Entrypoint {self._identifier}: {source}>"
@export
class DependencyCollector(metaclass=ExtendedType, slots=True):
"""
The entrypoints a build declared, the package index it queries, and what querying it cost.
One collector is shared by every table of a build: a package required by two entrypoints is downloaded once, and
the time is accumulated so the build can report a total. It exists because the alternative - a table that queries
the index for itself - multiplies a documentation build's runtime by however many tables it has, and
:file:`requirements.txt`, :file:`tests/requirements.txt` and :file:`doc/requirements.txt` share most of what they
require.
The index is opened the first time a table asks for a package, not when the collector is created: a project may
declare its entrypoints and then build a document that shows none of them, and that build should not open an
HTTP session.
"""
_entrypoints: dict[str, Entrypoint] #: The entrypoints declared in :file:`conf.py`.
_indexURL: str #: URL of the package index's website.
_apiURL: str #: URL of the package index's JSON API.
_overrides: LicenseOverrides #: Licenses stated by hand, where the index can't.
_graph: Nullable[PythonPackageDependencyGraph] #: Graph the downloaded packages are collected in.
_index: Nullable[PythonPackageIndex] #: The package index this build queries, once opened.
_projects: dict[str, Nullable[Project]] #: Projects downloaded so far; ``None`` if unknown.
_detailed: set[str] #: Releases whose details were downloaded.
_undescribed: set[str] #: Releases the index lists but can't describe.
_stopwatch: Stopwatch #: Runs only while a request to the index is in flight.
_unresolved: dict[str, tuple[str, ...]] #: Packages whose license the index couldn't answer for,
#: mapped to what it published instead.
def __init__(
self,
entrypoints: dict[str, Entrypoint],
indexURL: str,
apiURL: str,
overrides: LicenseOverrides
) -> None:
"""
Collect what a build declared, without opening the package index yet.
:param entrypoints: The entrypoints declared in :file:`conf.py`, by identifier.
:param indexURL: URL of the package index's website.
:param apiURL: URL of the package index's JSON API.
:param overrides: Licenses stated by hand.
"""
self._entrypoints = entrypoints
self._indexURL = indexURL
self._apiURL = apiURL
self._overrides = overrides
self._graph = None
self._index = None
self._projects = {}
self._detailed = set()
self._undescribed = set()
self._unresolved = {}
# 'preferPause', so each 'with' around a request is one active span: 'Activity' is the time spent waiting for
# the index rather than the age of the collector, and 'ActiveCount' is the number of requests
self._stopwatch = Stopwatch(preferPause=True)
@readonly
def Entrypoints(self) -> dict[str, Entrypoint]:
"""
The entrypoints declared in :file:`conf.py`.
:returns: Every entrypoint by its identifier.
"""
return self._entrypoints
@readonly
def Index(self) -> PythonPackageIndex:
"""
The package index this build queries, opened the first time it is asked for.
:returns: The package index.
"""
from pyTooling.Dependency.Python import PythonPackageDependencyGraph, PythonPackageIndex
if self._index is None:
self._graph = PythonPackageDependencyGraph("documentation")
self._index = PythonPackageIndex("index", self._indexURL, self._apiURL, self._graph, self._overrides)
return self._index
@readonly
def RequestCount(self) -> int:
"""
Number of requests sent to the package index.
The stopwatch runs for exactly one span per request, so this is its
:attr:`~pyTooling.Stopwatch.Stopwatch.ActiveCount` - counting them a second time in a field of our own
would be a second answer to one question.
:returns: Number of requests sent.
"""
return self._stopwatch.ActiveCount
@readonly
def Seconds(self) -> float:
"""
Time spent waiting for the package index, in seconds.
This is the stopwatch's :attr:`~pyTooling.Stopwatch.Stopwatch.Activity` - the sum of the intervals it ran -
not its duration, because it is paused between requests and everything the build does in between is not
time this collector spent.
:returns: Seconds spent on the index.
"""
return self._stopwatch.Activity
@readonly
def UnresolvedLicenses(self) -> dict[str, tuple[str, ...]]:
"""
Packages whose license the index couldn't answer for, and what it published instead.
The published fields are what the override file has to answer for, so they are kept rather than only the
package's name: ``License :: OSI Approved :: BSD License`` names three licenses and is never guessed at, and
a ``license`` field holding a license's title instead of its SPDX identifier doesn't parse.
:returns: Names of the packages needing a license override, mapped to what the index published for them.
"""
return self._unresolved
def Project(self, packageName: str) -> Nullable[Project]:
"""
Return a project, downloading it the first time it is asked for.
A package the index doesn't know is remembered as unknown, so a table naming it doesn't ask again for every
row that mentions it.
:param packageName: Name of the package to look up.
:returns: The project, or ``None`` if the index doesn't know it.
:raises MissingDependencyError: If the 'pypi' extra isn't installed.
"""
try:
from requests import RequestException
except ImportError as ex: # pragma: no cover
raise MissingDependencyError(dependency="requests", extra="pypi") from ex
from pyTooling.Dependency import DependencyError
from pyTooling.Dependency.Python import LazyLoaderState
if packageName in self._projects:
return self._projects[packageName]
project: Nullable[Project]
with self._stopwatch:
try:
project = self.Index.DownloadProject(packageName, LazyLoaderState.PartiallyLoaded)
except (DependencyError, RequestException, ValueError, KeyError):
project = None
self._projects[packageName] = project
return project
def Details(self, release: Release) -> Nullable[Release]:
"""
Make sure a release knows its own requirements and its license.
A release the index lists but can't describe - a yanked one, or a version its release endpoint spells
differently - is remembered as unusable and answered with ``None``. Handing back the release itself would
be worse than useless: its lazily loaded properties would each retry the download and raise.
:param release: The release to fill in.
:returns: The release with its details, or ``None`` if the index can't describe it.
:raises MissingDependencyError: If the 'pypi' extra isn't installed.
"""
try:
from requests import RequestException
except ImportError as ex: # pragma: no cover
raise MissingDependencyError(dependency="requests", extra="pypi") from ex
from pyTooling.Dependency import DependencyError
key = f"{release.Package.Name}=={release.Version}"
if key in self._detailed:
return release if key not in self._undescribed else None
warnings: list[BaseException] = []
with self._stopwatch, WarningCollector(warnings):
try:
release.DownloadDetails()
except (DependencyError, RequestException, ValueError, KeyError):
self._undescribed.add(key)
self._detailed.add(key)
for warning in warnings:
if isinstance(warning, UnknownLicenseWarning):
# the warning's notes are what the index published, which is the reason an override is needed
self._unresolved[release.Package.Name] = warning.Notes
return None if key in self._undescribed else release
@export
def readEntrypoints(configuration: Any, confDirectory: Path) -> dict[str, Entrypoint]:
"""
Turn ``pyTooling_Dependency_Requirements`` into entrypoints, reading every requirements file it names.
A requirements file is read here rather than when a table is built, so a path that doesn't exist ends the build
with one message naming the identifier instead of an error box in the middle of a page - and so two tables
naming the same file read it once.
:param configuration: Value of ``pyTooling_Dependency_Requirements``.
:param confDirectory: Directory :file:`conf.py` lives in; relative paths are resolved against it.
:returns: Every declared entrypoint, by its identifier.
:raises MissingDependencyError: If the 'pypi' extra isn't installed.
:raises SphinxExtensionError: If the configuration is malformed, or a requirements file can't be read.
"""
if not isinstance(configuration, dict):
raise SphinxExtensionError(
f"conf.py: {CONFIG_PREFIX}_Requirements: Expected a dictionary, "
f"got '{getFullyQualifiedName(configuration)}'."
)
entrypoints: dict[str, Entrypoint] = {}
for identifier, declaration in configuration.items():
location = f"conf.py: {CONFIG_PREFIX}_Requirements:[{identifier}]"
if not isinstance(declaration, dict):
raise SphinxExtensionError(
f"{location}: Expected a dictionary, got '{getFullyQualifiedName(declaration)}'."
)
if (unknown := set(declaration) - set(ENTRYPOINT_FIELDS)) != set():
raise SphinxExtensionError(
f"{location}: Unknown field(s): {', '.join(sorted(unknown))}. "
f"Known are: {', '.join(ENTRYPOINT_FIELDS)}."
)
if len(stated := [field for field in ENTRYPOINT_FIELDS if field in declaration]) != 1:
known = ", ".join(ENTRYPOINT_FIELDS)
raise SphinxExtensionError(
f"{location}: Exactly one of {known} has to be configured, "
f"{'none is' if len(stated) == 0 else f'{len(stated)} are'}."
)
field = stated[0]
fieldLocation = f"{location}.{field}"
value = declaration[field]
values: tuple[str, ...]
if field in ("file", "package"):
if not isinstance(value, str):
raise SphinxExtensionError(
f"{fieldLocation}: Expected a string, got '{getFullyQualifiedName(value)}'."
)
values = (value,)
else:
# a string is an iterable of strings itself, so the plural form has to reject one explicitly - otherwise
# {"files": "requirements.txt"} would silently become sixteen one-character paths
if isinstance(value, str) or not isinstance(value, Iterable):
raise SphinxExtensionError(
f"{fieldLocation}: Expected an iterable of strings, got '{getFullyQualifiedName(value)}'. "
f"Use '{field[:-1]}' for a single value."
)
# materialized before the items are checked, because an iterable may be a generator this would consume
values = tuple(value)
for item in values:
if not isinstance(item, str):
raise SphinxExtensionError(
f"{fieldLocation}: Expected strings, got '{getFullyQualifiedName(item)}'."
)
if field in ("file", "files"):
entrypoints[identifier] = _FileEntrypoint(identifier, fieldLocation, values, confDirectory)
else:
entrypoints[identifier] = _PackageEntrypoint(identifier, values)
return entrypoints
def _FileEntrypoint(
identifier: str,
location: str,
files: tuple[str, ...],
confDirectory: Path
) -> Entrypoint:
"""
Read one entrypoint's requirements files.
Several files are read as several trees and flattened in the order they are declared, so a later file's
statement wins - the rule a single file's ``-r`` references already follow.
:param identifier: Identifier of the entrypoint.
:param location: Where in :file:`conf.py` this came from.
:param files: The declared paths.
:param confDirectory: Directory relative paths resolve against.
:returns: The entrypoint, with its requirements read.
:raises MissingDependencyError: If the 'pypi' extra isn't installed.
:raises SphinxExtensionError: If a file can't be read.
"""
try:
from packaging.utils import canonicalize_name
except ImportError as ex: # pragma: no cover
raise MissingDependencyError(dependency="packaging", extra="pypi") from ex
from pyTooling.Dependency import DependencyError
from pyTooling.Dependency.Python import RequirementsFile
readFiles: list[Path] = []
requirements: dict[str, Requirement] = {}
for file in files:
path = Path(file)
if not path.is_absolute():
path = confDirectory / path
try:
requirementsFile = RequirementsFile(path)
except (DependencyError, OSError, UnicodeDecodeError) as cause:
raise SphinxExtensionError(f"{location}: Requirements file '{path}' can't be read: {cause}") from cause
# the tree knows every file it was read from; walking it here would be a second answer to one question
readFiles.extend(requirementsFile.AnalyzedRequirementFiles)
requirements.update({canonicalize_name(req.name): req for req in requirementsFile.AllRequirements})
return Entrypoint(identifier, files=tuple(readFiles), requirements=requirements)
def _PackageEntrypoint(identifier: str, packages: tuple[str, ...]) -> Entrypoint:
"""
Describe one entrypoint's packages, which only the package index can resolve.
:param identifier: Identifier of the entrypoint.
:param packages: The declared packages, each optionally with one extra.
:returns: The entrypoint, with its packages recorded and its requirements still unresolved.
"""
requested: list[tuple[str, Nullable[str]]] = []
for package in packages:
name, _, bracket = package.partition("[")
requested.append((name.strip(), bracket.rstrip("]").strip() or None))
return Entrypoint(identifier, packages=tuple(requested))
@export
def prepareEntrypoints(sphinx: Sphinx, config: Config) -> None:
"""
Call-back for Sphinx' ``config-inited`` event, reading the entrypoints and the license overrides.
A build declaring no entrypoint does nothing here - not even import :mod:`pyTooling.Dependency.Python`, so a
project using only this extension's roles doesn't need the ``pypi`` extra.
:param sphinx: The Sphinx application.
:param config: The configuration, after :file:`conf.py` was read.
:raises SphinxExtensionError: If the configuration is malformed, or a
requirements or license override file can't be read.
"""
if len(declarations := getattr(config, f"{CONFIG_PREFIX}_Requirements", {})) == 0:
return
confDirectory = Path(sphinx.confdir)
try:
from pyTooling.Dependency import DependencyError
from pyTooling.Dependency.Python import LicenseOverrides
except MissingDependencyError as cause: # pragma: no cover
raise SphinxExtensionError(
f"conf.py: {CONFIG_PREFIX}_Requirements: Querying a package index needs the 'pypi' extra: "
f"pip install pyTooling[pypi]"
) from cause
overrides = LicenseOverrides()
if (overrideFile := getattr(config, f"{CONFIG_PREFIX}_PackageOverrides", None)) is not None:
path = Path(overrideFile)
if not path.is_absolute():
path = confDirectory / path
try:
overrides = LicenseOverrides.FromFile(path)
except (DependencyError, ConfigurationError, OSError) as cause:
raise SphinxExtensionError(
f"conf.py: {CONFIG_PREFIX}_PackageOverrides: Override file '{path}' can't be read: {cause}"
) from cause
_COLLECTORS[id(sphinx)] = DependencyCollector(
readEntrypoints(declarations, confDirectory),
getattr(config, f"{CONFIG_PREFIX}_IndexURL", DEFAULT_INDEX_URL),
getattr(config, f"{CONFIG_PREFIX}_APIURL", DEFAULT_API_URL),
overrides
)
@export
class DependencyTable(BaseDirective):
"""
The ``dependency-table`` directive: an entrypoint's dependencies, rendered from the requirements.
One argument, the identifier of an entrypoint declared in ``pyTooling_Dependency_Requirements``. ``:depth:``
says how many levels of sub-dependencies to expand, ``:simplified-versions:`` whether a constraint is reduced to
its lower bound, and ``:caption:`` puts a caption under the table; which package index is queried and which
licenses are stated by hand are build-wide and configured in :file:`conf.py`.
"""
directiveName: str = "dependency-table" #: Name the directive is invoked by.
#: The configuration values this directive adds to :file:`conf.py`, as ``name: (default, rebuild, types)``. Each is
#: registered with :data:`CONFIG_PREFIX` as prefix, e.g. ``pyTooling_Dependency_Requirements``.
#:
#: ``Requirements`` maps an identifier to what it names - a file, files, a package or packages. The other three are
#: build-wide, because one package index is queried per build and one override file answers for it. All four are
#: ``"env"``-rebuilt: changing any of them changes every table.
configValues: ClassVar[dict[str, tuple[Any, _ConfigRebuild, Any]]] = {
"Requirements": ({}, "env", dict),
"PackageOverrides": (None, "env", (str, Path)),
"IndexURL": (DEFAULT_INDEX_URL, "env", str),
"APIURL": (DEFAULT_API_URL, "env", str),
}
_simplify: bool #: Whether this table's version constraints are reduced to their lower bound.
_versionFormat: VersionFormat #: How many parts of a version number this table prints.
_dependencyFormat: DependencyFormat #: What a line of this table's dependency trees states.
has_content = False #: A boolean; ``True`` if content is allowed.
required_arguments = 1 #: Number of required directive arguments: the entrypoint's identifier.
optional_arguments = 0 #: Number of optional arguments after the required ones.
final_argument_whitespace = False #: A boolean; ``True`` if the last argument may contain spaces.
# docutils declares 'option_spec' on 'Directive' and 'BaseDirective' assigns it, so mypy calls every
# spelling of this override a conflict with one of them
#: Mapping of option names to validator functions.
option_spec: dict[str, Any] = { # type: ignore[misc]
"caption": strip,
"depth": directives.nonnegative_int,
"simplified-versions": stripAndNormalize,
"version-format": stripAndNormalize,
"dependency-format": stripAndNormalize,
}
def run(self) -> list[nodes.Node]:
"""
Resolve the named entrypoint against the package index and return its requirements as a table.
:returns: A ``table`` node, or an error node when the entrypoint couldn't be resolved.
"""
identifier = self.arguments[0].strip()
self._simplify = self._ParseBooleanOption("simplified-versions", DEFAULT_SIMPLIFIED_VERSIONS)
self._versionFormat = self._ParseFormatOption("version-format", VersionFormat, DEFAULT_VERSION_FORMAT)
self._dependencyFormat = self._ParseFormatOption(
"dependency-format", DependencyFormat, DEFAULT_DEPENDENCY_FORMAT
)
with Stopwatch() as stopwatch:
try:
collector = self._Collector()
requestsBefore = collector.RequestCount
requirements = self._Resolve(identifier, collector)
table = self._CreateTable(identifier, requirements, collector)
except SphinxExtensionError as cause:
return [self.state.document.reporter.error(
f"{self.directiveName}: {cause}", line=self.lineno
)]
# what this table cost, not what the build has spent so far - a package another table already downloaded is
# free here, and that is the point of sharing the collector
_logger.info(
f"[{self.directiveName}] {identifier}: {len(requirements)} package(s), "
f"{collector.RequestCount - requestsBefore} request(s), {stopwatch.Duration:.2f} s"
)
return [table]
def _ParseFormatOption(self, optionName: str, enumType: type[_FormatType], default: _FormatType) -> _FormatType:
"""
Read an option naming a member of an enumeration, or fall back to its default.
:attr:`~pyTooling.Sphinx.BaseDirective._ParseEnumOption` requires the option and
lower-cases what it reads; these two have a default and are written the way the members are spelled, so a
document says ``:version-format: MajorMinor`` rather than ``major_minor``.
:param optionName: Name of the option to read.
:param enumType: The enumeration its value names a member of.
:param default: The member to use when the option isn't given.
:returns: The named member.
:raises SphinxExtensionError: If the value names no member.
"""
if (option := self.options.get(optionName, None)) is None:
return default
for member in enumType:
if option.lower() == member.name.lower():
return member
known = ", ".join(member.name for member in enumType)
raise SphinxExtensionError(
f"{self.directiveName}::{optionName}: '{option}' is not one of: {known}."
)
def _Collector(self) -> DependencyCollector:
"""
Return the build's collector, which :func:`prepareEntrypoints` created when :file:`conf.py` was read.
The collector belongs to the running application rather than to the directive, because a document with three
tables would otherwise open three indexes and download the same packages three times. It deliberately does
*not* live on the build environment: Sphinx pickles that between runs, and neither an open HTTP session nor a
cached view of a package index survives being pickled - or should.
:returns: The collector shared by every table of this build.
:raises SphinxExtensionError: If no entrypoint was configured.
"""
if (collector := _COLLECTORS.get(id(self.env.app), None)) is None:
raise SphinxExtensionError(
f"No entrypoint is configured. Declare one in conf.py: {CONFIG_PREFIX}_Requirements."
)
return collector
def _Resolve(self, identifier: str, collector: DependencyCollector) -> dict[str, Requirement]:
"""
Return what the named entrypoint requires.
A file entrypoint was read when :file:`conf.py` was processed and answers immediately; a package entrypoint
is resolved against the package index the first time a table names it, and remembers the answer.
:param identifier: Identifier the document names.
:param collector: The build's collector.
:returns: Every required package, by its canonical name.
:raises MissingDependencyError: If the 'pypi' extra isn't installed.
:raises SphinxExtensionError: If the identifier is unknown, or the
package index can't answer for the entrypoint's package.
"""
try:
from packaging.utils import canonicalize_name
except ImportError as ex: # pragma: no cover
raise MissingDependencyError(dependency="packaging", extra="pypi") from ex
if (entrypoint := collector.Entrypoints.get(identifier, None)) is None:
known = ", ".join(sorted(collector.Entrypoints)) or "none"
raise SphinxExtensionError(
f"Entrypoint '{identifier}' is not configured in conf.py: {CONFIG_PREFIX}_Requirements. "
f"Known are: {known}."
)
for file in entrypoint.Files:
self.env.note_dependency(str(file))
if (requirements := entrypoint.Requirements) is not None:
return requirements
# several packages flatten in the order they are declared, the rule a file's '-r' references already follow
requirements = {}
for packageName, extra in entrypoint.Packages:
for requirement in self._PublishedRequirements(packageName, extra, collector):
requirements[canonicalize_name(requirement.name)] = requirement
entrypoint.CacheRequirements(requirements)
return requirements
def _PublishedRequirements(
self,
packageName: str,
extra: Nullable[str],
collector: DependencyCollector
) -> list[Requirement]:
"""
Ask the package index what a package's latest release requires.
:param packageName: Name of the package to ask about.
:param extra: Extra whose requirements are wanted, or ``None`` for the package's own.
:param collector: The build's collector.
:returns: What that release requires.
:raises SphinxExtensionError: If the index doesn't know the
package, can't describe its latest release, or the package has no such extra.
"""
if (project := collector.Project(packageName)) is None:
raise SphinxExtensionError(f"Package '{packageName}' is unknown to the package index.")
if (release := collector.Details(project.LatestRelease)) is None:
raise SphinxExtensionError(f"The package index can't describe the latest release of '{packageName}'.")
published: Nullable[list[Requirement]] = release.Requirements.get(extra, None)
if published is None:
known = ", ".join(sorted(str(key) for key in release.Requirements if key is not None))
raise SphinxExtensionError(f"Package '{packageName}' has no extra '{extra}'. Known are: {known}.")
return published
def _CreateTable(
self,
identifier: str,
requirements: dict[str, Requirement],
collector: DependencyCollector
) -> nodes.table:
"""
Render the requirements as a four-column table.
:param identifier: Identifier of the entrypoint, used as the table's identifier.
:param requirements: Every required package, by its canonical name.
:param collector: The build's collector.
:returns: The finished table.
"""
tableGroup = self._CreateSingleRowTableHeader(
columns=list(TABLE_COLUMNS),
identifier=identifier,
classes=["dependency-table"]
)
tableGroup += (tableBody := nodes.tbody())
# ':depth: 0' - and the default - means expand until the tree ends; 'None' is that, internally, because
# 'depth - 1' would otherwise walk 0 into negative numbers and mean two different things at once
depth = self.options.get("depth", DEFAULT_DEPTH)
levels: Nullable[int] = None if depth == 0 else depth
if len(requirements) == 0:
tableBody += self._CreateEmptyRow(len(TABLE_COLUMNS))
else:
for name in sorted(requirements, key=str.lower):
tableBody += self._CreateRow(requirements[name], collector, levels)
table = cast(nodes.table, tableGroup.parent)
if (caption := self.options.get("caption", None)) is not None:
# the caption is ReST, not text: it is written with markup - ``packaging`` in pyTooling's own captions -
# and a 'title' built from a string would print the backticks
captionNodes, messages = self.state.inline_text(caption, self.lineno)
table.insert(0, nodes.title(caption, "", *captionNodes, *messages))
return table
@staticmethod
def _CreateEmptyRow(columnCount: int) -> nodes.row:
"""
Render the one row a table with no requirements gets: a single cell spanning every column.
A table showing nothing but its header reads as a defect. pyTooling's own :file:`requirements.txt` is empty
- the package has no mandatory dependencies - and that is a statement worth printing.
:param columnCount: Number of columns the cell has to span.
:returns: The table row.
"""
tableRow = nodes.row("", classes=["dependency-table-row"])
entry = nodes.entry("", morecols=columnCount - 1)
entry += nodes.paragraph("", "", nodes.emphasis(text="No dependencies"))
tableRow += entry
return tableRow
def _CreateRow(
self,
requirement: Requirement,
collector: DependencyCollector,
depth: Nullable[int]
) -> nodes.row:
"""
Render one required package as a table row.
A package the index doesn't know, or one with no release matching the requirement, still gets a row - the
specifier the entrypoint states is worth showing even when nothing else could be resolved.
:param requirement: The requirement to render.
:param collector: The build's collector.
:param depth: Levels of sub-dependencies still to expand.
:returns: The table row.
"""
tableRow = nodes.row("", classes=["dependency-table-row"])
project = collector.Project(requirement.name)
release = self._SelectRelease(project, requirement, collector)
tableRow += self._PackageEntry(requirement, project)
specifier = self._FormatSpecifier(requirement.specifier, self._simplify, self._versionFormat)
tableRow += nodes.entry("", nodes.paragraph(text=specifier))
tableRow += self._LicenseEntry(release)
tableRow += self._DependenciesEntry(release, collector, depth, {requirement.name.lower()})
return tableRow
def _SelectRelease(
self,
project: Nullable[Project],
requirement: Requirement,
collector: DependencyCollector
) -> Nullable[Release]:
"""
Return the newest release satisfying a requirement.
Pre-releases are skipped unless the specifier asks for them, because that is what an installer would resolve
to and the table describes what would be installed.
:param project: The project to pick a release of, or ``None`` if the index doesn't know it.
:param requirement: The requirement to satisfy.
:param collector: The build's collector.
:returns: The newest matching release, or ``None`` if nothing matches.
"""
if project is None:
return None
matching = [
release for version, release in project.Releases.items()
if requirement.specifier.contains(str(version))
]
if len(matching) == 0:
return None
return collector.Details(max(matching, key=lambda release: release.Version))
@staticmethod
def _FormatVersion(version: str, versionFormat: VersionFormat) -> str:
"""
Shorten a version number to the parts a table prints.
``≥0.4.6`` says more than a reader of a dependency table needs; the parts that matter are the ones a
constraint is usually written against. A version with fewer parts than asked for is left as it is - ``≥9``
does not become ``≥9.0`` - because padding would state a precision the requirement didn't.
:param version: The version, as the constraint writes it.
:param versionFormat: How many parts to keep.
:returns: The shortened version.
"""
if versionFormat is VersionFormat.All:
return version
parts = {VersionFormat.Major: 1, VersionFormat.MajorMinor: 2, VersionFormat.MajorMinorPatch: 3}[versionFormat]
return ".".join(version.split(".")[:parts])
@staticmethod
def _FormatSpecifier(specifier: SpecifierSet, simplify: bool, versionFormat: VersionFormat) -> str:
"""
Render a version constraint the way a reader writes one.
The comparison operators become their mathematical symbols and the versions are shortened to
``versionFormat``. A simplified constraint keeps only what a package *has to be at least*: an upper bound
and an exclusion say what a release must not be, which is the packaging problem rather than the reader's,
and ``~=`` is written as the lower bound it implies. Simplifying everything away leaves the constraint as it
was written - ``<4.0`` alone is still the whole statement.
:param specifier: The constraint to render.
:param simplify: Whether to reduce the constraint to its lower bound.
:param versionFormat: How many parts of each version to keep.
:returns: The constraint, or ``any`` when nothing is constrained.
"""
def render(operator: str, version: str) -> str:
shortened = DependencyTable._FormatVersion(version, versionFormat)
for written, symbol in OPERATOR_SYMBOLS:
if operator == written:
return f"{symbol}{shortened}"
return f"{operator}{shortened}"
if len(specifier) == 0:
return "any"
specifiers = sorted(specifier, key=lambda item: (item.version, item.operator))
if simplify:
kept = [
render(">=" if item.operator == "~=" else item.operator, item.version)
for item in specifiers if item.operator not in _DROPPED_OPERATORS
]
if len(kept) > 0:
# shortening can make two constraints identical - '>=1.2.3, >1.2.9' is '≥1.2, >1.2' at MajorMinor -
# and printing one statement twice reads as a defect
return ", ".join(dict.fromkeys(kept))
rendered = [render(item.operator, item.version) for item in specifiers]
return ", ".join(dict.fromkeys(rendered))
@staticmethod
def _PackageURL(project: Nullable[Project]) -> Nullable[str]:
"""
Return the page a package's name should link to, most useful first.
A project states none of these reliably, so there are three chances at one: its **documentation** answers
*what is this*, its **repository** answers *where does it come from*, and its page on the **package index**
is what the index itself can always answer. Only a package the index doesn't know at all goes unlinked.
:param project: The project, or ``None`` if the index doesn't know it.
:returns: The URL to link the name to, or ``None`` if there is nothing to link to.
"""
if project is None:
return None
for url in (project.DocumentationURL, project.RepositoryURL, project.URL):
if url is not None:
return str(url)
return None
@classmethod
def _PackageEntry(cls, requirement: Requirement, project: Nullable[Project]) -> nodes.entry:
"""
Render the package's name, linked to where a reader can find out about it.
:param requirement: The requirement naming the package.
:param project: The project, or ``None`` if the index doesn't know it.
:returns: The table entry.
"""
entry = nodes.entry()
name = project.Name if project is not None else requirement.name
if (url := cls._PackageURL(project)) is not None:
entry += nodes.paragraph("", "", nodes.reference("", name, refuri=url))
else:
entry += nodes.paragraph(text=name)
return entry
@staticmethod
def _LicenseEntry(release: Nullable[Release]) -> nodes.entry:
"""
Render a release's license, linked to its text where one is known.
The license' **name** is shown rather than its SPDX identifier - ``Apache License 2.0``, not ``Apache-2.0`` -
because the table is read by a person and the identifier is what an expression writes.
A license that didn't resolve is an :class:`~pyTooling.Licensing.UnknownLicense`, never a blank cell, and it
is shown **as the index published it** - in italics, so it reads as a quotation rather than as an identifier.
The reader should see that the index said *something*, and what it was.
:param release: The release to render the license of, or ``None``.
:returns: The table entry.
"""
entry = nodes.entry()
if (text := DependencyTable._LicenseName(release)) is None:
entry += nodes.paragraph("", "", nodes.emphasis(text=DependencyTable._PublishedLicense(release)))
return entry
if (url := DependencyTable._LicenseURL(release)) is not None:
entry += nodes.paragraph("", "", nodes.reference("", text, refuri=url))
else:
entry += nodes.paragraph(text=text)
return entry
@staticmethod
def _LicenseName(release: Nullable[Release]) -> Nullable[str]:
"""
Return the name(s) of the licenses a release is published under.
:param release: The release to name the license of, or ``None``.
:returns: The license' name, or ``None`` if nothing resolved.
"""
from pyTooling.Licensing import UnknownLicense
if release is None:
return None
# 'Licenses' never comes back empty: what didn't resolve is an 'UnknownLicense', which is SPDX's own way of
# saying so and keeps the published text
licenses = release.Licenses
if all(isinstance(license, UnknownLicense) for license in licenses):
return None
return ", ".join(license.Name for license in licenses)
@staticmethod
def _PublishedLicense(release: Nullable[Release]) -> str:
"""
Return what the package index published, for a license that didn't resolve.
:param release: The release, or ``None`` if the index couldn't describe it.
:returns: What was published, or ``unknown`` when that was nothing either.
"""
if release is None:
return "unknown"
published = release.LicenseExpression.OriginalText.strip()
return published if published != "" else ", ".join(license.Name for license in release.Licenses) or "unknown"
@staticmethod
def _LicenseURL(release: Nullable[Release]) -> Nullable[str]:
"""
Return the page a license should link to, most specific first.
**The project's own** :file:`LICENSE` **file wins**: it is the license as this project publishes it, which
is the document a reader auditing a dependency actually wants. Most projects don't state one, though - it
comes from ``project_urls`` or from the override file - so a license on the SPDX List falls back to its own
published pages, in the order of who is speaking: the licensor's own page, then OSI's entry, then SPDX's.
A ``LicenseRef-`` has none of those and stays unlinked, because nothing published it.
:param release: The release to link the license of, or ``None``.
:returns: The URL to link to, or ``None`` if nothing published this license.
"""
from pyTooling.Licensing import SPDXLicense
if release is None:
return None
if release.LicenseURL is not None:
return str(release.LicenseURL)
for license in release.Licenses:
if isinstance(license, SPDXLicense):
for url in (license.License.URL, license.License.OSIURL, license.License.SPDXURL):
if url is not None:
return url
return None
def _DependenciesEntry(
self,
release: Nullable[Release],
collector: DependencyCollector,
depth: Nullable[int],
visited: set[str]
) -> nodes.entry:
"""
Render a release's own requirements as a nested bullet list.
Only the unconditional requirements are listed - what an extra pulls in is that extra's table, not this one.
A package already on the path is not expanded again, so a dependency cycle terminates.
:param release: The release to render the dependencies of, or ``None``.
:param collector: The build's collector.
:param depth: Levels still to expand; at zero nothing is expanded.
:param visited: Packages already on this path, lower-cased.
:returns: The table entry.
"""
entry = nodes.entry()
if release is None or (depth is not None and depth <= 0):
entry += nodes.paragraph("", "", nodes.emphasis(text="not evaluated"))
return entry
requirements = [
requirement for requirement in release.Requirements.get(None, [])
if requirement.name.lower() not in visited
]
if len(requirements) == 0:
entry += nodes.paragraph("", "", nodes.emphasis(text="none"))
return entry
entry += self._CreateBulletList(requirements, collector, _OneLevelDown(depth), visited)
return entry
def _CreateBulletList(
self,
requirements: list[Requirement],
collector: DependencyCollector,
depth: Nullable[int],
visited: set[str]
) -> nodes.bullet_list:
"""
Render requirements as a bullet list, each item expanded by one more level.
:param requirements: The requirements to list.
:param collector: The build's collector.
:param depth: Levels still to expand below this list.
:param visited: Packages already on this path, lower-cased.
:returns: The bullet list.
"""
bulletList = nodes.bullet_list()
for requirement in sorted(requirements, key=lambda item: item.name.lower()):
item = nodes.list_item()
# a leaf is resolved too when the line states a license - that is the whole point of stating it - but a
# ':dependency-format:' that prints no license has no reason to send the requests
project = collector.Project(requirement.name)
release = (
self._SelectRelease(project, requirement, collector)
if depth is None or depth > 0 or self._dependencyFormat.ShowsLicense
else None
)
item += self._RequirementParagraph(requirement, project, release)
if (depth is None or depth > 0) and release is not None:
nested = [
nestedRequirement for nestedRequirement in release.Requirements.get(None, [])
if nestedRequirement.name.lower() not in visited
]
if len(nested) > 0:
item += self._CreateBulletList(
nested, collector, _OneLevelDown(depth), visited | {requirement.name.lower()}
)
bulletList += item
return bulletList
def _RequirementParagraph(
self,
requirement: Requirement,
project: Nullable[Project],
release: Nullable[Release]
) -> nodes.paragraph:
"""
Render one line of a dependency tree: the package, what is required of it, and what it is licensed under.
The license is the reason a dependency tree is in this table at all - a package pulls in what its own
dependencies are licensed under, and reading that off the tree is the point. It is linked and parenthesised
so the line still reads as one requirement.
:param requirement: The requirement to render.
:param project: The project, or ``None`` if the index doesn't know it.
:param release: The release satisfying the requirement, or ``None`` if none was found.
:returns: The paragraph.
"""
paragraph = nodes.paragraph()
name = project.Name if project is not None else requirement.name
if (url := self._PackageURL(project)) is not None:
paragraph += nodes.reference("", name, refuri=url)
else:
paragraph += nodes.Text(name)
if self._dependencyFormat.ShowsVersion:
specifier = self._FormatSpecifier(requirement.specifier, self._simplify, self._versionFormat)
if specifier != "any":
paragraph += nodes.Text(f" {specifier}")
if self._dependencyFormat.ShowsLicense:
paragraph += nodes.Text(" (")
if (license := self._LicenseName(release)) is None:
# the same statement the License column makes: what the index published, in italics, so it reads as
# a quotation rather than as an identifier
paragraph += nodes.emphasis(text=self._PublishedLicense(release))
elif (licenseURL := self._LicenseURL(release)) is not None:
paragraph += nodes.reference("", license, refuri=licenseURL)
else:
paragraph += nodes.Text(license)
paragraph += nodes.Text(")")
return paragraph
@export
def formatUnresolvedLicenses(unresolved: Mapping[str, tuple[str, ...]]) -> str:
"""
Describe the packages needing a license override, grouped by what the package index published for them.
Grouped rather than listed one per line, because one ambiguous statement usually accounts for most of the list:
``License :: OSI Approved :: BSD License`` names three licenses, so every package whose only license information
is that classifier lands here for the same reason and is worth reading as one group.
:param unresolved: Names of the packages needing an override, mapped to what the index published for them.
:returns: The message, as one line naming the count and two lines per reason.
"""
byReason: dict[tuple[str, ...], list[str]] = {}
for packageName, published in sorted(unresolved.items()):
byReason.setdefault(published, []).append(packageName)
lines = [f"[dependency-table] {len(unresolved)} package(s) need a license override:"]
# the biggest group first, so the statement to fix first is the one at the top
for published, packageNames in sorted(byReason.items(), key=lambda item: (-len(item[1]), item[0])):
reason = "; ".join(published) if len(published) > 0 else "the index published no license information"
lines.append(f" {reason}")
lines.append(f" {', '.join(packageNames)}")
return "\n".join(lines)
def reportBuildTime(app: Sphinx, exception: Nullable[Exception]) -> None:
"""
Report what querying the package index cost this build.
The tables are fetched live, so this is the number to look at before deciding what a cache would be worth. The
packages whose license had to be guessed at - or couldn't be - are named too, because that is the list the
override file has to answer for.
:param app: The Sphinx application that finished building.
:param exception: The exception that ended the build, or ``None`` if it succeeded.
"""
if (collector := _COLLECTORS.pop(id(app), None)) is None or collector.RequestCount == 0:
return
_logger.info(
f"[dependency-table] {collector.RequestCount} request(s) to the package index, "
f"{collector.Seconds:.2f} s in total."
)
if len(unresolved := collector.UnresolvedLicenses) > 0:
_logger.warning(formatUnresolvedLicenses(unresolved))
|