Coverage for pyTooling/Configuration/YAML.py: 97%

1# ==================================================================================================================== #

2# _____ _ _ ____ __ _ _ _ #

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ / ___|___ _ __ / _(_) __ _ _ _ _ __ __ _| |_(_) ___ _ __ #

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` || | / _ \| '_ \| |_| |/ _` | | | | '__/ _` | __| |/ _ \| '_ \ #

5# | |_) | |_| || | (_) | (_) | | | | | | (_| || |__| (_) | | | | _| | (_| | |_| | | | (_| | |_| | (_) | | | | #

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____\___/|_| |_|_| |_|\__, |\__,_|_| \__,_|\__|_|\___/|_| |_| #

7# |_| |___/ |___/ |___/ #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32Configuration reader for YAML files.

34:raises MissingDependencyError: If the 'yaml' extra isn't installed.

36.. hint::

38 See :ref:`high-level help <CONFIG/FileFormat/YAML>` for explanations and usage examples.

39"""

40from __future__ import annotations

42from pathlib import Path

43from datetime import date, datetime

44from typing import Any, Union, Iterator as typing_Iterator, Optional as Nullable, Self

46from pyTooling.Exceptions import MissingDependencyError

48try:

49 from ruamel.yaml import YAML, CommentedMap, CommentedSeq

50except ImportError as ex: # pragma: no cover

51 raise MissingDependencyError(dependency="ruamel.yaml", extra="yaml") from ex

53from pyTooling.Common import getFullyQualifiedName

54from pyTooling.Decorators import export, InheritDocString

55from pyTooling.MetaClasses import ExtendedType

56from pyTooling.Configuration import ConfigurationError, KeyT, NodeT, ValueT

57from pyTooling.Configuration import InterpolationError, KeyNotFoundError, PathExpressionError

58from pyTooling.Configuration import UnsupportedValueTypeError

59from pyTooling.Configuration import Node as Abstract_Node

60from pyTooling.Configuration import Dictionary as Abstract_Dict

61from pyTooling.Configuration import Sequence as Abstract_Seq

62from pyTooling.Configuration import Configuration as Abstract_Configuration

65@export

66class Node(Abstract_Node):

67 """

68 Node in a YAML configuration data structure.

69 """

71 _yamlNode: Union[CommentedMap, CommentedSeq] #: Reference to the associated YAML node.

72 _cache: dict[str, ValueT] #: Cache of already converted sub-nodes and values, by key.

73 _key: KeyT #: Key of this node.

74 _length: int #: Number of sub-elements.

76 def __init__(

77 self,

78 root: Configuration,

79 parent: Nullable[NodeT],

80 key: KeyT,

81 yamlNode: Union[CommentedMap, CommentedSeq]

82 ) -> None:

83 """

84 Initializes a YAML node.

86 :param root: Reference to the root node.

87 :param parent: Reference to the parent node, or ``None`` for the root node.

88 :param key: Key of the node within its parent.

89 :param yamlNode: Reference to the YAML node.

90 """

91 Abstract_Node.__init__(self, root, parent)

93 self._yamlNode = yamlNode

94 self._cache = {}

95 self._key = key

96 self._length = len(yamlNode)

98 @InheritDocString(Abstract_Node)

99 def __len__(self) -> int:

100 return self._length

101

102 @InheritDocString(Abstract_Node)

103 def __getitem__(self, key: KeyT) -> ValueT:

104 return self._GetNodeOrValue(str(key))

105

106 @property

107 def Key(self) -> KeyT:

108 """

109 Property to access the node's key.

110

111 :returns: Key of the node.

112 :raises NotImplementedError: If a new key is assigned; renaming a key is not supported by this configuration

113 implementation.

114 """

115 return self._key

116

117 @Key.setter

118 def Key(self, value: KeyT) -> None:

119 raise NotImplementedError("Renaming a key isn't supported by this configuration implementation.")

120

121 @InheritDocString(Abstract_Node)

122 def QueryPath(self, query: str) -> ValueT:

123 path = self._ToPath(query)

124 return self._GetNodeOrValueByPathExpression(path)

125

126 @staticmethod

127 def _ToPath(query: str) -> list[Union[str, int]]:

128 """

129 Split a path expression into its elements.

130

131 :param query: Path expression, with its elements separated by ``:``.

132 :returns: List of keys and indices.

133 """

134 return query.split(":")

135

136 def _LookupKey(self, key: str) -> Any:

137 """

138 Look up a key in the YAML node, trying it as string, integer and float.

139

140 :param key: Key or index to look up.

141 :returns: The raw value as returned by the YAML parser.

142 :raises KeyNotFoundError: If the key exists neither as string, nor as integer or float.

143 """

144 try:

145 return self._yamlNode[key]

146 except (KeyError, TypeError):

147 pass

148

149 for conversion in (int, float):

150 try:

151 convertedKey = conversion(key)

152 except ValueError:

153 continue

154

155 try:

156 return self._yamlNode[convertedKey]

157 except (KeyError, IndexError, TypeError):

158 pass

159

160 ex = KeyNotFoundError(f"Key '{key}' not found in node '{self._key}'.")

161 ex.add_note(self._DescribeKeys())

162 raise ex

163

164 def _DescribeKeys(self) -> str:

165 """

166 Describe the keys or indices offered by this node, so it can be used as an exception note.

167

168 :returns: A one-line description of the node's keys or index range.

169 """

170 if isinstance(self._yamlNode, CommentedMap):

171 if self._length == 0:

172 return f"Node '{self._key}' is an empty dictionary."

173

174 keys = "', '".join(str(key) for key in self._yamlNode)

175 return f"Available keys: '{keys}'."

176 else:

177 if self._length == 0:

178 return f"Node '{self._key}' is an empty sequence."

179

180 return f"Node '{self._key}' is a sequence with indices 0..{self._length - 1}."

181

182 def _GetNodeOrValue(self, key: str) -> ValueT:

183 """

184 Return a sub-node or a value by key, converting it on first access.

185

186 The converted object is cached, so a second access returns the same node object rather than a new one.

187

188 A date or a datetime is a scalar too - YAML reads ``2026-09-02`` and ``2026-09-02T10:30:00`` as those - and is

189 returned in ISO-8601 spelling. Like every other scalar it comes back as a :class:`str`, so

190 :meth:`datetime.date.fromisoformat` turns it back into a date where a caller wants one.

191

192 :param key: Key or index to look up.

193 :returns: A dictionary node, a sequence node, or a scalar value with its variables

194 resolved.

195 :raises KeyNotFoundError: If the key doesn't exist in this node.

196 :raises UnsupportedValueTypeError: If the YAML parser returned a value that is neither a scalar, nor a

197 node.

198 """

199 try:

200 value = self._cache[key]

201 except KeyError:

202 value = self._LookupKey(key)

203

204 if isinstance(value, str):

205 value = self._ResolveVariables(value)

206 elif isinstance(value, (int, float)):

207 value = str(value)

208 # YAML reads an unquoted '2026-09-02' as a date and '2026-09-02T10:30:00' as a datetime, which no other

209 # scalar type covers. 'isoformat' rather than 'str', because 'str' writes a datetime with a space where

210 # ISO-8601 writes a 'T'. JSON has no date type, so its backend needs no such branch.

211 elif isinstance(value, (date, datetime)):

212 value = value.isoformat()

213 elif isinstance(value, CommentedMap):

214 value = self.DICT_TYPE(self._root, self, key, value)

215 elif isinstance(value, CommentedSeq):

216 value = self.SEQ_TYPE(self._root, self, key, value)

217 else:

218 typeName = getFullyQualifiedName(value)

219 ex = UnsupportedValueTypeError(f"Unsupported type '{typeName}' for key '{key}' in node '{self._key}'.")

220 ex.add_note("The YAML parser returned a value that is neither a scalar (str, int, float, date, "

221 "datetime), nor a map or sequence.")

222 raise ex

223

224 self._cache[key] = value

225

226 return value

227

228 def _ResolveVariables(self, value: str) -> str:

229 """

230 Resolve the ``${...}`` variables inside a value.

231

232 A variable references another node by a path expression, so a value can be composed from other values of the

233 same configuration.

234

235 :param value: The raw value, possibly containing variables.

236 :returns: The value with every variable replaced by what it references.

237 :raises InterpolationError: If a variable is malformed - a dangling ``$`` at the end of the value, or a

238 missing closing ``}`` for a ``${`` at some position. |br|

239 Use ``$$`` to escape a literal dollar sign.

240 :raises KeyNotFoundError: If a referenced key doesn't exist.

241 """

242 if value == "": 242 ↛ 243line 242 didn't jump to line 243 because the condition on line 242 was never true

243 return ""

244 elif "$" not in value:

245 return value

246

247 rawValue = value

248 result = ""

249

250 while (len(rawValue) > 0):

251# print(f"_ResolveVariables: LOOP rawValue='{rawValue}'")

252 beginPos = rawValue.find("$")

253 if beginPos < 0:

254 result += rawValue

255 rawValue = ""

256 else:

257 result += rawValue[:beginPos]

258 if beginPos + 1 >= len(rawValue):

259 ex = InterpolationError(f"Dangling '$' at the end of value '{value}'.")

260 ex.add_note("Use '$$' to escape a literal dollar sign.")

261 raise ex

262 elif rawValue[beginPos + 1] == "$": 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true

263 result += "$"

264 rawValue = rawValue[1:]

265 elif rawValue[beginPos + 1] == "{": 265 ↛ 250line 265 didn't jump to line 250 because the condition on line 265 was always true

266 endPos = rawValue.find("}", beginPos)

267 nextPos = rawValue.rfind("$", beginPos, endPos)

268 if endPos < 0:

269 ex = InterpolationError(f"Unclosed variable reference in value '{value}'.")

270 ex.add_note(f"Missing closing '}}' for the '${{' at position {beginPos}.")

271 raise ex

272

273 if (nextPos > 0) and (nextPos < endPos): # an embedded $-sign

274 path = rawValue[nextPos+2:endPos]

275# print(f"_ResolveVariables: path='{path}'")

276 innervalue = self._GetValueByPathExpression(self._ToPath(path))

277# print(f"_ResolveVariables: innervalue='{innervalue}'")

278 rawValue = rawValue[beginPos:nextPos] + str(innervalue) + rawValue[endPos + 1:]

279# print(f"_ResolveVariables: new rawValue='{rawValue}'")

280 else:

281 path = rawValue[beginPos+2:endPos]

282 rawValue = rawValue[endPos+1:]

283 result += str(self._GetValueByPathExpression(self._ToPath(path)))

284

285 return result

286

287 def _GetValueByPathExpression(self, path: list[KeyT]) -> ValueT:

288 """

289 Return the value the given path refers to.

290

291 :param path: Path elements, where ``..`` selects the parent node.

292 :returns: The scalar value at that path.

293 :raises KeyNotFoundError: If a path element doesn't exist.

294 :raises PathExpressionError: If the path resolves to a node instead of a value - extend the path expression

295 to address a scalar value - or if a ``..`` element is applied to the root node,

296 which has no parent.

297 """

298 node = self

299 for p in path:

300 if p == "..":

301 if node._parent is None:

302 pathExpression = ":".join(str(element) for element in path)

303 ex = PathExpressionError(f"Path expression '{pathExpression}' navigates beyond the root node.")

304 ex.add_note("Element '..' was applied to the root node, which has no parent node.")

305 raise ex

306

307 node = node._parent

308 else:

309 node = node._GetNodeOrValue(p)

310

311 if isinstance(node, Dictionary):

312 pathExpression = ":".join(str(element) for element in path)

313 ex = PathExpressionError(f"Path expression '{pathExpression}' resolves to a dictionary, not to a value.")

314 ex.add_note(f"Element '{p}' is a dictionary. Extend the path expression to address a scalar value.")

315 raise ex

316

317 return node

318

319 def _GetNodeOrValueByPathExpression(self, path: list[KeyT]) -> ValueT:

320 """

321 Return the node or value the given path refers to.

322

323 :param path: Path elements, where ``..`` selects the parent node.

324 :returns: A node or a scalar value at that path.

325 :raises KeyNotFoundError: If a path element doesn't exist.

326 :raises PathExpressionError: If a ``..`` element is applied to the root node, which has no parent.

327 """

328 node = self

329 for p in path:

330 if p == "..":

331 if node._parent is None: 331 ↛ 337line 331 didn't jump to line 337 because the condition on line 331 was always true

332 pathExpression = ":".join(str(element) for element in path)

333 ex = PathExpressionError(f"Path expression '{pathExpression}' navigates beyond the root node.")

334 ex.add_note("Element '..' was applied to the root node, which has no parent node.")

335 raise ex

336

337 node = node._parent

338 else:

339 node = node._GetNodeOrValue(p)

340

341 return node

342

343

344@export

345class Dictionary(Node, Abstract_Dict):

346 """A dictionary node in a YAML data file."""

347

348

349 def __init__(

350 self,

351 root: Configuration,

352 parent: Nullable[NodeT],

353 key: KeyT,

354 yamlNode: CommentedMap

355 ) -> None:

356 """

357 Initializes a YAML dictionary.

358

359 :param root: Reference to the root node.

360 :param parent: Reference to the parent node, or ``None`` for the root node.

361 :param key: Key of the node within its parent.

362 :param yamlNode: Reference to the YAML node.

363 """

364 keys: list[KeyT] = [str(k) for k in yamlNode.keys()]

365

366 Node.__init__(self, root, parent, key, yamlNode)

367 Abstract_Dict.__init__(self, keys)

368

369 def __iter__(self) -> typing_Iterator[ValueT]:

370 """

371 Returns an iterator to iterate dictionary keys.

372

373 :returns: Dictionary key iterator.

374 """

375

376 class Iterator(metaclass=ExtendedType, slots=True):

377 """Iterator to iterate dictionary items."""

378

379 _iter: typing_Iterator[ValueT] #: Iterator over the underlying dictionary's keys.

380 _obj: Dictionary #: The dictionary being iterated.

381

382 def __init__(self, obj: Dictionary) -> None:

383 """

384 Initializes an iterator for a YAML dictionary node.

385

386 :param obj: YAML dictionary to iterate.

387 """

388 self._iter = iter(obj._keys)

389 self._obj = obj

390

391 def __iter__(self) -> Self:

392 """

393 Return itself to fulfil the iterator protocol.

394

395 :returns: Itself.

396 """

397 return self # pragma: no cover

398

399 def __next__(self) -> ValueT:

400 """

401 Returns the next item in the dictionary.

402

403 :returns: Next item.

404 """

405 key = next(self._iter)

406 return self._obj[key]

407

408 return Iterator(self)

409

410

411@export

412class Sequence(Node, Abstract_Seq):

413 """A sequence node (ordered list) in a YAML data file."""

414

415 def __init__(

416 self,

417 root: Configuration,

418 parent: NodeT,

419 key: KeyT,

420 yamlNode: CommentedSeq

421 ) -> None:

422 """

423 Initializes a YAML sequence (list).

424

425 :param root: Reference to the root node.

426 :param parent: Reference to the parent node.

427 :param key: Key of the node within its parent.

428 :param yamlNode: Reference to the YAML node.

429 """

430 Node.__init__(self, root, parent, key, yamlNode)

431

432 self._length = len(yamlNode)

433

434 def __iter__(self) -> typing_Iterator[ValueT]:

435 """

436 Returns an iterator to iterate items in the sequence of sub-nodes.

437

438 :returns: Iterator to iterate items in a sequence.

439 """

440 class Iterator(metaclass=ExtendedType, slots=True):

441 """Iterator to iterate sequence items."""

442

443 _i: int #: internal iterator position

444 _obj: Sequence #: Sequence object to iterate

445

446 def __init__(self, obj: Sequence) -> None:

447 """

448 Initializes an iterator for a YAML sequence node.

449

450 :param obj: YAML sequence to iterate.

451 """

452 self._i = 0

453 self._obj = obj

454

455 def __iter__(self) -> Self:

456 """

457 Return itself to fulfil the iterator protocol.

458

459 :returns: Itself.

460 """

461 return self # pragma: no cover

462

463 def __next__(self) -> ValueT:

464 """

465 Returns the next item in the sequence.

466

467 :returns: Next item.

468 :raises StopIteration: If end of sequence is reached.

469 """

470 if self._i >= len(self._obj):

471 raise StopIteration

472

473 result = self._obj[str(self._i)]

474 self._i += 1

475 return result

476

477 return Iterator(self)

478

479

480setattr(Node, "DICT_TYPE", Dictionary)

481setattr(Node, "SEQ_TYPE", Sequence)

482

483

484@export

485class Configuration(Dictionary, Abstract_Configuration):

486 """A configuration read from a YAML file."""

487

488 _yamlConfig: YAML #: The parsed YAML document this configuration is based on.

489

490 def __init__(self, configFile: Path) -> None:

491 """

492 Initializes a configuration instance that reads a YAML file as input.

493

494 All sequence items or dictionaries key-value-pairs in the YAML file are accessible via Python's dictionary syntax.

495

496 A configuration's root **is** a mapping - this class derives from :class:`Dictionary` - so a document

497 describing anything else is rejected here rather than failing on the first access. A document with **no

498 content** carries no settings and reads as an empty configuration; :mod:`pyTooling.Configuration.JSON` does

499 the same, so both formats answer alike.

500

501 :param configFile: Configuration file to read and parse.

502 :raises ConfigurationError: If the YAML file doesn't exist.

503 :raises ConfigurationError: If the YAML file's root isn't a mapping. |br|

504 An empty file and an explicit ``null`` are the exception: both are *no settings*

505 and read as an empty configuration.

506 """

507 if not configFile.exists():

508 raise ConfigurationError(f"YAML configuration file '{configFile}' not found.") from FileNotFoundError(configFile)

509

510 with configFile.open("r", encoding="utf-8") as file:

511 document = YAML().load(file)

512

513 # 'load' returns None for an empty file and for an explicit 'null' - a *null document*, which is valid YAML.

514 if document is None:

515 document = CommentedMap()

516 elif not isinstance(document, CommentedMap):

517 ex = ConfigurationError(f"YAML configuration file '{configFile}' doesn't describe a mapping.")

518 ex.add_note(f"Got type '{getFullyQualifiedName(document)}' at the document's root.")

519 ex.add_note("A configuration needs a mapping of keys to values at its root.")

520 raise ex

521

522 self._yamlConfig = document

523

524 Dictionary.__init__(self, self, None, None, self._yamlConfig)

525 Abstract_Configuration.__init__(self, configFile)

526

527 def __getitem__(self, key: str) -> ValueT:

528 """

529 Access a configuration node by key.

530

531 :param key: The key to look for.

532 :returns: A node (sequence or dictionary) or scalar value (int, float, str).

533 """

534 return self._GetNodeOrValue(str(key))

535

536 #

537 # :param key: Key of the value to write.

538 # :param value: The new value.

539 # :raises NotImplementedError: Writing a configuration is not supported by this implementation.

540 # """

541 # raise NotImplementedError()