Coverage for pyTooling/Configuration/YAML.py: 97%

1# ==================================================================================================================== #

2# _____ _ _ ____ __ _ _ _ #

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ / ___|___ _ __ / _(_) __ _ _ _ _ __ __ _| |_(_) ___ _ __ #

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` || | / _ \| '_ \| |_| |/ _` | | | | '__/ _` | __| |/ _ \| '_ \ #

5# | |_) | |_| || | (_) | (_) | | | | | | (_| || |__| (_) | | | | _| | (_| | |_| | | | (_| | |_| | (_) | | | | #

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____\___/|_| |_|_| |_|\__, |\__,_|_| \__,_|\__|_|\___/|_| |_| #

7# |_| |___/ |___/ |___/ #

8# ==================================================================================================================== #

9# Authors: #

10# Patrick Lehmann #

11# #

12# License: #

13# ==================================================================================================================== #

15# #

16# Licensed under the Apache License, Version 2.0 (the "License"); #

17# you may not use this file except in compliance with the License. #

18# You may obtain a copy of the License at #

19# #

20# http://www.apache.org/licenses/LICENSE-2.0 #

21# #

22# Unless required by applicable law or agreed to in writing, software #

23# distributed under the License is distributed on an "AS IS" BASIS, #

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #

25# See the License for the specific language governing permissions and #

26# limitations under the License. #

27# #

28# SPDX-License-Identifier: Apache-2.0 #

29# ==================================================================================================================== #

30#

31"""

32Configuration reader for YAML files.

34:raises MissingDependencyError: If the 'yaml' extra isn't installed.

36.. hint::

38 See :ref:`high-level help <CONFIG/FileFormat/YAML>` for explanations and usage examples.

39"""

40from __future__ import annotations

42from pathlib import Path

43from datetime import date, datetime

44from typing import Any, Union, Iterator as typing_Iterator, Optional as Nullable, Self

46from pyTooling.Exceptions import MissingDependencyError

48try:

49 from ruamel.yaml import YAML, CommentedMap, CommentedSeq

50except ImportError as ex: # pragma: no cover

51 raise MissingDependencyError(dependency="ruamel.yaml", extra="yaml") from ex

53from pyTooling.Common import getFullyQualifiedName

54from pyTooling.Decorators import export, InheritDocString

55from pyTooling.MetaClasses import ExtendedType

56from pyTooling.Configuration import ConfigurationError, KeyT, NodeT, ValueT

57from pyTooling.Configuration import InterpolationError, KeyNotFoundError, PathExpressionError

58from pyTooling.Configuration import UnsupportedValueTypeError

59from pyTooling.Configuration import Node as Abstract_Node

60from pyTooling.Configuration import Dictionary as Abstract_Dict

61from pyTooling.Configuration import Sequence as Abstract_Seq

62from pyTooling.Configuration import Configuration as Abstract_Configuration

65@export

66class Node(Abstract_Node):

67 """

68 Node in a YAML configuration data structure.

69 """

71 _yamlNode: Union[CommentedMap, CommentedSeq] #: Reference to the associated YAML node.

72 _cache: dict[str, ValueT] #: Cache of already converted sub-nodes and values, by key.

73 _key: KeyT #: Key of this node.

74 _length: int #: Number of sub-elements.

76 def __init__(

77 self,

78 root: Configuration,

79 parent: Nullable[NodeT],

80 key: KeyT,

81 yamlNode: Union[CommentedMap, CommentedSeq]

82 ) -> None:

83 """

84 Initializes a YAML node.

86 :param root: Reference to the root node.

87 :param parent: Reference to the parent node, or ``None`` for the root node.

88 :param key: Key of the node within its parent.

89 :param yamlNode: Reference to the YAML node.

90 """

91 Abstract_Node.__init__(self, root, parent)

93 self._yamlNode = yamlNode

94 self._cache = {}

95 self._key = key

96 self._length = len(yamlNode)

98 @InheritDocString(Abstract_Node)

99 def __len__(self) -> int:

100 return self._length

101

102 @InheritDocString(Abstract_Node)

103 def __getitem__(self, key: KeyT) -> ValueT:

104 return self._GetNodeOrValue(str(key))

105

106 @property

107 def Key(self) -> KeyT:

108 """

109 Property to access the node's key.

110

111 :returns: Key of the node.

112 :raises NotImplementedError: If a new key is assigned; renaming a key is not supported by this configuration

113 implementation.

114 """

115 return self._key

116

117 @Key.setter

118 def Key(self, value: KeyT) -> None:

119 raise NotImplementedError("Renaming a key isn't supported by this configuration implementation.")

120

121 @InheritDocString(Abstract_Node)

122 def QueryPath(self, query: str) -> ValueT:

123 path = self._ToPath(query)

124 return self._GetNodeOrValueByPathExpression(path)

125

126 @staticmethod

127 def _ToPath(query: str) -> list[Union[str, int]]:

128 """

129 Split a path expression into its elements.

130

131 :param query: Path expression, with its elements separated by ``:``.

132 :returns: List of keys and indices.

133 """

134 return query.split(":")

135

136 def _LookupKey(self, key: str) -> Any:

137 """

138 Look up a key in the YAML node, trying it as string, integer and float.

139

140 :param key: Key or index to look up.

141 :returns: The raw value as returned by the YAML parser.

142 :raises KeyNotFoundError: If the key exists neither as string, nor as integer or float.

143 """

144 try:

145 return self._yamlNode[key]

146 except (KeyError, TypeError):

147 pass

148

149 for conversion in (int, float):

150 try:

151 convertedKey = conversion(key)

152 except ValueError:

153 continue

154

155 try:

156 return self._yamlNode[convertedKey]

157 except (KeyError, IndexError, TypeError):

158 pass

159

160 ex = KeyNotFoundError(f"Key '{key}' not found in node '{self._key}'.")

161 ex.add_note(self._DescribeKeys())

162 raise ex

163

164 def _DescribeKeys(self) -> str:

165 """

166 Describe the keys or indices offered by this node, so it can be used as an exception note.

167

168 :returns: A one-line description of the node's keys or index range.

169 """

170 if isinstance(self._yamlNode, CommentedMap):

171 if self._length == 0:

172 return f"Node '{self._key}' is an empty dictionary."

173

174 keys = "', '".join(str(key) for key in self._yamlNode)

175 return f"Available keys: '{keys}'."

176 else:

177 if self._length == 0:

178 return f"Node '{self._key}' is an empty sequence."

179

180 return f"Node '{self._key}' is a sequence with indices 0..{self._length - 1}."

181

182 def _GetNodeOrValue(self, key: str) -> ValueT:

183 """

184 Return a sub-node or a value by key, converting it on first access.

185

186 The converted object is cached, so a second access returns the same node object rather than a new one.

187

188 A date or a datetime is a scalar too - YAML reads ``2026-09-02`` and ``2026-09-02T10:30:00`` as those - and is

189 returned in ISO-8601 spelling. Like every other scalar it comes back as a :class:`str`, so

190 :meth:`datetime.date.fromisoformat` turns it back into a date where a caller wants one.

191

192 :param key: Key or index to look up.

193 :returns: A dictionary node, a sequence node, a scalar value with its variables

194 resolved, or ``None`` for a null value.

195 :raises KeyNotFoundError: If the key doesn't exist in this node.

196 :raises UnsupportedValueTypeError: If the YAML parser returned a value that is neither a scalar, nor a

197 node.

198 """

199 try:

200 value = self._cache[key]

201 except KeyError:

202 value = self._LookupKey(key)

203

204 if value is None:

205 pass

206 elif isinstance(value, str):

207 value = self._ResolveVariables(value)

208 elif isinstance(value, (int, float)):

209 value = str(value)

210 # YAML reads an unquoted '2026-09-02' as a date and '2026-09-02T10:30:00' as a datetime, which no other

211 # scalar type covers. 'isoformat' rather than 'str', because 'str' writes a datetime with a space where

212 # ISO-8601 writes a 'T'. JSON has no date type, so its backend needs no such branch.

213 elif isinstance(value, (date, datetime)):

214 value = value.isoformat()

215 elif isinstance(value, CommentedMap):

216 value = self.DICT_TYPE(self._root, self, key, value)

217 elif isinstance(value, CommentedSeq):

218 value = self.SEQ_TYPE(self._root, self, key, value)

219 else:

220 typeName = getFullyQualifiedName(value)

221 ex = UnsupportedValueTypeError(f"Unsupported type '{typeName}' for key '{key}' in node '{self._key}'.")

222 ex.add_note("The YAML parser returned a value that is neither a scalar (str, int, float, date, "

223 "datetime), nor a map or sequence.")

224 raise ex

225

226 self._cache[key] = value

227

228 return value

229

230 def _ResolveVariables(self, value: str) -> str:

231 """

232 Resolve the ``${...}`` variables inside a value.

233

234 A variable references another node by a path expression, so a value can be composed from other values of the

235 same configuration.

236

237 :param value: The raw value, possibly containing variables.

238 :returns: The value with every variable replaced by what it references.

239 :raises InterpolationError: If a variable is malformed - a dangling ``$`` at the end of the value, or a

240 missing closing ``}`` for a ``${`` at some position. |br|

241 Use ``$$`` to escape a literal dollar sign.

242 :raises KeyNotFoundError: If a referenced key doesn't exist.

243 """

244 if value == "": 244 ↛ 245line 244 didn't jump to line 245 because the condition on line 244 was never true

245 return ""

246 elif "$" not in value:

247 return value

248

249 rawValue = value

250 result = ""

251

252 while (len(rawValue) > 0):

253# print(f"_ResolveVariables: LOOP rawValue='{rawValue}'")

254 beginPos = rawValue.find("$")

255 if beginPos < 0:

256 result += rawValue

257 rawValue = ""

258 else:

259 result += rawValue[:beginPos]

260 if beginPos + 1 >= len(rawValue):

261 ex = InterpolationError(f"Dangling '$' at the end of value '{value}'.")

262 ex.add_note("Use '$$' to escape a literal dollar sign.")

263 raise ex

264 elif rawValue[beginPos + 1] == "$": 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true

265 result += "$"

266 rawValue = rawValue[1:]

267 elif rawValue[beginPos + 1] == "{": 267 ↛ 252line 267 didn't jump to line 252 because the condition on line 267 was always true

268 endPos = rawValue.find("}", beginPos)

269 nextPos = rawValue.rfind("$", beginPos, endPos)

270 if endPos < 0:

271 ex = InterpolationError(f"Unclosed variable reference in value '{value}'.")

272 ex.add_note(f"Missing closing '}}' for the '${{' at position {beginPos}.")

273 raise ex

274

275 if (nextPos > 0) and (nextPos < endPos): # an embedded $-sign

276 path = rawValue[nextPos+2:endPos]

277# print(f"_ResolveVariables: path='{path}'")

278 innervalue = self._GetValueByPathExpression(self._ToPath(path))

279# print(f"_ResolveVariables: innervalue='{innervalue}'")

280 rawValue = rawValue[beginPos:nextPos] + str(innervalue) + rawValue[endPos + 1:]

281# print(f"_ResolveVariables: new rawValue='{rawValue}'")

282 else:

283 path = rawValue[beginPos+2:endPos]

284 rawValue = rawValue[endPos+1:]

285 result += str(self._GetValueByPathExpression(self._ToPath(path)))

286

287 return result

288

289 def _GetValueByPathExpression(self, path: list[KeyT]) -> ValueT:

290 """

291 Return the value the given path refers to.

292

293 :param path: Path elements, where ``..`` selects the parent node.

294 :returns: The scalar value at that path.

295 :raises KeyNotFoundError: If a path element doesn't exist.

296 :raises PathExpressionError: If the path resolves to a null value. |br|

297 A variable can't be replaced by a value the document doesn't state.

298 :raises PathExpressionError: If the path resolves to a node instead of a value - extend the path expression

299 to address a scalar value - or if a ``..`` element is applied to the root node,

300 which has no parent.

301 """

302 node = self

303 for p in path:

304 if p == "..":

305 if node._parent is None:

306 pathExpression = ":".join(str(element) for element in path)

307 ex = PathExpressionError(f"Path expression '{pathExpression}' navigates beyond the root node.")

308 ex.add_note("Element '..' was applied to the root node, which has no parent node.")

309 raise ex

310

311 node = node._parent

312 else:

313 node = node._GetNodeOrValue(p)

314

315 if node is None:

316 pathExpression = ":".join(str(element) for element in path)

317 ex = PathExpressionError(f"Path expression '{pathExpression}' resolves to a null value.")

318 ex.add_note("A variable can't be replaced by a value the document doesn't state.")

319 raise ex

320 elif isinstance(node, Dictionary):

321 pathExpression = ":".join(str(element) for element in path)

322 ex = PathExpressionError(f"Path expression '{pathExpression}' resolves to a dictionary, not to a value.")

323 ex.add_note(f"Element '{p}' is a dictionary. Extend the path expression to address a scalar value.")

324 raise ex

325

326 return node

327

328 def _GetNodeOrValueByPathExpression(self, path: list[KeyT]) -> ValueT:

329 """

330 Return the node or value the given path refers to.

331

332 :param path: Path elements, where ``..`` selects the parent node.

333 :returns: A node or a scalar value at that path.

334 :raises KeyNotFoundError: If a path element doesn't exist.

335 :raises PathExpressionError: If a ``..`` element is applied to the root node, which has no parent.

336 """

337 node = self

338 for p in path:

339 if p == "..":

340 if node._parent is None: 340 ↛ 346line 340 didn't jump to line 346 because the condition on line 340 was always true

341 pathExpression = ":".join(str(element) for element in path)

342 ex = PathExpressionError(f"Path expression '{pathExpression}' navigates beyond the root node.")

343 ex.add_note("Element '..' was applied to the root node, which has no parent node.")

344 raise ex

345

346 node = node._parent

347 else:

348 node = node._GetNodeOrValue(p)

349

350 return node

351

352

353@export

354class Dictionary(Node, Abstract_Dict):

355 """A dictionary node in a YAML data file."""

356

357

358 def __init__(

359 self,

360 root: Configuration,

361 parent: Nullable[NodeT],

362 key: KeyT,

363 yamlNode: CommentedMap

364 ) -> None:

365 """

366 Initializes a YAML dictionary.

367

368 :param root: Reference to the root node.

369 :param parent: Reference to the parent node, or ``None`` for the root node.

370 :param key: Key of the node within its parent.

371 :param yamlNode: Reference to the YAML node.

372 """

373 keys: list[KeyT] = [str(k) for k in yamlNode.keys()]

374

375 Node.__init__(self, root, parent, key, yamlNode)

376 Abstract_Dict.__init__(self, keys)

377

378 def __iter__(self) -> typing_Iterator[ValueT]:

379 """

380 Returns an iterator to iterate dictionary keys.

381

382 :returns: Dictionary key iterator.

383 """

384

385 class Iterator(metaclass=ExtendedType, slots=True):

386 """Iterator to iterate dictionary items."""

387

388 _iter: typing_Iterator[ValueT] #: Iterator over the underlying dictionary's keys.

389 _obj: Dictionary #: The dictionary being iterated.

390

391 def __init__(self, obj: Dictionary) -> None:

392 """

393 Initializes an iterator for a YAML dictionary node.

394

395 :param obj: YAML dictionary to iterate.

396 """

397 self._iter = iter(obj._keys)

398 self._obj = obj

399

400 def __iter__(self) -> Self:

401 """

402 Return itself to fulfil the iterator protocol.

403

404 :returns: Itself.

405 """

406 return self # pragma: no cover

407

408 def __next__(self) -> ValueT:

409 """

410 Returns the next item in the dictionary.

411

412 :returns: Next item.

413 """

414 key = next(self._iter)

415 return self._obj[key]

416

417 return Iterator(self)

418

419

420@export

421class Sequence(Node, Abstract_Seq):

422 """A sequence node (ordered list) in a YAML data file."""

423

424 def __init__(

425 self,

426 root: Configuration,

427 parent: NodeT,

428 key: KeyT,

429 yamlNode: CommentedSeq

430 ) -> None:

431 """

432 Initializes a YAML sequence (list).

433

434 :param root: Reference to the root node.

435 :param parent: Reference to the parent node.

436 :param key: Key of the node within its parent.

437 :param yamlNode: Reference to the YAML node.

438 """

439 Node.__init__(self, root, parent, key, yamlNode)

440

441 self._length = len(yamlNode)

442

443 def __iter__(self) -> typing_Iterator[ValueT]:

444 """

445 Returns an iterator to iterate items in the sequence of sub-nodes.

446

447 :returns: Iterator to iterate items in a sequence.

448 """

449 class Iterator(metaclass=ExtendedType, slots=True):

450 """Iterator to iterate sequence items."""

451

452 _i: int #: internal iterator position

453 _obj: Sequence #: Sequence object to iterate

454

455 def __init__(self, obj: Sequence) -> None:

456 """

457 Initializes an iterator for a YAML sequence node.

458

459 :param obj: YAML sequence to iterate.

460 """

461 self._i = 0

462 self._obj = obj

463

464 def __iter__(self) -> Self:

465 """

466 Return itself to fulfil the iterator protocol.

467

468 :returns: Itself.

469 """

470 return self # pragma: no cover

471

472 def __next__(self) -> ValueT:

473 """

474 Returns the next item in the sequence.

475

476 :returns: Next item.

477 :raises StopIteration: If end of sequence is reached.

478 """

479 if self._i >= len(self._obj):

480 raise StopIteration

481

482 result = self._obj[str(self._i)]

483 self._i += 1

484 return result

485

486 return Iterator(self)

487

488

489setattr(Node, "DICT_TYPE", Dictionary)

490setattr(Node, "SEQ_TYPE", Sequence)

491

492

493@export

494class Configuration(Dictionary, Abstract_Configuration):

495 """A configuration read from a YAML file."""

496

497 _yamlConfig: YAML #: The parsed YAML document this configuration is based on.

498

499 def __init__(self, configFile: Path) -> None:

500 """

501 Initializes a configuration instance that reads a YAML file as input.

502

503 All sequence items or dictionaries key-value-pairs in the YAML file are accessible via Python's dictionary syntax.

504

505 A configuration's root **is** a mapping - this class derives from :class:`Dictionary` - so a document

506 describing anything else is rejected here rather than failing on the first access. A document with **no

507 content** carries no settings and reads as an empty configuration; :mod:`pyTooling.Configuration.JSON` does

508 the same, so both formats answer alike.

509

510 :param configFile: Configuration file to read and parse.

511 :raises ConfigurationError: If the YAML file doesn't exist.

512 :raises ConfigurationError: If the YAML file's root isn't a mapping. |br|

513 An empty file and an explicit ``null`` are the exception: both are *no settings*

514 and read as an empty configuration.

515 """

516 if not configFile.exists():

517 raise ConfigurationError(f"YAML configuration file '{configFile}' not found.") from FileNotFoundError(configFile)

518

519 with configFile.open("r", encoding="utf-8") as file:

520 document = YAML().load(file)

521

522 # 'load' returns None for an empty file and for an explicit 'null' - a *null document*, which is valid YAML.

523 if document is None:

524 document = CommentedMap()

525 elif not isinstance(document, CommentedMap):

526 ex = ConfigurationError(f"YAML configuration file '{configFile}' doesn't describe a mapping.")

527 ex.add_note(f"Got type '{getFullyQualifiedName(document)}' at the document's root.")

528 ex.add_note("A configuration needs a mapping of keys to values at its root.")

529 raise ex

530

531 self._yamlConfig = document

532

533 Dictionary.__init__(self, self, None, None, self._yamlConfig)

534 Abstract_Configuration.__init__(self, configFile)

535

536 def __getitem__(self, key: str) -> ValueT:

537 """

538 Access a configuration node by key.

539

540 :param key: The key to look for.

541 :returns: A node (sequence or dictionary) or scalar value (int, float, str).

542 """

543 return self._GetNodeOrValue(str(key))

544

545 #

546 # :param key: Key of the value to write.

547 # :param value: The new value.

548 # :raises NotImplementedError: Writing a configuration is not supported by this implementation.

549 # """

550 # raise NotImplementedError()