Coverage for pyTooling/Configuration/YAML.py: 97%

1# ==================================================================================================================== # 

2# _____ _ _ ____ __ _ _ _ # 

3# _ __ _ |_ _|__ ___ | (_)_ __ __ _ / ___|___ _ __ / _(_) __ _ _ _ _ __ __ _| |_(_) ___ _ __ # 

4# | '_ \| | | || |/ _ \ / _ \| | | '_ \ / _` || | / _ \| '_ \| |_| |/ _` | | | | '__/ _` | __| |/ _ \| '_ \ # 

5# | |_) | |_| || | (_) | (_) | | | | | | (_| || |__| (_) | | | | _| | (_| | |_| | | | (_| | |_| | (_) | | | | # 

6# | .__/ \__, ||_|\___/ \___/|_|_|_| |_|\__, (_)____\___/|_| |_|_| |_|\__, |\__,_|_| \__,_|\__|_|\___/|_| |_| # 

7# |_| |___/ |___/ |___/ # 

8# ==================================================================================================================== # 

9# Authors: # 

10# Patrick Lehmann # 

11# # 

12# License: # 

13# ==================================================================================================================== # 

14# Copyright 2021-2026 Patrick Lehmann - Bötzingen, Germany # 

15# # 

16# Licensed under the Apache License, Version 2.0 (the "License"); # 

17# you may not use this file except in compliance with the License. # 

18# You may obtain a copy of the License at # 

19# # 

20# http://www.apache.org/licenses/LICENSE-2.0 # 

21# # 

22# Unless required by applicable law or agreed to in writing, software # 

23# distributed under the License is distributed on an "AS IS" BASIS, # 

24# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 

25# See the License for the specific language governing permissions and # 

26# limitations under the License. # 

27# # 

28# SPDX-License-Identifier: Apache-2.0 # 

29# ==================================================================================================================== # 

30# 

31""" 

32Configuration reader for YAML files. 

33 

34:raises MissingDependencyError: If the 'yaml' extra isn't installed. 

35 

36.. hint:: 

37 

38 See :ref:`high-level help <CONFIG/FileFormat/YAML>` for explanations and usage examples. 

39""" 

40from __future__ import annotations 

41 

42from pathlib import Path 

43from datetime import date, datetime 

44from typing import Any, Union, Iterator as typing_Iterator, Optional as Nullable, Self 

45 

46from pyTooling.Exceptions import MissingDependencyError 

47 

48try: 

49 from ruamel.yaml import YAML, CommentedMap, CommentedSeq 

50except ImportError as ex: # pragma: no cover 

51 raise MissingDependencyError(dependency="ruamel.yaml", extra="yaml") from ex 

52 

53from pyTooling.Common import getFullyQualifiedName 

54from pyTooling.Decorators import export, InheritDocString 

55from pyTooling.MetaClasses import ExtendedType 

56from pyTooling.Configuration import ConfigurationError, KeyT, NodeT, ValueT, InterpolationError, KeyNotFoundError, \ 

57 PathExpressionError, UnsupportedValueTypeError, Node as Abstract_Node, \ 

58 Dictionary as Abstract_Dict, Sequence as Abstract_Seq, \ 

59 Configuration as Abstract_Configuration 

60 

61 

62@export 

63class Node(Abstract_Node): 

64 """ 

65 Node in a YAML configuration data structure. 

66 """ 

67 

68 _yamlNode: Union[CommentedMap, CommentedSeq] #: Reference to the associated YAML node. 

69 _cache: dict[str, ValueT] #: Cache of already converted sub-nodes and values, by key. 

70 _key: KeyT #: Key of this node. 

71 

72 def __init__( 

73 self, 

74 root: Configuration, 

75 parent: Nullable[NodeT], 

76 key: KeyT, 

77 yamlNode: Union[CommentedMap, CommentedSeq] 

78 ) -> None: 

79 """ 

80 Initializes a YAML node. 

81 

82 :param root: Reference to the root node. 

83 :param parent: Reference to the parent node, or ``None`` for the root node. 

84 :param key: Key of the node within its parent. 

85 :param yamlNode: Reference to the YAML node. 

86 """ 

87 Abstract_Node.__init__(self, root, parent) 

88 

89 self._yamlNode = yamlNode 

90 self._cache = {} 

91 self._key = key 

92 

93 @InheritDocString(Abstract_Node) 

94 def __len__(self) -> int: 

95 return len(self._yamlNode) 

96 

97 @InheritDocString(Abstract_Node) 

98 def __getitem__(self, key: KeyT) -> ValueT: 

99 return self._GetNodeOrValue(str(key)) 

100 

101 @property 

102 def Key(self) -> KeyT: 

103 """ 

104 Property to access the node's key. 

105 

106 :returns: Key of the node. 

107 :raises NotImplementedError: If a new key is assigned; renaming a key is not supported by this configuration 

108 implementation. 

109 """ 

110 return self._key 

111 

112 @Key.setter 

113 def Key(self, value: KeyT) -> None: 

114 raise NotImplementedError("Renaming a key isn't supported by this configuration implementation.") 

115 

116 @InheritDocString(Abstract_Node) 

117 def QueryPath(self, query: str) -> ValueT: 

118 path = self._ToPath(query) 

119 return self._GetNodeOrValueByPathExpression(path) 

120 

121 @staticmethod 

122 def _ToPath(query: str) -> list[Union[str, int]]: 

123 """ 

124 Split a path expression into its elements. 

125 

126 :param query: Path expression, with its elements separated by ``:``. 

127 :returns: List of keys and indices. 

128 """ 

129 return query.split(":") 

130 

131 def _LookupKey(self, key: str) -> Any: 

132 """ 

133 Look up a key in the YAML node, trying it as string, integer and float. 

134 

135 :param key: Key or index to look up. 

136 :returns: The raw value as returned by the YAML parser. 

137 :raises KeyNotFoundError: If the key exists neither as string, nor as integer or float. 

138 """ 

139 try: 

140 return self._yamlNode[key] 

141 except (KeyError, TypeError): 

142 pass 

143 

144 for conversion in (int, float): 

145 try: 

146 convertedKey = conversion(key) 

147 except ValueError: 

148 continue 

149 

150 try: 

151 return self._yamlNode[convertedKey] 

152 except (KeyError, IndexError, TypeError): 

153 pass 

154 

155 ex = KeyNotFoundError(f"Key '{key}' not found in node '{self._key}'.", key=key) 

156 ex.add_note(self._DescribeKeys()) 

157 raise ex 

158 

159 def _DescribeKeys(self) -> str: 

160 """ 

161 Describe the keys or indices offered by this node, so it can be used as an exception note. 

162 

163 :returns: A one-line description of the node's keys or index range. 

164 """ 

165 if isinstance(self._yamlNode, CommentedMap): 

166 if len(self._yamlNode) == 0: 

167 return f"Node '{self._key}' is an empty dictionary." 

168 

169 keys = "', '".join(str(key) for key in self._yamlNode) 

170 return f"Available keys: '{keys}'." 

171 else: 

172 if len(self._yamlNode) == 0: 

173 return f"Node '{self._key}' is an empty sequence." 

174 

175 return f"Node '{self._key}' is a sequence with indices 0..{len(self._yamlNode) - 1}." 

176 

177 def _GetNodeOrValue(self, key: str) -> ValueT: 

178 """ 

179 Return a sub-node or a value by key, converting it on first access. 

180 

181 The converted object is cached, so a second access returns the same node object rather than a new one. 

182 

183 A date or a datetime is a scalar too - YAML reads ``2026-09-02`` and ``2026-09-02T10:30:00`` as those - and is 

184 returned in ISO-8601 spelling. Like every other scalar it comes back as a :class:`str`, so 

185 :meth:`datetime.date.fromisoformat` turns it back into a date where a caller wants one. 

186 

187 :param key: Key or index to look up. 

188 :returns: A dictionary node, a sequence node, a scalar value with its variables 

189 resolved, or ``None`` for a null value. 

190 :raises KeyNotFoundError: If the key doesn't exist in this node. 

191 :raises UnsupportedValueTypeError: If the YAML parser returned a value that is neither a scalar, nor a 

192 node. 

193 """ 

194 try: 

195 value = self._cache[key] 

196 except KeyError: 

197 value = self._LookupKey(key) 

198 

199 if value is None: 

200 pass 

201 elif isinstance(value, str): 

202 value = self._ResolveVariables(value) 

203 elif isinstance(value, (int, float)): 

204 value = str(value) 

205 # YAML reads an unquoted '2026-09-02' as a date and '2026-09-02T10:30:00' as a datetime, which no other 

206 # scalar type covers. 'isoformat' rather than 'str', because 'str' writes a datetime with a space where 

207 # ISO-8601 writes a 'T'. JSON has no date type, so its backend needs no such branch. 

208 elif isinstance(value, (date, datetime)): 

209 value = value.isoformat() 

210 elif isinstance(value, CommentedMap): 

211 value = self.DICT_TYPE(self._root, self, key, value) 

212 elif isinstance(value, CommentedSeq): 

213 value = self.SEQ_TYPE(self._root, self, key, value) 

214 else: 

215 typeName = getFullyQualifiedName(value) 

216 ex = UnsupportedValueTypeError( 

217 f"Unsupported type '{typeName}' for key '{key}' in node '{self._key}'.", 

218 value=value 

219 ) 

220 ex.add_note("The YAML parser returned a value that is neither a scalar (str, int, float, date, " 

221 "datetime), nor a map or sequence.") 

222 raise ex 

223 

224 self._cache[key] = value 

225 

226 return value 

227 

228 def _ResolveVariables(self, value: str) -> str: 

229 """ 

230 Resolve the ``${...}`` variables inside a value. 

231 

232 A variable references another node by a path expression, so a value can be composed from other values of the 

233 same configuration. 

234 

235 :param value: The raw value, possibly containing variables. 

236 :returns: The value with every variable replaced by what it references. 

237 :raises InterpolationError: If a variable is malformed - a dangling ``$`` at the end of the value, or a 

238 missing closing ``}`` for a ``${`` at some position. |br| 

239 Use ``$$`` to escape a literal dollar sign. 

240 :raises KeyNotFoundError: If a referenced key doesn't exist. 

241 """ 

242 if value == "": 242 ↛ 243line 242 didn't jump to line 243 because the condition on line 242 was never true

243 return "" 

244 elif "$" not in value: 

245 return value 

246 

247 rawValue = value 

248 result = "" 

249 

250 while (len(rawValue) > 0): 

251# print(f"_ResolveVariables: LOOP rawValue='{rawValue}'") 

252 beginPos = rawValue.find("$") 

253 if beginPos < 0: 

254 result += rawValue 

255 rawValue = "" 

256 else: 

257 result += rawValue[:beginPos] 

258 if beginPos + 1 >= len(rawValue): 

259 ex = InterpolationError(f"Dangling '$' at the end of value '{value}'.", value=value, position=beginPos) 

260 ex.add_note("Use '$$' to escape a literal dollar sign.") 

261 raise ex 

262 elif rawValue[beginPos + 1] == "$": 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true

263 result += "$" 

264 rawValue = rawValue[1:] 

265 elif rawValue[beginPos + 1] == "{": 265 ↛ 250line 265 didn't jump to line 250 because the condition on line 265 was always true

266 endPos = rawValue.find("}", beginPos) 

267 nextPos = rawValue.rfind("$", beginPos, endPos) 

268 if endPos < 0: 

269 ex = InterpolationError(f"Unclosed variable reference in value '{value}'.", value=value, position=beginPos) 

270 ex.add_note(f"Missing closing '}}' for the '${{' at position {beginPos}.") 

271 raise ex 

272 

273 if (nextPos > 0) and (nextPos < endPos): # an embedded $-sign 

274 path = rawValue[nextPos+2:endPos] 

275# print(f"_ResolveVariables: path='{path}'") 

276 innervalue = self._GetValueByPathExpression(self._ToPath(path)) 

277# print(f"_ResolveVariables: innervalue='{innervalue}'") 

278 rawValue = rawValue[beginPos:nextPos] + str(innervalue) + rawValue[endPos + 1:] 

279# print(f"_ResolveVariables: new rawValue='{rawValue}'") 

280 else: 

281 path = rawValue[beginPos+2:endPos] 

282 rawValue = rawValue[endPos+1:] 

283 result += str(self._GetValueByPathExpression(self._ToPath(path))) 

284 

285 return result 

286 

287 def _GetValueByPathExpression(self, path: list[KeyT]) -> ValueT: 

288 """ 

289 Return the value the given path refers to. 

290 

291 :param path: Path elements, where ``..`` selects the parent node. 

292 :returns: The scalar value at that path. 

293 :raises KeyNotFoundError: If a path element doesn't exist. 

294 :raises PathExpressionError: If the path resolves to a null value. |br| 

295 A variable can't be replaced by a value the document doesn't state. 

296 :raises PathExpressionError: If the path resolves to a node instead of a value - extend the path expression 

297 to address a scalar value - or if a ``..`` element is applied to the root node, 

298 which has no parent. 

299 """ 

300 node = self 

301 for p in path: 

302 if p == "..": 

303 if node._parent is None: 

304 pathExpression = ":".join(str(element) for element in path) 

305 ex = PathExpressionError( 

306 f"Path expression '{pathExpression}' navigates beyond the root node.", 

307 pathExpression=pathExpression 

308 ) 

309 ex.add_note("Element '..' was applied to the root node, which has no parent node.") 

310 raise ex 

311 

312 node = node._parent 

313 else: 

314 node = node._GetNodeOrValue(p) 

315 

316 if node is None: 

317 pathExpression = ":".join(str(element) for element in path) 

318 ex = PathExpressionError( 

319 f"Path expression '{pathExpression}' resolves to a null value.", 

320 pathExpression=pathExpression 

321 ) 

322 ex.add_note("A variable can't be replaced by a value the document doesn't state.") 

323 raise ex 

324 elif isinstance(node, Dictionary): 

325 pathExpression = ":".join(str(element) for element in path) 

326 ex = PathExpressionError( 

327 f"Path expression '{pathExpression}' resolves to a dictionary, not to a value.", 

328 pathExpression=pathExpression 

329 ) 

330 ex.add_note(f"Element '{p}' is a dictionary. Extend the path expression to address a scalar value.") 

331 raise ex 

332 

333 return node 

334 

335 def _GetNodeOrValueByPathExpression(self, path: list[KeyT]) -> ValueT: 

336 """ 

337 Return the node or value the given path refers to. 

338 

339 :param path: Path elements, where ``..`` selects the parent node. 

340 :returns: A node or a scalar value at that path. 

341 :raises KeyNotFoundError: If a path element doesn't exist. 

342 :raises PathExpressionError: If a ``..`` element is applied to the root node, which has no parent. 

343 """ 

344 node = self 

345 for p in path: 

346 if p == "..": 

347 if node._parent is None: 347 ↛ 356line 347 didn't jump to line 356 because the condition on line 347 was always true

348 pathExpression = ":".join(str(element) for element in path) 

349 ex = PathExpressionError( 

350 f"Path expression '{pathExpression}' navigates beyond the root node.", 

351 pathExpression=pathExpression 

352 ) 

353 ex.add_note("Element '..' was applied to the root node, which has no parent node.") 

354 raise ex 

355 

356 node = node._parent 

357 else: 

358 node = node._GetNodeOrValue(p) 

359 

360 return node 

361 

362 

363@export 

364class Dictionary(Node, Abstract_Dict): 

365 """A dictionary node in a YAML data file.""" 

366 

367 

368 def __init__( 

369 self, 

370 root: Configuration, 

371 parent: Nullable[NodeT], 

372 key: KeyT, 

373 yamlNode: CommentedMap 

374 ) -> None: 

375 """ 

376 Initializes a YAML dictionary. 

377 

378 :param root: Reference to the root node. 

379 :param parent: Reference to the parent node, or ``None`` for the root node. 

380 :param key: Key of the node within its parent. 

381 :param yamlNode: Reference to the YAML node. 

382 """ 

383 keys: list[KeyT] = [str(k) for k in yamlNode.keys()] 

384 

385 Node.__init__(self, root, parent, key, yamlNode) 

386 Abstract_Dict.__init__(self, keys) 

387 

388 def __iter__(self) -> typing_Iterator[ValueT]: 

389 """ 

390 Returns an iterator to iterate dictionary keys. 

391 

392 :returns: Dictionary key iterator. 

393 """ 

394 

395 class Iterator(metaclass=ExtendedType, slots=True): 

396 """Iterator to iterate dictionary items.""" 

397 

398 _iter: typing_Iterator[ValueT] #: Iterator over the underlying dictionary's keys. 

399 _obj: Dictionary #: The dictionary being iterated. 

400 

401 def __init__(self, obj: Dictionary) -> None: 

402 """ 

403 Initializes an iterator for a YAML dictionary node. 

404 

405 :param obj: YAML dictionary to iterate. 

406 """ 

407 self._iter = iter(obj._keys) 

408 self._obj = obj 

409 

410 def __iter__(self) -> Self: 

411 """ 

412 Return itself to fulfil the iterator protocol. 

413 

414 :returns: Itself. 

415 """ 

416 return self # pragma: no cover 

417 

418 def __next__(self) -> ValueT: 

419 """ 

420 Returns the next item in the dictionary. 

421 

422 :returns: Next item. 

423 """ 

424 key = next(self._iter) 

425 return self._obj[key] 

426 

427 return Iterator(self) 

428 

429 

430@export 

431class Sequence(Node, Abstract_Seq): 

432 """A sequence node (ordered list) in a YAML data file.""" 

433 

434 def __init__( 

435 self, 

436 root: Configuration, 

437 parent: NodeT, 

438 key: KeyT, 

439 yamlNode: CommentedSeq 

440 ) -> None: 

441 """ 

442 Initializes a YAML sequence (list). 

443 

444 :param root: Reference to the root node. 

445 :param parent: Reference to the parent node. 

446 :param key: Key of the node within its parent. 

447 :param yamlNode: Reference to the YAML node. 

448 """ 

449 Node.__init__(self, root, parent, key, yamlNode) 

450 

451 

452 def __iter__(self) -> typing_Iterator[ValueT]: 

453 """ 

454 Returns an iterator to iterate items in the sequence of sub-nodes. 

455 

456 :returns: Iterator to iterate items in a sequence. 

457 """ 

458 class Iterator(metaclass=ExtendedType, slots=True): 

459 """Iterator to iterate sequence items.""" 

460 

461 _i: int #: internal iterator position 

462 _obj: Sequence #: Sequence object to iterate 

463 

464 def __init__(self, obj: Sequence) -> None: 

465 """ 

466 Initializes an iterator for a YAML sequence node. 

467 

468 :param obj: YAML sequence to iterate. 

469 """ 

470 self._i = 0 

471 self._obj = obj 

472 

473 def __iter__(self) -> Self: 

474 """ 

475 Return itself to fulfil the iterator protocol. 

476 

477 :returns: Itself. 

478 """ 

479 return self # pragma: no cover 

480 

481 def __next__(self) -> ValueT: 

482 """ 

483 Returns the next item in the sequence. 

484 

485 :returns: Next item. 

486 :raises StopIteration: If end of sequence is reached. 

487 """ 

488 if self._i >= len(self._obj): 

489 raise StopIteration 

490 

491 result = self._obj[str(self._i)] 

492 self._i += 1 

493 return result 

494 

495 return Iterator(self) 

496 

497 

498setattr(Node, "DICT_TYPE", Dictionary) 

499setattr(Node, "SEQ_TYPE", Sequence) 

500 

501 

502@export 

503class Configuration(Dictionary, Abstract_Configuration): 

504 """A configuration read from a YAML file.""" 

505 

506 _yamlConfig: YAML #: The parsed YAML document this configuration is based on. 

507 

508 def __init__(self, configFile: Path) -> None: 

509 """ 

510 Initializes a configuration instance that reads a YAML file as input. 

511 

512 All sequence items or dictionaries key-value-pairs in the YAML file are accessible via Python's dictionary syntax. 

513 

514 A configuration's root **is** a mapping - this class derives from :class:`Dictionary` - so a document 

515 describing anything else is rejected here rather than failing on the first access. A document with **no 

516 content** carries no settings and reads as an empty configuration; :mod:`pyTooling.Configuration.JSON` does 

517 the same, so both formats answer alike. 

518 

519 :param configFile: Configuration file to read and parse. 

520 :raises ConfigurationError: If the YAML file doesn't exist. 

521 :raises ConfigurationError: If the YAML file's root isn't a mapping. |br| 

522 An empty file and an explicit ``null`` are the exception: both are *no settings* 

523 and read as an empty configuration. 

524 """ 

525 if not configFile.exists(): 

526 raise ConfigurationError(f"YAML configuration file '{configFile}' not found.") from FileNotFoundError(configFile) 

527 

528 with configFile.open("r", encoding="utf-8") as file: 

529 document = YAML().load(file) 

530 

531 # 'load' returns None for an empty file and for an explicit 'null' - a *null document*, which is valid YAML. 

532 if document is None: 

533 document = CommentedMap() 

534 elif not isinstance(document, CommentedMap): 

535 ex = ConfigurationError(f"YAML configuration file '{configFile}' doesn't describe a mapping.") 

536 ex.add_note(f"Got type '{getFullyQualifiedName(document)}' at the document's root.") 

537 ex.add_note("A configuration needs a mapping of keys to values at its root.") 

538 raise ex 

539 

540 self._yamlConfig = document 

541 

542 Dictionary.__init__(self, self, None, None, self._yamlConfig) 

543 Abstract_Configuration.__init__(self, configFile) 

544 

545 def __getitem__(self, key: str) -> ValueT: 

546 """ 

547 Access a configuration node by key. 

548 

549 :param key: The key to look for. 

550 :returns: A node (sequence or dictionary) or scalar value (int, float, str). 

551 """ 

552 return self._GetNodeOrValue(str(key)) 

553 

554 # 

555 # :param key: Key of the value to write. 

556 # :param value: The new value. 

557 # :raises NotImplementedError: Writing a configuration is not supported by this implementation. 

558 # """ 

559 # raise NotImplementedError()